Merge tag 'dmaengine-fix2-5.11' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine
Pull dmaengine fixes from Vinod Koul:
"Some late fixes for dmaengine:
Core:
- fix channel device_node deletion
Driver fixes:
- dw: revert of runtime pm enabling
- idxd: device state fix, interrupt completion and list corruption
- ti: resource leak
* tag 'dmaengine-fix2-5.11' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine:
dmaengine dw: Revert "dmaengine: dw: Enable runtime PM"
dmaengine: idxd: check device state before issue command
dmaengine: ti: k3-udma: Fix a resource leak in an error handling path
dmaengine: move channel device_node deletion to driver
dmaengine: idxd: fix misc interrupt completion
dmaengine: idxd: Fix list corruption in description completion
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 962cbb5..fe6a460 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -1110,7 +1110,6 @@
"%s called while %d clients hold a reference\n",
__func__, chan->client_count);
mutex_lock(&dma_list_mutex);
- list_del(&chan->device_node);
device->chancnt--;
chan->dev->chan = NULL;
mutex_unlock(&dma_list_mutex);
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index 19a2376..7ab83fe 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -982,11 +982,8 @@
dev_vdbg(chan2dev(chan), "%s\n", __func__);
- pm_runtime_get_sync(dw->dma.dev);
-
/* ASSERT: channel is idle */
if (dma_readl(dw, CH_EN) & dwc->mask) {
- pm_runtime_put_sync_suspend(dw->dma.dev);
dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
return -EIO;
}
@@ -1003,7 +1000,6 @@
* We need controller-specific data to set up slave transfers.
*/
if (chan->private && !dw_dma_filter(chan, chan->private)) {
- pm_runtime_put_sync_suspend(dw->dma.dev);
dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
return -EINVAL;
}
@@ -1047,8 +1043,6 @@
if (!dw->in_use)
do_dw_dma_off(dw);
- pm_runtime_put_sync_suspend(dw->dma.dev);
-
dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
}
diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index 95f94a3..84a6ea6 100644
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -398,17 +398,31 @@
return false;
}
+static inline bool idxd_device_is_halted(struct idxd_device *idxd)
+{
+ union gensts_reg gensts;
+
+ gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
+
+ return (gensts.state == IDXD_DEVICE_STATE_HALT);
+}
+
/*
* This is function is only used for reset during probe and will
* poll for completion. Once the device is setup with interrupts,
* all commands will be done via interrupt completion.
*/
-void idxd_device_init_reset(struct idxd_device *idxd)
+int idxd_device_init_reset(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
union idxd_command_reg cmd;
unsigned long flags;
+ if (idxd_device_is_halted(idxd)) {
+ dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
+ return -ENXIO;
+ }
+
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = IDXD_CMD_RESET_DEVICE;
dev_dbg(dev, "%s: sending reset for init.\n", __func__);
@@ -419,6 +433,7 @@
IDXD_CMDSTS_ACTIVE)
cpu_relax();
spin_unlock_irqrestore(&idxd->dev_lock, flags);
+ return 0;
}
static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
@@ -428,6 +443,12 @@
DECLARE_COMPLETION_ONSTACK(done);
unsigned long flags;
+ if (idxd_device_is_halted(idxd)) {
+ dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
+ *status = IDXD_CMDSTS_HW_ERR;
+ return;
+ }
+
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = cmd_code;
cmd.operand = operand;
diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c
index 8ed2773..71fd6e4 100644
--- a/drivers/dma/idxd/dma.c
+++ b/drivers/dma/idxd/dma.c
@@ -205,5 +205,8 @@
void idxd_unregister_dma_channel(struct idxd_wq *wq)
{
- dma_async_device_channel_unregister(&wq->idxd->dma_dev, &wq->dma_chan);
+ struct dma_chan *chan = &wq->dma_chan;
+
+ dma_async_device_channel_unregister(&wq->idxd->dma_dev, chan);
+ list_del(&chan->device_node);
}
diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index 5a50e91..81a0e65 100644
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -326,7 +326,7 @@
void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
/* device control */
-void idxd_device_init_reset(struct idxd_device *idxd);
+int idxd_device_init_reset(struct idxd_device *idxd);
int idxd_device_enable(struct idxd_device *idxd);
int idxd_device_disable(struct idxd_device *idxd);
void idxd_device_reset(struct idxd_device *idxd);
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index 2c051e0..fa04acd 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -335,7 +335,10 @@
int rc;
dev_dbg(dev, "%s entered and resetting device\n", __func__);
- idxd_device_init_reset(idxd);
+ rc = idxd_device_init_reset(idxd);
+ if (rc < 0)
+ return rc;
+
dev_dbg(dev, "IDXD reset complete\n");
if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) {
diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c
index 593a2f6..a60ca11 100644
--- a/drivers/dma/idxd/irq.c
+++ b/drivers/dma/idxd/irq.c
@@ -111,19 +111,14 @@
return IRQ_WAKE_THREAD;
}
-irqreturn_t idxd_misc_thread(int vec, void *data)
+static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
{
- struct idxd_irq_entry *irq_entry = data;
- struct idxd_device *idxd = irq_entry->idxd;
struct device *dev = &idxd->pdev->dev;
union gensts_reg gensts;
- u32 cause, val = 0;
+ u32 val = 0;
int i;
bool err = false;
- cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
- iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
-
if (cause & IDXD_INTC_ERR) {
spin_lock_bh(&idxd->dev_lock);
for (i = 0; i < 4; i++)
@@ -181,7 +176,7 @@
val);
if (!err)
- goto out;
+ return 0;
/*
* This case should rarely happen and typically is due to software
@@ -211,37 +206,58 @@
gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
"FLR" : "system reset");
spin_unlock_bh(&idxd->dev_lock);
+ return -ENXIO;
}
}
- out:
+ return 0;
+}
+
+irqreturn_t idxd_misc_thread(int vec, void *data)
+{
+ struct idxd_irq_entry *irq_entry = data;
+ struct idxd_device *idxd = irq_entry->idxd;
+ int rc;
+ u32 cause;
+
+ cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
+ if (cause)
+ iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
+
+ while (cause) {
+ rc = process_misc_interrupts(idxd, cause);
+ if (rc < 0)
+ break;
+ cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
+ if (cause)
+ iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
+ }
+
idxd_unmask_msix_vector(idxd, irq_entry->id);
return IRQ_HANDLED;
}
-static bool process_fault(struct idxd_desc *desc, u64 fault_addr)
+static inline bool match_fault(struct idxd_desc *desc, u64 fault_addr)
{
/*
* Completion address can be bad as well. Check fault address match for descriptor
* and completion address.
*/
- if ((u64)desc->hw == fault_addr ||
- (u64)desc->completion == fault_addr) {
- idxd_dma_complete_txd(desc, IDXD_COMPLETE_DEV_FAIL);
+ if ((u64)desc->hw == fault_addr || (u64)desc->completion == fault_addr) {
+ struct idxd_device *idxd = desc->wq->idxd;
+ struct device *dev = &idxd->pdev->dev;
+
+ dev_warn(dev, "desc with fault address: %#llx\n", fault_addr);
return true;
}
return false;
}
-static bool complete_desc(struct idxd_desc *desc)
+static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason)
{
- if (desc->completion->status) {
- idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
- return true;
- }
-
- return false;
+ idxd_dma_complete_txd(desc, reason);
+ idxd_free_desc(desc->wq, desc);
}
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
@@ -251,25 +267,25 @@
struct idxd_desc *desc, *t;
struct llist_node *head;
int queued = 0;
- bool completed = false;
unsigned long flags;
+ enum idxd_complete_type reason;
*processed = 0;
head = llist_del_all(&irq_entry->pending_llist);
if (!head)
goto out;
- llist_for_each_entry_safe(desc, t, head, llnode) {
- if (wtype == IRQ_WORK_NORMAL)
- completed = complete_desc(desc);
- else if (wtype == IRQ_WORK_PROCESS_FAULT)
- completed = process_fault(desc, data);
+ if (wtype == IRQ_WORK_NORMAL)
+ reason = IDXD_COMPLETE_NORMAL;
+ else
+ reason = IDXD_COMPLETE_DEV_FAIL;
- if (completed) {
- idxd_free_desc(desc->wq, desc);
+ llist_for_each_entry_safe(desc, t, head, llnode) {
+ if (desc->completion->status) {
+ if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS)
+ match_fault(desc, data);
+ complete_desc(desc, reason);
(*processed)++;
- if (wtype == IRQ_WORK_PROCESS_FAULT)
- break;
} else {
spin_lock_irqsave(&irq_entry->list_lock, flags);
list_add_tail(&desc->list,
@@ -287,42 +303,46 @@
enum irq_work_type wtype,
int *processed, u64 data)
{
- struct list_head *node, *next;
int queued = 0;
- bool completed = false;
unsigned long flags;
+ LIST_HEAD(flist);
+ struct idxd_desc *desc, *n;
+ enum idxd_complete_type reason;
*processed = 0;
+ if (wtype == IRQ_WORK_NORMAL)
+ reason = IDXD_COMPLETE_NORMAL;
+ else
+ reason = IDXD_COMPLETE_DEV_FAIL;
+
+ /*
+ * This lock protects list corruption from access of list outside of the irq handler
+ * thread.
+ */
spin_lock_irqsave(&irq_entry->list_lock, flags);
- if (list_empty(&irq_entry->work_list))
- goto out;
-
- list_for_each_safe(node, next, &irq_entry->work_list) {
- struct idxd_desc *desc =
- container_of(node, struct idxd_desc, list);
-
+ if (list_empty(&irq_entry->work_list)) {
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
- if (wtype == IRQ_WORK_NORMAL)
- completed = complete_desc(desc);
- else if (wtype == IRQ_WORK_PROCESS_FAULT)
- completed = process_fault(desc, data);
+ return 0;
+ }
- if (completed) {
- spin_lock_irqsave(&irq_entry->list_lock, flags);
+ list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) {
+ if (desc->completion->status) {
list_del(&desc->list);
- spin_unlock_irqrestore(&irq_entry->list_lock, flags);
- idxd_free_desc(desc->wq, desc);
(*processed)++;
- if (wtype == IRQ_WORK_PROCESS_FAULT)
- return queued;
+ list_add_tail(&desc->list, &flist);
} else {
queued++;
}
- spin_lock_irqsave(&irq_entry->list_lock, flags);
}
- out:
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
+
+ list_for_each_entry(desc, &flist, list) {
+ if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS)
+ match_fault(desc, data);
+ complete_desc(desc, reason);
+ }
+
return queued;
}
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index 2984604..f474a12 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -2401,7 +2401,8 @@
dev_err(ud->ddev.dev,
"Descriptor pool allocation failed\n");
uc->use_dma_pool = false;
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto err_res_free;
}
uc->use_dma_pool = true;
