Merge tag 'dmaengine-7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine
Pull dmaengine updates from Vinod Koul:
"Core:
- New devm_of_dma_controller_register() API
- Refactor devm_dma_request_chan() API
New Support:
- Loongson Multi-Channel DMA controller support
- Renesas RZ/{T2H,N2H} support
- Dw CV1800B DMA support
- Switchtec DMA engine driver
U pdates:
- Xilinx AXI dma binding conversion
- Renesas CHCTRL register read updates
- AMD MDB Endpoint and non-LL mode Support
- AXI dma handling of SW and HW cyclic transfers termination
- Intel ioatdma and idxd driver updates"
* tag 'dmaengine-7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (62 commits)
dt-bindings: dma: snps,dw-axi-dmac: Add fallback compatible for CV1800B
MAINTAINERS: dmaengine/ti: Remove myself and add Vignesh as maintainer
dmaengine: qcom: Unify user-visible "Qualcomm" name
dt-bindings: dma: qcom,gpi: Document GPI DMA engine for Shikra SoC
dmaengine: qcom: hidma: use sysfs_emit() in sysfs show callbacks
dmaengine: dw-axi-dmac: fix PM for system sleep and channel alloc
dmaengine: dw-axi-dmac: drop redundant DMAC enable in block start
dmaengine: altera-msgdma: Use memcpy_toio for descriptor FIFO writes
dt-bindings: dma: fsl-edma: add dma-channel-mask property description
dmaengine: tegra: Fix burst size calculation
dmaengine: iop32x-adma: Remove a leftover header file
dmaengine: dma-axi-dmac: use DMA pool to manange DMA descriptor
dmaengine: dma-axi-dmac: Drop struct clk from main struct
dmaengine: dma-axi-dmac: Properly free struct axi_dmac_desc
dmaengine: Fix possible use after free
dmaengine: dw-edma: Add spinlock to protect DONE_INT_MASK and ABORT_INT_MASK
dmaengine: dw-edma-pcie: Reject devices without driver data
dmaengine: sh: rz-dmac: Add DMA ACK signal routing support
irqchip/renesas-rzv2h: Add DMA ACK signal routing support
dmaengine: dw-edma: Remove dw_edma_add_irq_mask()
...
diff --git a/Documentation/devicetree/bindings/dma/fsl,edma.yaml b/Documentation/devicetree/bindings/dma/fsl,edma.yaml
index fa4248e..f609038 100644
--- a/Documentation/devicetree/bindings/dma/fsl,edma.yaml
+++ b/Documentation/devicetree/bindings/dma/fsl,edma.yaml
@@ -12,6 +12,9 @@
DMAMUX0 and DMAMUX1, specific DMA request source can only be multiplexed
by any channel of certain group, DMAMUX0 or DMAMUX1, but not both.
+ This binding has an inverted dma-channel-mask definition compared to
+ the common DMA binding for historical reasons.
+
maintainers:
- Peng Fan <peng.fan@nxp.com>
@@ -95,6 +98,12 @@
eDMA are implemented in big endian mode, otherwise in little mode.
type: boolean
+ dma-channel-mask:
+ description: |
+ Bitmask of available DMA channels (inverted definition).
+ Bit semantics: 0 means channel available, 1 means channel unavailable
+ default: 0
+
required:
- "#dma-cells"
- compatible
diff --git a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml
index 0dabe9b..bc093c7 100644
--- a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml
+++ b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml
@@ -14,17 +14,16 @@
maintainers:
- Jon Hunter <jonathanh@nvidia.com>
- Rajesh Gumasta <rgumasta@nvidia.com>
-
-allOf:
- - $ref: dma-controller.yaml#
+ - Akhil R <akhilrajeev@nvidia.com>
properties:
compatible:
oneOf:
- - const: nvidia,tegra186-gpcdma
+ - enum:
+ - nvidia,tegra264-gpcdma
+ - nvidia,tegra186-gpcdma
- items:
- enum:
- - nvidia,tegra264-gpcdma
- nvidia,tegra234-gpcdma
- nvidia,tegra194-gpcdma
- const: nvidia,tegra186-gpcdma
@@ -51,6 +50,14 @@
iommus:
maxItems: 1
+ iommu-map:
+ description:
+ Maps DMA channel numbers to IOMMU stream IDs. A single entry can map all
+ channels when stream IDs are contiguous. In systems where the channels or
+ stream IDs are not contiguous, multiple entries may be needed.
+ minItems: 1
+ maxItems: 32
+
dma-coherent: true
dma-channel-mask:
@@ -60,12 +67,23 @@
- compatible
- reg
- interrupts
- - resets
- - reset-names
- "#dma-cells"
- iommus
- dma-channel-mask
+allOf:
+ - $ref: dma-controller.yaml#
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - nvidia,tegra186-gpcdma
+ then:
+ required:
+ - resets
+ - reset-names
+
additionalProperties: false
examples:
diff --git a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
index 6493a69..0923fb1 100644
--- a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
+++ b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
@@ -23,6 +23,8 @@
- qcom,bam-v1.4.0
# MSM8916, SDM630
- qcom,bam-v1.7.0
+ # Kaanapali
+ - qcom,bam-v2.0.0
- items:
- enum:
# SDM845, SM6115, SM8150, SM8250 and QCM2290
@@ -118,4 +120,23 @@
#dma-cells = <1>;
qcom,ee = <0>;
};
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+
+ dma-controller@1dc4000 {
+ compatible = "qcom,bam-v2.0.0";
+ reg = <0x0 0x01dc4000 0x0 0x22000>;
+ interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+ #dma-cells = <1>;
+ iommus = <&apps_smmu 0xc0 0>, <&apps_smmu 0xc1 0>;
+ qcom,ee = <0>;
+ qcom,num-ees = <4>;
+ num-channels = <20>;
+ qcom,controlled-remotely;
+ };
+ };
...
diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml
index fde1df0..54dca62 100644
--- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml
+++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml
@@ -24,7 +24,9 @@
- qcom,sm6350-gpi-dma
- items:
- enum:
+ - qcom,eliza-gpi-dma
- qcom,glymur-gpi-dma
+ - qcom,hawi-gpi-dma
- qcom,kaanapali-gpi-dma
- qcom,milos-gpi-dma
- qcom,qcm2290-gpi-dma
@@ -35,6 +37,7 @@
- qcom,sc7280-gpi-dma
- qcom,sc8280xp-gpi-dma
- qcom,sdx75-gpi-dma
+ - qcom,shikra-gpi-dma
- qcom,sm6115-gpi-dma
- qcom,sm6375-gpi-dma
- qcom,sm8350-gpi-dma
diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
index 8045147..0a30a45 100644
--- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
+++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
@@ -21,11 +21,12 @@
- enum:
- snps,axi-dma-1.01a
- intel,kmb-axi-dma
- - sophgo,cv1800b-axi-dma
- starfive,jh7110-axi-dma
- starfive,jh8100-axi-dma
- items:
- - const: altr,agilex5-axi-dma
+ - enum:
+ - altr,agilex5-axi-dma
+ - sophgo,cv1800b-axi-dma
- const: snps,axi-dma-1.01a
reg:
diff --git a/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml b/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml
index ec06235..62ce6d8 100644
--- a/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml
+++ b/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml
@@ -14,7 +14,9 @@
properties:
compatible:
- const: spacemit,k1-pdma
+ enum:
+ - spacemit,k1-pdma
+ - spacemit,k3-pdma
reg:
maxItems: 1
diff --git a/MAINTAINERS b/MAINTAINERS
index 364e9ee..9b787bc 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -26697,7 +26697,7 @@
F: sound/soc/codecs/tpa6130a2.*
TEXAS INSTRUMENTS DMA DRIVERS
-M: Peter Ujfalusi <peter.ujfalusi@gmail.com>
+M: Vignesh Raghavendra <vigneshr@ti.com>
L: dmaengine@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c
index b46999c..e23e5b4 100644
--- a/drivers/dma/altera-msgdma.c
+++ b/drivers/dma/altera-msgdma.c
@@ -496,6 +496,11 @@ static void msgdma_copy_one(struct msgdma_device *mdev,
{
void __iomem *hw_desc = mdev->desc;
+ /* Ensure control is the last field — required for correct FIFO flush ordering */
+ static_assert(offsetof(struct msgdma_extended_desc, control) ==
+ sizeof(struct msgdma_extended_desc) - sizeof(u32),
+ "control must be the last field in msgdma_extended_desc");
+
/*
* Check if the DESC FIFO it not full. If its full, we need to wait
* for at least one entry to become free again
@@ -504,17 +509,18 @@ static void msgdma_copy_one(struct msgdma_device *mdev,
MSGDMA_CSR_STAT_DESC_BUF_FULL)
mdelay(1);
+ /* Ensure control is the last field — required for correct FIFO flush ordering */
+ static_assert(offsetof(struct msgdma_extended_desc, control) ==
+ sizeof(struct msgdma_extended_desc) - sizeof(u32),
+ "control must be the last field in msgdma_extended_desc");
+
/*
- * The descriptor needs to get copied into the descriptor FIFO
- * of the DMA controller. The descriptor will get flushed to the
- * FIFO, once the last word (control word) is written. Since we
- * are not 100% sure that memcpy() writes all word in the "correct"
- * order (address from low to high) on all architectures, we make
- * sure this control word is written last by single coding it and
- * adding some write-barriers here.
+ * Copy the descriptor into the descriptor FIFO of the DMA controller,
+ * excluding the control word. The FIFO is flushed and the descriptor
+ * becomes valid once the control word is written last.
*/
- memcpy((void __force *)hw_desc, &desc->hw_desc,
- sizeof(desc->hw_desc) - sizeof(u32));
+ memcpy_toio(hw_desc, &desc->hw_desc,
+ offsetof(struct msgdma_extended_desc, control));
/* Write control word last to flush this descriptor into the FIFO */
mdev->idle = false;
diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c
index 45c2c8e..d47ff27 100644
--- a/drivers/dma/dma-axi-dmac.c
+++ b/drivers/dma/dma-axi-dmac.c
@@ -13,6 +13,7 @@
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
+#include <linux/dmapool.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/io.h>
@@ -147,6 +148,7 @@ struct axi_dmac_chan {
struct virt_dma_chan vchan;
struct axi_dmac_desc *next_desc;
+ void *pool;
struct list_head active_descs;
enum dma_transfer_direction direction;
@@ -170,8 +172,6 @@ struct axi_dmac {
void __iomem *base;
int irq;
- struct clk *clk;
-
struct dma_device dma_dev;
struct axi_dmac_chan chan;
};
@@ -650,11 +650,17 @@ static void axi_dmac_issue_pending(struct dma_chan *c)
spin_unlock_irqrestore(&chan->vchan.lock, flags);
}
+static void axi_dmac_free_desc(struct axi_dmac_desc *desc)
+{
+ for (unsigned int i = 0; i < desc->num_sgs; i++)
+ dma_pool_free(desc->chan->pool, desc->sg[i].hw, desc->sg[i].hw_phys);
+
+ kfree(desc);
+}
+
static struct axi_dmac_desc *
axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
{
- struct axi_dmac *dmac = chan_to_axi_dmac(chan);
- struct device *dev = dmac->dma_dev.dev;
struct axi_dmac_hw_desc *hws;
struct axi_dmac_desc *desc;
dma_addr_t hw_phys;
@@ -666,22 +672,22 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
desc->num_sgs = num_sgs;
desc->chan = chan;
- hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)),
- &hw_phys, GFP_ATOMIC);
- if (!hws) {
- kfree(desc);
- return NULL;
- }
-
for (i = 0; i < num_sgs; i++) {
- desc->sg[i].hw = &hws[i];
- desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws);
+ hws = dma_pool_zalloc(chan->pool, GFP_NOWAIT, &hw_phys);
+ if (!hws) {
+ desc->num_sgs = i;
+ axi_dmac_free_desc(desc);
+ return NULL;
+ }
- hws[i].id = AXI_DMAC_SG_UNUSED;
- hws[i].flags = 0;
+ desc->sg[i].hw = hws;
+ desc->sg[i].hw_phys = hw_phys;
+
+ hws->id = AXI_DMAC_SG_UNUSED;
/* Link hardware descriptors */
- hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws);
+ if (i)
+ desc->sg[i - 1].hw->next_sg_addr = hw_phys;
}
/* The last hardware descriptor will trigger an interrupt */
@@ -690,18 +696,6 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
return desc;
}
-static void axi_dmac_free_desc(struct axi_dmac_desc *desc)
-{
- struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan);
- struct device *dev = dmac->dma_dev.dev;
- struct axi_dmac_hw_desc *hw = desc->sg[0].hw;
- dma_addr_t hw_phys = desc->sg[0].hw_phys;
-
- dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)),
- hw, hw_phys);
- kfree(desc);
-}
-
static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
enum dma_transfer_direction direction, dma_addr_t addr,
unsigned int num_periods, unsigned int period_len,
@@ -769,7 +763,7 @@ axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs,
for (i = 0; i < nb; i++) {
if (!axi_dmac_check_addr(chan, vecs[i].addr) ||
!axi_dmac_check_len(chan, vecs[i].len)) {
- kfree(desc);
+ axi_dmac_free_desc(desc);
return NULL;
}
@@ -935,9 +929,26 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}
+static int axi_dmac_alloc_chan_resources(struct dma_chan *c)
+{
+ struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
+ struct device *dev = c->device->dev;
+
+ chan->pool = dma_pool_create(dev_name(dev), dev,
+ sizeof(struct axi_dmac_hw_desc),
+ __alignof__(struct axi_dmac_hw_desc), 0);
+ if (!chan->pool)
+ return -ENOMEM;
+
+ return 0;
+}
+
static void axi_dmac_free_chan_resources(struct dma_chan *c)
{
+ struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
+
vchan_free_chan_resources(to_virt_chan(c));
+ dma_pool_destroy(chan->pool);
}
static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
@@ -1198,6 +1209,7 @@ static int axi_dmac_probe(struct platform_device *pdev)
{
struct dma_device *dma_dev;
struct axi_dmac *dmac;
+ struct clk *clk;
struct regmap *regmap;
unsigned int version;
u32 irq_mask = 0;
@@ -1217,9 +1229,9 @@ static int axi_dmac_probe(struct platform_device *pdev)
if (IS_ERR(dmac->base))
return PTR_ERR(dmac->base);
- dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL);
- if (IS_ERR(dmac->clk))
- return PTR_ERR(dmac->clk);
+ clk = devm_clk_get_enabled(&pdev->dev, NULL);
+ if (IS_ERR(clk))
+ return PTR_ERR(clk);
version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION);
@@ -1239,6 +1251,7 @@ static int axi_dmac_probe(struct platform_device *pdev)
dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);
+ dma_dev->device_alloc_chan_resources = axi_dmac_alloc_chan_resources;
dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
dma_dev->device_tx_status = dma_cookie_status;
dma_dev->device_issue_pending = axi_dmac_issue_pending;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 405bd2f..9049171 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -905,11 +905,12 @@ void dma_release_channel(struct dma_chan *chan)
mutex_lock(&dma_list_mutex);
WARN_ONCE(chan->client_count != 1,
"chan reference count %d != 1\n", chan->client_count);
- dma_chan_put(chan);
/* drop PRIVATE cap enabled by __dma_request_channel() */
if (--chan->device->privatecnt == 0)
dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
+ dma_chan_put(chan);
+
if (chan->slave) {
sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);
sysfs_remove_link(&chan->slave->kobj, chan->name);
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index df38681..2ae3469 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -137,7 +137,7 @@ struct dmatest_params {
* @did_init: module has been initialized completely
* @last_error: test has faced configuration issues
*/
-static struct dmatest_info {
+struct dmatest_info {
/* Test parameters */
struct dmatest_params params;
@@ -147,7 +147,9 @@ static struct dmatest_info {
int last_error;
struct mutex lock;
bool did_init;
-} test_info = {
+};
+
+static struct dmatest_info test_info = {
.channels = LIST_HEAD_INIT(test_info.channels),
.lock = __MUTEX_INITIALIZER(test_info.lock),
};
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index 4d53f07..bcefaff 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -437,8 +437,6 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,
return;
}
- axi_dma_enable(chan->chip);
-
config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;
config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;
config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC;
@@ -518,11 +516,17 @@ static void dw_axi_dma_synchronize(struct dma_chan *dchan)
static int dma_chan_alloc_chan_resources(struct dma_chan *dchan)
{
struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);
+ int ret;
+
+ ret = pm_runtime_resume_and_get(chan->chip->dev);
+ if (ret < 0)
+ return ret;
/* ASSERT: channel is idle */
if (axi_chan_is_hw_enable(chan)) {
dev_err(chan2dev(chan), "%s is non-idle!\n",
axi_chan_name(chan));
+ pm_runtime_put(chan->chip->dev);
return -EBUSY;
}
@@ -533,12 +537,11 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan)
64, 0);
if (!chan->desc_pool) {
dev_err(chan2dev(chan), "No memory for descriptors\n");
+ pm_runtime_put(chan->chip->dev);
return -ENOMEM;
}
dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan));
- pm_runtime_get(chan->chip->dev);
-
return 0;
}
@@ -1665,6 +1668,8 @@ static void dw_remove(struct platform_device *pdev)
}
static const struct dev_pm_ops dw_axi_dma_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+ pm_runtime_force_resume)
SET_RUNTIME_PM_OPS(axi_dma_runtime_suspend, axi_dma_runtime_resume, NULL)
};
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index c2feb3a..89a4c49 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -988,20 +988,12 @@ static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
}
}
-static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
-{
- while (*mask * alloc < cnt)
- (*mask)++;
-}
-
static int dw_edma_irq_request(struct dw_edma *dw,
u32 *wr_alloc, u32 *rd_alloc)
{
struct dw_edma_chip *chip = dw->chip;
struct device *dev = dw->chip->dev;
struct msi_desc *msi_desc;
- u32 wr_mask = 1;
- u32 rd_mask = 1;
int i, err = 0;
u32 ch_cnt;
int irq;
@@ -1038,9 +1030,6 @@ static int dw_edma_irq_request(struct dw_edma *dw,
dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
}
- dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
- dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
-
for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
irq = chip->ops->irq_vector(dev, i);
err = request_irq(irq,
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 902574b..6474cac 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -109,7 +109,7 @@ struct dw_edma {
struct dw_edma_chan *chan;
- raw_spinlock_t lock; /* Only for legacy */
+ raw_spinlock_t lock; /* Protect v0 shared registers */
struct dw_edma_chip *chip;
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index 0b30ce1..791c46e 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -27,6 +27,7 @@
/* AMD MDB (Xilinx) specific defines */
#define PCI_DEVICE_ID_XILINX_B054 0xb054
+#define PCI_DEVICE_ID_XILINX_B00F 0xb00f
#define DW_PCIE_XILINX_MDB_VSEC_DMA_ID 0x6
#define DW_PCIE_XILINX_MDB_VSEC_ID 0x20
@@ -125,6 +126,19 @@ static const struct dw_edma_pcie_data xilinx_mdb_data = {
.rd_ch_cnt = 8,
};
+static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = {
+ /* MDB registers location */
+ .rg.bar = BAR_0,
+ .rg.off = SZ_4K, /* 4 Kbytes */
+ .rg.sz = SZ_8K, /* 8 Kbytes */
+
+ /* Other */
+ .mf = EDMA_MF_HDMA_NATIVE,
+ .irqs = 1,
+ .wr_ch_cnt = 8,
+ .rd_ch_cnt = 8,
+};
+
static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata,
enum pci_barno bar, off_t start_off,
off_t ll_off_gap, size_t ll_size,
@@ -314,6 +328,9 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
int i, mask;
bool non_ll = false;
+ if (!pdata)
+ return -ENODEV;
+
struct dw_edma_pcie_data *vsec_data __free(kfree) =
kmalloc_obj(*vsec_data);
if (!vsec_data)
@@ -547,6 +564,8 @@ static const struct pci_device_id dw_edma_pcie_id_table[] = {
{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) },
{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B054),
(kernel_ulong_t)&xilinx_mdb_data },
+ { PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B00F),
+ .driver_data = (kernel_ulong_t)&xilinx_cpm6_dma_data },
{ }
};
MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table);
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index 69e8279..cfdd646 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -364,6 +364,7 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
{
struct dw_edma_chan *chan = chunk->chan;
struct dw_edma *dw = chan->dw;
+ unsigned long flags;
u32 tmp;
dw_edma_v0_core_write_chunk(chunk);
@@ -408,6 +409,8 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
}
}
/* Interrupt unmask - done, abort */
+ raw_spin_lock_irqsave(&dw->lock, flags);
+
tmp = GET_RW_32(dw, chan->dir, int_mask);
tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
@@ -416,6 +419,9 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
+
+ raw_spin_unlock_irqrestore(&dw->lock, flags);
+
/* Channel control */
SET_CH_32(dw, chan->dir, chan->id, ch_control1,
(DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c
index 8eceb96..a3395cf 100644
--- a/drivers/dma/ep93xx_dma.c
+++ b/drivers/dma/ep93xx_dma.c
@@ -1587,18 +1587,11 @@ static const struct of_device_id ep93xx_dma_of_ids[] = {
};
MODULE_DEVICE_TABLE(of, ep93xx_dma_of_ids);
-static const struct platform_device_id ep93xx_dma_driver_ids[] = {
- { "ep93xx-dma-m2p", 0 },
- { "ep93xx-dma-m2m", 1 },
- { },
-};
-
static struct platform_driver ep93xx_dma_driver = {
.driver = {
.name = "ep93xx-dma",
.of_match_table = ep93xx_dma_of_ids,
},
- .id_table = ep93xx_dma_driver_ids,
.probe = ep93xx_dma_probe,
};
diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c
index 32a0e95..28bf818 100644
--- a/drivers/dma/hisi_dma.c
+++ b/drivers/dma/hisi_dma.c
@@ -1037,6 +1037,7 @@ static const struct pci_device_id hisi_dma_pci_tbl[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
{ 0, }
};
+MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
static struct pci_driver hisi_dma_pci_driver = {
.name = "hisi_dma",
@@ -1050,4 +1051,3 @@ MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 3d52788..3636883 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -2364,7 +2364,9 @@ static int sdma_probe(struct platform_device *pdev)
return dev_err_probe(&pdev->dev, ret,
"failed to register controller\n");
- spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
+ struct device_node *sdma_parent_np __free(device_node) = of_get_parent(np);
+
+ spba_bus = of_get_compatible_child(sdma_parent_np, "fsl,spba-bus");
ret = of_address_to_resource(spba_bus, 0, &spba_res);
if (!ret) {
sdma->spba_start_addr = spba_res.start;
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index d12e729..386e85c 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -51,7 +51,10 @@
#define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */
#define DCSR_EORINTR BIT(9) /* The end of Receive */
-#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2))
+#define DRCMR_BASE 0x0100
+#define DRCMR_EXT_BASE_K3 0x1000
+#define DRCMR_EXT_BASE_DEFAULT 0x1100
+#define DRCMR_REQ_LIMIT 64
#define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */
#define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */
@@ -154,6 +157,7 @@ struct mmp_pdma_phy {
* @run_bits: Control bits in DCSR register for channel start/stop
* @dma_width: DMA addressing width in bits (32 or 64). Determines the
* DMA mask capability of the controller hardware.
+ * @drcmr_ext_base: Base DRCMR address for extended requests
*/
struct mmp_pdma_ops {
/* Hardware Register Operations */
@@ -174,6 +178,7 @@ struct mmp_pdma_ops {
/* Controller Configuration */
u32 run_bits;
u32 dma_width;
+ u32 drcmr_ext_base;
};
struct mmp_pdma_device {
@@ -195,6 +200,13 @@ struct mmp_pdma_device {
#define to_mmp_pdma_dev(dmadev) \
container_of(dmadev, struct mmp_pdma_device, device)
+static u32 mmp_pdma_get_drcmr(struct mmp_pdma_device *pdev, u32 drcmr)
+{
+ if (drcmr < DRCMR_REQ_LIMIT)
+ return DRCMR_BASE + (drcmr << 2);
+ return pdev->ops->drcmr_ext_base + ((drcmr - DRCMR_REQ_LIMIT) << 2);
+}
+
/* For 32-bit PDMA */
static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr)
{
@@ -301,7 +313,7 @@ static void enable_chan(struct mmp_pdma_phy *phy)
pdev = to_mmp_pdma_dev(phy->vchan->chan.device);
- reg = DRCMR(phy->vchan->drcmr);
+ reg = mmp_pdma_get_drcmr(pdev, phy->vchan->drcmr);
writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
dalgn = readl(phy->base + DALGN);
@@ -437,7 +449,7 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
return;
/* clear the channel mapping in DRCMR */
- reg = DRCMR(pchan->drcmr);
+ reg = mmp_pdma_get_drcmr(pdev, pchan->drcmr);
writel(0, pchan->phy->base + reg);
spin_lock_irqsave(&pdev->phy_lock, flags);
@@ -1179,6 +1191,7 @@ static const struct mmp_pdma_ops marvell_pdma_v1_ops = {
.get_desc_dst_addr = get_desc_dst_addr_32,
.run_bits = (DCSR_RUN),
.dma_width = 32,
+ .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
};
static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
@@ -1192,6 +1205,21 @@ static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
.get_desc_dst_addr = get_desc_dst_addr_64,
.run_bits = (DCSR_RUN | DCSR_LPAEEN),
.dma_width = 64,
+ .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
+};
+
+static const struct mmp_pdma_ops spacemit_k3_pdma_ops = {
+ .write_next_addr = write_next_addr_64,
+ .read_src_addr = read_src_addr_64,
+ .read_dst_addr = read_dst_addr_64,
+ .set_desc_next_addr = set_desc_next_addr_64,
+ .set_desc_src_addr = set_desc_src_addr_64,
+ .set_desc_dst_addr = set_desc_dst_addr_64,
+ .get_desc_src_addr = get_desc_src_addr_64,
+ .get_desc_dst_addr = get_desc_dst_addr_64,
+ .run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN),
+ .dma_width = 64,
+ .drcmr_ext_base = DRCMR_EXT_BASE_K3,
};
static const struct of_device_id mmp_pdma_dt_ids[] = {
@@ -1202,6 +1230,9 @@ static const struct of_device_id mmp_pdma_dt_ids[] = {
.compatible = "spacemit,k1-pdma",
.data = &spacemit_k1_pdma_ops
}, {
+ .compatible = "spacemit,k3-pdma",
+ .data = &spacemit_k3_pdma_ops
+ }, {
/* sentinel */
}
};
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index 334425f..05d7321 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -1486,20 +1486,6 @@ static void nbpf_remove(struct platform_device *pdev)
clk_disable_unprepare(nbpf->clk);
}
-static const struct platform_device_id nbpf_ids[] = {
- {"nbpfaxi64dmac1b4", (kernel_ulong_t)&nbpf_cfg[NBPF1B4]},
- {"nbpfaxi64dmac1b8", (kernel_ulong_t)&nbpf_cfg[NBPF1B8]},
- {"nbpfaxi64dmac1b16", (kernel_ulong_t)&nbpf_cfg[NBPF1B16]},
- {"nbpfaxi64dmac4b4", (kernel_ulong_t)&nbpf_cfg[NBPF4B4]},
- {"nbpfaxi64dmac4b8", (kernel_ulong_t)&nbpf_cfg[NBPF4B8]},
- {"nbpfaxi64dmac4b16", (kernel_ulong_t)&nbpf_cfg[NBPF4B16]},
- {"nbpfaxi64dmac8b4", (kernel_ulong_t)&nbpf_cfg[NBPF8B4]},
- {"nbpfaxi64dmac8b8", (kernel_ulong_t)&nbpf_cfg[NBPF8B8]},
- {"nbpfaxi64dmac8b16", (kernel_ulong_t)&nbpf_cfg[NBPF8B16]},
- {},
-};
-MODULE_DEVICE_TABLE(platform, nbpf_ids);
-
static int nbpf_runtime_suspend(struct device *dev)
{
struct nbpf_device *nbpf = dev_get_drvdata(dev);
@@ -1523,7 +1509,6 @@ static struct platform_driver nbpf_driver = {
.of_match_table = nbpf_match,
.pm = pm_ptr(&nbpf_pm_ops),
},
- .id_table = nbpf_ids,
.probe = nbpf_probe,
.remove = nbpf_remove,
};
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index e9fbfd5..bf805f10 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -970,6 +970,7 @@ static const struct pci_device_id pch_dma_id_table[] = {
{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_DMA2_4CH), 4}, /* SPI */
{ 0, },
};
+MODULE_DEVICE_TABLE(pci, pch_dma_id_table);
static SIMPLE_DEV_PM_OPS(pch_dma_pm_ops, pch_dma_suspend, pch_dma_resume);
@@ -987,4 +988,3 @@ MODULE_DESCRIPTION("Intel EG20T PCH / LAPIS Semicon ML7213/ML7223/ML7831 IOH "
"DMA controller driver");
MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>");
MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, pch_dma_id_table);
diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig
index ace75d7..c71b0b5 100644
--- a/drivers/dma/qcom/Kconfig
+++ b/drivers/dma/qcom/Kconfig
@@ -11,7 +11,7 @@
and on-chip peripheral devices.
config QCOM_BAM_DMA
- tristate "QCOM BAM DMA support"
+ tristate "Qualcomm BAM DMA support"
depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM)
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
@@ -20,7 +20,7 @@
provides DMA capabilities for a variety of on-chip devices.
config QCOM_GPI_DMA
- tristate "Qualcomm Technologies GPI DMA support"
+ tristate "Qualcomm GPI DMA support"
depends on ARCH_QCOM
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
@@ -32,7 +32,7 @@
transfer data between DDR and peripheral.
config QCOM_HIDMA_MGMT
- tristate "Qualcomm Technologies HIDMA Management support"
+ tristate "Qualcomm HIDMA Management support"
depends on HAS_IOMEM
select DMA_ENGINE
help
@@ -44,7 +44,7 @@
host would run the QCOM_HIDMA_MGMT management driver.
config QCOM_HIDMA
- tristate "Qualcomm Technologies HIDMA Channel support"
+ tristate "Qualcomm HIDMA Channel support"
depends on HAS_IOMEM
select DMA_ENGINE
help
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index 1911629..1bb26af 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -199,6 +199,35 @@ static const struct reg_offset_data bam_v1_7_reg_info[] = {
[BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 },
};
+static const struct reg_offset_data bam_v2_0_reg_info[] = {
+ [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 },
+ [BAM_REVISION] = { 0x1000, 0x00, 0x00, 0x00 },
+ [BAM_NUM_PIPES] = { 0x1008, 0x00, 0x00, 0x00 },
+ [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_SRCS] = { 0x3010, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_SRCS_MSK] = { 0x3014, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_SRCS_UNMASKED] = { 0x3018, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 },
+ [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 },
+ [BAM_IRQ_SRCS_EE] = { 0x3000, 0x00, 0x00, 0x1000 },
+ [BAM_IRQ_SRCS_MSK_EE] = { 0x3004, 0x00, 0x00, 0x1000 },
+ [BAM_P_CTRL] = { 0xC000, 0x1000, 0x00, 0x00 },
+ [BAM_P_RST] = { 0xC004, 0x1000, 0x00, 0x00 },
+ [BAM_P_HALT] = { 0xC008, 0x1000, 0x00, 0x00 },
+ [BAM_P_IRQ_STTS] = { 0xC010, 0x1000, 0x00, 0x00 },
+ [BAM_P_IRQ_CLR] = { 0xC014, 0x1000, 0x00, 0x00 },
+ [BAM_P_IRQ_EN] = { 0xC018, 0x1000, 0x00, 0x00 },
+ [BAM_P_EVNT_DEST_ADDR] = { 0xC82C, 0x00, 0x1000, 0x00 },
+ [BAM_P_EVNT_REG] = { 0xC818, 0x00, 0x1000, 0x00 },
+ [BAM_P_SW_OFSTS] = { 0xC800, 0x00, 0x1000, 0x00 },
+ [BAM_P_DATA_FIFO_ADDR] = { 0xC824, 0x00, 0x1000, 0x00 },
+ [BAM_P_DESC_FIFO_ADDR] = { 0xC81C, 0x00, 0x1000, 0x00 },
+ [BAM_P_EVNT_GEN_TRSHLD] = { 0xC828, 0x00, 0x1000, 0x00 },
+ [BAM_P_FIFO_SIZES] = { 0xC820, 0x00, 0x1000, 0x00 },
+};
+
/* BAM CTRL */
#define BAM_SW_RST BIT(0)
#define BAM_EN BIT(1)
@@ -1208,6 +1237,7 @@ static const struct of_device_id bam_of_match[] = {
{ .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
{ .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
{ .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
+ { .compatible = "qcom,bam-v2.0.0", .data = &bam_v2_0_reg_info },
{}
};
diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c
index c9a6f61..a5055a6 100644
--- a/drivers/dma/qcom/gpi.c
+++ b/drivers/dma/qcom/gpi.c
@@ -2260,6 +2260,7 @@ static int gpi_probe(struct platform_device *pdev)
/* clear and Set capabilities */
dma_cap_zero(gpi_dev->dma_device.cap_mask);
dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask);
+ dma_cap_set(DMA_PRIVATE, gpi_dev->dma_device.cap_mask);
/* configure dmaengine apis */
gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index 5a8dca8..7a7f302 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -624,12 +624,10 @@ static ssize_t hidma_show_values(struct device *dev,
{
struct hidma_dev *mdev = dev_get_drvdata(dev);
- buf[0] = 0;
-
if (strcmp(attr->attr.name, "chid") == 0)
- sprintf(buf, "%d\n", mdev->chidx);
+ return sysfs_emit(buf, "%d\n", mdev->chidx);
- return strlen(buf);
+ return 0;
}
static inline void hidma_sysfs_uninit(struct hidma_dev *dev)
diff --git a/drivers/dma/qcom/hidma_mgmt_sys.c b/drivers/dma/qcom/hidma_mgmt_sys.c
index 930eae0..9672ef9 100644
--- a/drivers/dma/qcom/hidma_mgmt_sys.c
+++ b/drivers/dma/qcom/hidma_mgmt_sys.c
@@ -102,15 +102,12 @@ static ssize_t show_values(struct device *dev, struct device_attribute *attr,
struct hidma_mgmt_dev *mdev = dev_get_drvdata(dev);
unsigned int i;
- buf[0] = 0;
-
for (i = 0; i < ARRAY_SIZE(hidma_mgmt_files); i++) {
- if (strcmp(attr->attr.name, hidma_mgmt_files[i].name) == 0) {
- sprintf(buf, "%d\n", hidma_mgmt_files[i].get(mdev));
- break;
- }
+ if (strcmp(attr->attr.name, hidma_mgmt_files[i].name) == 0)
+ return sysfs_emit(buf, "%d\n",
+ hidma_mgmt_files[i].get(mdev));
}
- return strlen(buf);
+ return 0;
}
static ssize_t set_values(struct device *dev, struct device_attribute *attr,
@@ -143,15 +140,15 @@ static ssize_t show_values_channel(struct kobject *kobj,
struct hidma_chan_attr *chattr;
struct hidma_mgmt_dev *mdev;
- buf[0] = 0;
chattr = container_of(attr, struct hidma_chan_attr, attr);
mdev = chattr->mdev;
- if (strcmp(attr->attr.name, "priority") == 0)
- sprintf(buf, "%d\n", mdev->priority[chattr->index]);
- else if (strcmp(attr->attr.name, "weight") == 0)
- sprintf(buf, "%d\n", mdev->weight[chattr->index]);
- return strlen(buf);
+ if (strcmp(attr->attr.name, "priority") == 0)
+ return sysfs_emit(buf, "%d\n", mdev->priority[chattr->index]);
+ else if (strcmp(attr->attr.name, "weight") == 0)
+ return sysfs_emit(buf, "%d\n", mdev->weight[chattr->index]);
+
+ return 0;
}
static ssize_t set_values_channel(struct kobject *kobj,
diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c
index 625ff29..ca76f1bb 100644
--- a/drivers/dma/sh/rz-dmac.c
+++ b/drivers/dma/sh/rz-dmac.c
@@ -18,6 +18,7 @@
#include <linux/irqchip/irq-renesas-rzv2h.h>
#include <linux/irqchip/irq-renesas-rzt2h.h>
#include <linux/list.h>
+#include <linux/lockdep.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
@@ -34,6 +35,7 @@
enum rz_dmac_prep_type {
RZ_DMAC_DESC_MEMCPY,
RZ_DMAC_DESC_SLAVE_SG,
+ RZ_DMAC_DESC_CYCLIC,
};
struct rz_lmdesc {
@@ -58,10 +60,23 @@ struct rz_dmac_desc {
/* For slave sg */
struct scatterlist *sg;
unsigned int sgcount;
+ struct rz_lmdesc *start_lmdesc;
};
#define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd)
+/**
+ * enum rz_dmac_chan_status: RZ DMAC channel status
+ * @RZ_DMAC_CHAN_STATUS_PAUSED: Channel is paused though DMA engine callbacks
+ * @RZ_DMAC_CHAN_STATUS_CYCLIC: Channel is cyclic
+ * @RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL: Channel is paused through driver internal logic
+ */
+enum rz_dmac_chan_status {
+ RZ_DMAC_CHAN_STATUS_PAUSED,
+ RZ_DMAC_CHAN_STATUS_CYCLIC,
+ RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL,
+};
+
struct rz_dmac_chan {
struct virt_dma_chan vc;
void __iomem *ch_base;
@@ -73,13 +88,18 @@ struct rz_dmac_chan {
dma_addr_t src_per_address;
dma_addr_t dst_per_address;
+ unsigned long status;
+
u32 chcfg;
u32 chctrl;
int mid_rid;
+ int dmac_ack;
+
+ struct {
+ u32 nxla;
+ } pm_state;
struct list_head ld_free;
- struct list_head ld_queue;
- struct list_head ld_active;
struct {
struct rz_lmdesc *base;
@@ -99,6 +119,9 @@ struct rz_dmac_icu {
struct rz_dmac_info {
void (*icu_register_dma_req)(struct platform_device *icu_dev,
u8 dmac_index, u8 dmac_channel, u16 req_no);
+ void (*icu_register_dma_ack)(struct platform_device *icu_dev,
+ u8 dmac_index, u8 dmac_channel, u16 ack_no);
+ u16 default_dma_ack_no;
u16 default_dma_req_no;
};
@@ -181,6 +204,8 @@ struct rz_dmac {
/* LINK MODE DESCRIPTOR */
#define HEADER_LV BIT(0)
+#define HEADER_LE BIT(1)
+#define HEADER_WBD BIT(2)
#define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16
#define RZ_DMAC_MAX_CHANNELS 16
@@ -259,6 +284,12 @@ static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
* Descriptors preparation
*/
+static u32 rz_dmac_lmdesc_addr(struct rz_dmac_chan *channel, struct rz_lmdesc *lmdesc)
+{
+ return channel->lmdesc.base_dma +
+ (sizeof(struct rz_lmdesc) * (lmdesc - channel->lmdesc.base));
+}
+
static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
{
struct rz_lmdesc *lmdesc = channel->lmdesc.head;
@@ -272,30 +303,38 @@ static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
channel->lmdesc.head = lmdesc;
}
+static bool rz_dmac_chan_is_enabled(struct rz_dmac_chan *channel)
+{
+ u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1);
+
+ return !!(val & CHSTAT_EN);
+}
+
+static bool rz_dmac_chan_is_paused(struct rz_dmac_chan *channel)
+{
+ u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1);
+
+ return !!(val & CHSTAT_SUS);
+}
+
static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 nxla;
u32 chctrl;
- u32 chstat;
dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
rz_dmac_lmdesc_recycle(channel);
- nxla = channel->lmdesc.base_dma +
- (sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
- channel->lmdesc.base));
+ nxla = rz_dmac_lmdesc_addr(channel, channel->lmdesc.head);
- chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
- if (!(chstat & CHSTAT_EN)) {
- chctrl = (channel->chctrl | CHCTRL_SETEN);
- rz_dmac_ch_writel(channel, nxla, NXLA, 1);
- rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
- rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
- rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
- }
+ chctrl = (channel->chctrl | CHCTRL_SETEN);
+ rz_dmac_ch_writel(channel, nxla, NXLA, 1);
+ rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
+ rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
+ rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
}
static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
@@ -331,6 +370,60 @@ static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index,
rz_dmac_set_dmars_register(dmac, index, req_no);
}
+/*
+ * Map MID/RID request number (bits[0:9] of DMA specifier) to the ICU
+ * DMA ACK signal number, per RZ/G3E hardware manual Table 4.6-28.
+ *
+ * Three peripheral groups cover all ACK-capable peripherals:
+ *
+ * PFC external DMA pins (DREQ0..DREQ4):
+ * req_no 0x000-0x004 -> ACK No. 84-88 (ack = req_no + 84)
+ *
+ * SSIU BUSIFs (ssip00..ssip93):
+ * req_no 0x161-0x198 -> ACK No. 28-83 (ack = req_no - 0x145)
+ *
+ * SPDIF (CH0..CH2) + SCU SRC (sr0..sr9) + DVC (cmd0..cmd1):
+ * req_no 0x199-0x1b4 -> ACK No. 0-27 (ack = req_no - 0x199)
+ */
+static int rz_dmac_get_ack_no(const struct rz_dmac_info *info, u16 req_no)
+{
+ if (!info->icu_register_dma_ack)
+ return -EINVAL;
+
+ switch (req_no) {
+ case 0x000 ... 0x004:
+ /* PFC external DMA pins: ACK No. 84-88 */
+ return req_no + 84;
+ case 0x161 ... 0x198:
+ /* SSIU BUSIFs: ACK No. 28-83 */
+ return req_no - 0x145;
+ case 0x199 ... 0x1b4:
+ /* SPDIF + SCU SRC + DVC: ACK No. 0-27 */
+ return req_no - 0x199;
+ default:
+ return -EINVAL;
+ }
+}
+
+static void rz_dmac_set_dma_ack_no(struct rz_dmac *dmac, unsigned int index,
+ int ack_no)
+{
+ if (ack_no < 0 || !dmac->info->icu_register_dma_ack)
+ return;
+
+ dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index,
+ index, ack_no);
+}
+
+static void rz_dmac_reset_dma_ack_no(struct rz_dmac *dmac, int ack_no)
+{
+ if (ack_no < 0 || !dmac->info->icu_register_dma_ack)
+ return;
+
+ dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index,
+ dmac->info->default_dma_ack_no, ack_no);
+}
+
static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
@@ -339,6 +432,8 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
struct rz_dmac_desc *d = channel->desc;
u32 chcfg = CHCFG_MEM_COPY;
+ d->start_lmdesc = lmdesc;
+
/* prepare descriptor */
lmdesc->sa = d->src;
lmdesc->da = d->dest;
@@ -346,12 +441,12 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
lmdesc->chcfg = chcfg;
lmdesc->chitvl = 0;
lmdesc->chext = 0;
- lmdesc->header = HEADER_LV;
+ lmdesc->header = HEADER_LV | HEADER_LE;
rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no);
channel->chcfg = chcfg;
- channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
+ channel->chctrl = CHCTRL_STG;
}
static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
@@ -373,6 +468,7 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
}
lmdesc = channel->lmdesc.tail;
+ d->start_lmdesc = lmdesc;
for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
if (d->direction == DMA_DEV_TO_MEM) {
@@ -388,7 +484,7 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
lmdesc->chext = 0;
if (i == (sg_len - 1)) {
lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
- lmdesc->header = HEADER_LV;
+ lmdesc->header = HEADER_LV | HEADER_LE;
} else {
lmdesc->chcfg = channel->chcfg;
lmdesc->header = HEADER_LV;
@@ -400,22 +496,77 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
channel->lmdesc.tail = lmdesc;
rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
+ rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack);
- channel->chctrl = CHCTRL_SETEN;
+ channel->chctrl = 0;
}
-static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
+static void rz_dmac_prepare_descs_for_cyclic(struct rz_dmac_chan *channel)
{
- struct rz_dmac_desc *d = chan->desc;
+ struct dma_chan *chan = &channel->vc.chan;
+ struct rz_dmac *dmac = to_rz_dmac(chan->device);
+ struct rz_dmac_desc *d = channel->desc;
+ size_t period_len = d->sgcount;
+ struct rz_lmdesc *lmdesc;
+ size_t buf_len = d->len;
+ size_t periods = buf_len / period_len;
+
+ lockdep_assert_held(&channel->vc.lock);
+
+ channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DMS;
+
+ if (d->direction == DMA_DEV_TO_MEM) {
+ channel->chcfg |= CHCFG_SAD;
+ channel->chcfg &= ~CHCFG_REQD;
+ } else {
+ channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
+ }
+
+ lmdesc = channel->lmdesc.tail;
+ d->start_lmdesc = lmdesc;
+
+ for (size_t i = 0; i < periods; i++) {
+ if (d->direction == DMA_DEV_TO_MEM) {
+ lmdesc->sa = d->src;
+ lmdesc->da = d->dest + (i * period_len);
+ } else {
+ lmdesc->sa = d->src + (i * period_len);
+ lmdesc->da = d->dest;
+ }
+
+ lmdesc->tb = period_len;
+ lmdesc->chitvl = 0;
+ lmdesc->chext = 0;
+ lmdesc->chcfg = channel->chcfg;
+ lmdesc->header = HEADER_LV | HEADER_WBD;
+
+ if (i == periods - 1)
+ lmdesc->nxla = rz_dmac_lmdesc_addr(channel, d->start_lmdesc);
+
+ if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
+ lmdesc = channel->lmdesc.base;
+ }
+
+ channel->lmdesc.tail = lmdesc;
+
+ rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
+ rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack);
+}
+
+static void rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
+{
struct virt_dma_desc *vd;
vd = vchan_next_desc(&chan->vc);
- if (!vd)
- return 0;
+ if (!vd) {
+ chan->desc = NULL;
+ return;
+ }
list_del(&vd->node);
+ chan->desc = to_rz_dmac_desc(vd);
- switch (d->type) {
+ switch (chan->desc->type) {
case RZ_DMAC_DESC_MEMCPY:
rz_dmac_prepare_desc_for_memcpy(chan);
break;
@@ -424,13 +575,12 @@ static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
rz_dmac_prepare_descs_for_slave_sg(chan);
break;
- default:
- return -EINVAL;
+ case RZ_DMAC_DESC_CYCLIC:
+ rz_dmac_prepare_descs_for_cyclic(chan);
+ break;
}
rz_dmac_enable_hw(chan);
-
- return 0;
}
/*
@@ -466,29 +616,47 @@ static void rz_dmac_free_chan_resources(struct dma_chan *chan)
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *desc, *_desc;
unsigned long flags;
+ int ret;
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret) {
+ dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!",
+ dma_chan_name(chan), ret);
+ }
spin_lock_irqsave(&channel->vc.lock, flags);
rz_lmdesc_setup(channel, channel->lmdesc.base);
- rz_dmac_disable_hw(channel);
- list_splice_tail_init(&channel->ld_active, &channel->ld_free);
- list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
+ /* Skip touching HW if RPM resume failed. Let the cleanup do its jobs. */
+ if (!ret)
+ rz_dmac_disable_hw(channel);
if (channel->mid_rid >= 0) {
clear_bit(channel->mid_rid, dmac->modules);
channel->mid_rid = -EINVAL;
}
+ channel->status = 0;
+ rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack);
+ channel->dmac_ack = -EINVAL;
+
spin_unlock_irqrestore(&channel->vc.lock, flags);
+ vchan_free_chan_resources(&channel->vc);
+
+ spin_lock_irqsave(&channel->vc.lock, flags);
+
list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
+ list_del(&desc->node);
kfree(desc);
channel->descs_allocated--;
}
INIT_LIST_HEAD(&channel->ld_free);
- vchan_free_chan_resources(&channel->vc);
+
+ spin_unlock_irqrestore(&channel->vc.lock, flags);
}
static struct dma_async_tx_descriptor *
@@ -503,20 +671,19 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
__func__, channel->index, &src, &dest, len);
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
- if (list_empty(&channel->ld_free))
+ desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node);
+ if (!desc)
return NULL;
- desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
-
- desc->type = RZ_DMAC_DESC_MEMCPY;
- desc->src = src;
- desc->dest = dest;
- desc->len = len;
- desc->direction = DMA_MEM_TO_MEM;
-
- list_move_tail(channel->ld_free.next, &channel->ld_queue);
+ list_del(&desc->node);
}
+ desc->type = RZ_DMAC_DESC_MEMCPY;
+ desc->src = src;
+ desc->dest = dest;
+ desc->len = len;
+ desc->direction = DMA_MEM_TO_MEM;
+
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
@@ -533,26 +700,74 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
int i = 0;
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
- if (list_empty(&channel->ld_free))
+ desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node);
+ if (!desc)
return NULL;
- desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
+ list_del(&desc->node);
+ }
- for_each_sg(sgl, sg, sg_len, i)
- dma_length += sg_dma_len(sg);
+ for_each_sg(sgl, sg, sg_len, i)
+ dma_length += sg_dma_len(sg);
- desc->type = RZ_DMAC_DESC_SLAVE_SG;
- desc->sg = sgl;
- desc->sgcount = sg_len;
- desc->len = dma_length;
- desc->direction = direction;
+ desc->type = RZ_DMAC_DESC_SLAVE_SG;
+ desc->sg = sgl;
+ desc->sgcount = sg_len;
+ desc->len = dma_length;
+ desc->direction = direction;
- if (direction == DMA_DEV_TO_MEM)
- desc->src = channel->src_per_address;
- else
- desc->dest = channel->dst_per_address;
+ if (direction == DMA_DEV_TO_MEM)
+ desc->src = channel->src_per_address;
+ else
+ desc->dest = channel->dst_per_address;
- list_move_tail(channel->ld_free.next, &channel->ld_queue);
+ return vchan_tx_prep(&channel->vc, &desc->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *
+rz_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
+ size_t buf_len, size_t period_len,
+ enum dma_transfer_direction direction,
+ unsigned long flags)
+{
+ struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+ struct rz_dmac_desc *desc;
+ size_t periods;
+
+ if (!is_slave_direction(direction))
+ return NULL;
+
+ if (!period_len || !buf_len)
+ return NULL;
+
+ periods = buf_len / period_len;
+ if (!periods || periods > DMAC_NR_LMDESC)
+ return NULL;
+
+ scoped_guard(spinlock_irqsave, &channel->vc.lock) {
+ if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))
+ return NULL;
+
+ desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node);
+ if (!desc)
+ return NULL;
+
+ list_del(&desc->node);
+
+ channel->status |= BIT(RZ_DMAC_CHAN_STATUS_CYCLIC);
+ }
+
+ desc->type = RZ_DMAC_DESC_CYCLIC;
+ desc->sgcount = period_len;
+ desc->len = buf_len;
+ desc->direction = direction;
+
+ if (direction == DMA_DEV_TO_MEM) {
+ desc->src = channel->src_per_address;
+ desc->dest = buf_addr;
+ } else {
+ desc->src = buf_addr;
+ desc->dest = channel->dst_per_address;
}
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
@@ -561,44 +776,59 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
static int rz_dmac_terminate_all(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+ struct rz_dmac *dmac = to_rz_dmac(chan->device);
unsigned long flags;
LIST_HEAD(head);
+ int ret;
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret) {
+ dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!",
+ dma_chan_name(chan), ret);
+ }
spin_lock_irqsave(&channel->vc.lock, flags);
- rz_dmac_disable_hw(channel);
+ /* Don't return if RPM failed. Let the cleanup do its jobs. */
+ if (!ret)
+ rz_dmac_disable_hw(channel);
rz_lmdesc_setup(channel, channel->lmdesc.base);
- list_splice_tail_init(&channel->ld_active, &channel->ld_free);
- list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
+ if (channel->desc) {
+ vchan_terminate_vdesc(&channel->desc->vd);
+ channel->desc = NULL;
+ }
+
vchan_get_all_descriptors(&channel->vc, &head);
+
+ channel->status = 0;
+
spin_unlock_irqrestore(&channel->vc.lock, flags);
vchan_dma_desc_free_list(&channel->vc, &head);
- return 0;
+ return ret;
}
static void rz_dmac_issue_pending(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
- struct rz_dmac_desc *desc;
unsigned long flags;
+ int ret;
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret)
+ return;
spin_lock_irqsave(&channel->vc.lock, flags);
- if (!list_empty(&channel->ld_queue)) {
- desc = list_first_entry(&channel->ld_queue,
- struct rz_dmac_desc, node);
- channel->desc = desc;
- if (vchan_issue_pending(&channel->vc)) {
- if (rz_dmac_xfer_desc(channel) < 0)
- dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
- channel->index);
- else
- list_move_tail(channel->ld_queue.next,
- &channel->ld_active);
- }
- }
+ /*
+ * Issue the descriptor. If another transfer is already in progress, the
+ * issued descriptor will be handled after the current transfer finishes.
+ */
+ if (vchan_issue_pending(&channel->vc) && !channel->desc)
+ rz_dmac_xfer_desc(channel);
spin_unlock_irqrestore(&channel->vc.lock, flags);
}
@@ -656,13 +886,13 @@ static int rz_dmac_config(struct dma_chan *chan,
static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
{
- /*
- * Place holder
- * Descriptor allocation is done during alloc_chan_resources and
- * get freed during free_chan_resources.
- * list is used to manage the descriptors and avoid any memory
- * allocation/free during DMA read/write.
- */
+ struct rz_dmac_chan *channel = to_rz_dmac_chan(vd->tx.chan);
+ struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
+ struct rz_dmac_desc *desc = to_rz_dmac_desc(vd);
+
+ guard(spinlock_irqsave)(&vc->lock);
+
+ list_add_tail(&desc->node, &channel->ld_free);
}
static void rz_dmac_device_synchronize(struct dma_chan *chan)
@@ -672,12 +902,20 @@ static void rz_dmac_device_synchronize(struct dma_chan *chan)
u32 chstat;
int ret;
+ vchan_synchronize(&channel->vc);
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret)
+ return;
+
ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
100, 100000, false, channel, CHSTAT, 1);
if (ret < 0)
dev_warn(dmac->dev, "DMA Timeout");
rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no);
+ rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack);
}
static struct rz_lmdesc *
@@ -691,9 +929,10 @@ rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc)
return next;
}
-static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, u32 crla)
+static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel,
+ struct rz_dmac_desc *desc, u32 crla)
{
- struct rz_lmdesc *lmdesc = channel->lmdesc.head;
+ struct rz_lmdesc *lmdesc = desc->start_lmdesc;
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 residue = 0, i = 0;
@@ -705,9 +944,18 @@ static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, u
}
/* Calculate residue from next lmdesc to end of virtual desc */
- while (lmdesc->chcfg & CHCFG_DEM) {
- residue += lmdesc->tb;
- lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
+ if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) {
+ u32 start_lmdesc_addr = rz_dmac_lmdesc_addr(channel, desc->start_lmdesc);
+
+ while (lmdesc->nxla != start_lmdesc_addr) {
+ residue += lmdesc->tb;
+ lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
+ }
+ } else {
+ while (lmdesc->chcfg & CHCFG_DEM) {
+ residue += lmdesc->tb;
+ lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
+ }
}
dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue);
@@ -715,64 +963,36 @@ static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, u
return residue;
}
-static u32 rz_dmac_chan_get_residue(struct rz_dmac_chan *channel,
- dma_cookie_t cookie)
+static int rz_dmac_chan_get_residue(struct device *dev, struct rz_dmac_chan *channel,
+ dma_cookie_t cookie, u32 *residue)
{
- struct rz_dmac_desc *current_desc, *desc;
- enum dma_status status;
+ struct rz_dmac_desc *desc = NULL;
+ struct virt_dma_desc *vd;
u32 crla, crtb, i;
+ int ret;
- /* Get current processing virtual descriptor */
- current_desc = list_first_entry(&channel->ld_active,
- struct rz_dmac_desc, node);
- if (!current_desc)
+ vd = vchan_find_desc(&channel->vc, cookie);
+ if (vd) {
+ /* Descriptor has been issued but not yet processed. */
+ desc = to_rz_dmac_desc(vd);
+ *residue = desc->len;
return 0;
+ } else if (channel->desc && channel->desc->vd.tx.cookie == cookie) {
+ /* Descriptor is currently processed. */
+ desc = channel->desc;
+ }
- /*
- * If the cookie corresponds to a descriptor that has been completed
- * there is no residue. The same check has already been performed by the
- * caller but without holding the channel lock, so the descriptor could
- * now be complete.
- */
- status = dma_cookie_status(&channel->vc.chan, cookie, NULL);
- if (status == DMA_COMPLETE)
- return 0;
-
- /*
- * If the cookie doesn't correspond to the currently processing virtual
- * descriptor then the descriptor hasn't been processed yet, and the
- * residue is equal to the full descriptor size. Also, a client driver
- * is possible to call this function before rz_dmac_irq_handler_thread()
- * runs. In this case, the running descriptor will be the next
- * descriptor, and will appear in the done list. So, if the argument
- * cookie matches the done list's cookie, we can assume the residue is
- * zero.
- */
- if (cookie != current_desc->vd.tx.cookie) {
- list_for_each_entry(desc, &channel->ld_free, node) {
- if (cookie == desc->vd.tx.cookie)
- return 0;
- }
-
- list_for_each_entry(desc, &channel->ld_queue, node) {
- if (cookie == desc->vd.tx.cookie)
- return desc->len;
- }
-
- list_for_each_entry(desc, &channel->ld_active, node) {
- if (cookie == desc->vd.tx.cookie)
- return desc->len;
- }
-
- /*
- * No descriptor found for the cookie, there's thus no residue.
- * This shouldn't happen if the calling driver passes a correct
- * cookie value.
- */
- WARN(1, "No descriptor for cookie!");
+ if (!desc) {
+ /* Descriptor was not found. May be already completed by now. */
+ *residue = 0;
return 0;
}
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret)
+ return ret;
+
/*
* We need to read two registers. Make sure the hardware does not move
* to next lmdesc while reading the current lmdesc. Trying it 3 times
@@ -792,7 +1012,9 @@ static u32 rz_dmac_chan_get_residue(struct rz_dmac_chan *channel,
* Calculate number of bytes transferred in processing virtual descriptor.
* One virtual descriptor can have many lmdesc.
*/
- return crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, crla);
+ *residue = crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, desc, crla);
+
+ return 0;
}
static enum dma_status rz_dmac_tx_status(struct dma_chan *chan,
@@ -800,62 +1022,155 @@ static enum dma_status rz_dmac_tx_status(struct dma_chan *chan,
struct dma_tx_state *txstate)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+ struct rz_dmac *dmac = to_rz_dmac(chan->device);
enum dma_status status;
u32 residue;
- status = dma_cookie_status(chan, cookie, txstate);
- if (status == DMA_COMPLETE || !txstate)
- return status;
-
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
- u32 val;
+ int ret;
- residue = rz_dmac_chan_get_residue(channel, cookie);
+ status = dma_cookie_status(chan, cookie, txstate);
+ if (status == DMA_COMPLETE || !txstate)
+ return status;
- val = rz_dmac_ch_readl(channel, CHSTAT, 1);
- if (val & CHSTAT_SUS)
+ ret = rz_dmac_chan_get_residue(dmac->dev, channel, cookie, &residue);
+ if (ret)
+ return DMA_ERROR;
+
+ if (status == DMA_IN_PROGRESS && rz_dmac_chan_is_paused(channel))
status = DMA_PAUSED;
}
- /* if there's no residue and no paused, the cookie is complete */
- if (!residue && status != DMA_PAUSED)
- return DMA_COMPLETE;
-
dma_set_residue(txstate, residue);
return status;
}
+static int rz_dmac_device_pause_set(struct rz_dmac_chan *channel,
+ unsigned long set_bitmask)
+{
+ int ret = 0;
+ u32 val;
+
+ lockdep_assert_held(&channel->vc.lock);
+
+ if (!rz_dmac_chan_is_enabled(channel))
+ return 0;
+
+ if (rz_dmac_chan_is_paused(channel))
+ goto set_bit;
+
+ rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1);
+ ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val,
+ (val & CHSTAT_SUS), 1, 1024, false,
+ channel, CHSTAT, 1);
+
+set_bit:
+ channel->status |= set_bitmask;
+
+ return ret;
+}
+
static int rz_dmac_device_pause(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
- u32 val;
+ struct rz_dmac *dmac = to_rz_dmac(chan->device);
+ int ret;
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret)
+ return ret;
guard(spinlock_irqsave)(&channel->vc.lock);
- val = rz_dmac_ch_readl(channel, CHSTAT, 1);
- if (!(val & CHSTAT_EN))
+ return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED));
+}
+
+static int rz_dmac_device_pause_internal(struct rz_dmac_chan *channel)
+{
+ lockdep_assert_held(&channel->vc.lock);
+
+ /* Skip channels explicitly paused by consummers or disabled. */
+ if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED) ||
+ !rz_dmac_chan_is_enabled(channel))
return 0;
- rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1);
- return read_poll_timeout_atomic(rz_dmac_ch_readl, val,
- (val & CHSTAT_SUS), 1, 1024,
- false, channel, CHSTAT, 1);
+ return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL));
+}
+
+static int rz_dmac_device_resume_set(struct rz_dmac_chan *channel,
+ unsigned long clear_bitmask)
+{
+ u32 val;
+ int ret;
+
+ lockdep_assert_held(&channel->vc.lock);
+
+ /*
+ * We can be:
+ *
+ * 1/ after the channel was paused by a consummer and now it
+ * needs to be resummed
+ * 2/ after the channel was paused internally (as a result of
+ * a system suspend with power loss or not)
+ * 3/ after the channel was paused by a consummer, the system
+ * went through a system suspend (with power loss or not)
+ * and the consummer wants to resume the channel
+ *
+ * To cover all the above cases we set both CLRSUS and SETEN.
+ *
+ * In case 1/ setting SETEN while the channel is still enabled
+ * is harmless for the controller.
+ *
+ * In case 2/ the channel is disabled when calling this function
+ * and setting CLRSUS is harmless for the controller as the
+ * channel is disabled anyway.
+ *
+ * In case 3/ the channel is disabled/enabled if the system
+ * went though a suspend with power loss/or not and setting
+ * CLRSUS/SETEN is harmless for the controller as the channel
+ * is enabled/disabled anyway.
+ */
+
+ rz_dmac_ch_writel(channel, CHCTRL_CLRSUS | CHCTRL_SETEN, CHCTRL, 1);
+
+ ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val,
+ ((val & (CHSTAT_SUS | CHSTAT_EN)) == CHSTAT_EN),
+ 1, 1024, false, channel, CHSTAT, 1);
+
+ channel->status &= ~clear_bitmask;
+
+ return ret;
}
static int rz_dmac_device_resume(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
- u32 val;
+ struct rz_dmac *dmac = to_rz_dmac(chan->device);
+ int ret;
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret)
+ return ret;
guard(spinlock_irqsave)(&channel->vc.lock);
- /* Do not check CHSTAT_SUS but rely on HW capabilities. */
+ if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED)))
+ return 0;
- rz_dmac_ch_writel(channel, CHCTRL_CLRSUS, CHCTRL, 1);
- return read_poll_timeout_atomic(rz_dmac_ch_readl, val,
- !(val & CHSTAT_SUS), 1, 1024,
- false, channel, CHSTAT, 1);
+ return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED));
+}
+
+static int rz_dmac_device_resume_internal(struct rz_dmac_chan *channel)
+{
+ lockdep_assert_held(&channel->vc.lock);
+
+ if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)))
+ return 0;
+
+ return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL));
}
/*
@@ -875,7 +1190,7 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
channel->index, chstat);
scoped_guard(spinlock_irqsave, &channel->vc.lock)
- rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
+ rz_dmac_disable_hw(channel);
return;
}
@@ -901,29 +1216,23 @@ static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
{
struct rz_dmac_chan *channel = dev_id;
- struct rz_dmac_desc *desc = NULL;
- unsigned long flags;
+ struct rz_dmac_desc *desc;
- spin_lock_irqsave(&channel->vc.lock, flags);
+ guard(spinlock_irqsave)(&channel->vc.lock);
- if (list_empty(&channel->ld_active)) {
- /* Someone might have called terminate all */
- goto out;
+ desc = channel->desc;
+ if (!desc)
+ return IRQ_HANDLED;
+
+ if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) {
+ vchan_cyclic_callback(&desc->vd);
+ } else {
+ vchan_cookie_complete(&desc->vd);
+ channel->desc = NULL;
+
+ rz_dmac_xfer_desc(channel);
}
- desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
- vchan_cookie_complete(&desc->vd);
- list_move_tail(channel->ld_active.next, &channel->ld_free);
- if (!list_empty(&channel->ld_queue)) {
- desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
- node);
- channel->desc = desc;
- if (rz_dmac_xfer_desc(channel) == 0)
- list_move_tail(channel->ld_queue.next, &channel->ld_active);
- }
-out:
- spin_unlock_irqrestore(&channel->vc.lock, flags);
-
return IRQ_HANDLED;
}
@@ -944,6 +1253,8 @@ static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
+ channel->dmac_ack = rz_dmac_get_ack_no(dmac->info, channel->mid_rid);
+
return !test_and_set_bit(channel->mid_rid, dmac->modules);
}
@@ -980,6 +1291,35 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac,
channel->index = index;
channel->mid_rid = -EINVAL;
+ channel->dmac_ack = -EINVAL;
+
+ /* Set io base address for each channel */
+ if (index < 8) {
+ channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
+ EACH_CHANNEL_OFFSET * index;
+ channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
+ } else {
+ channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
+ EACH_CHANNEL_OFFSET * (index - 8);
+ channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
+ }
+
+ /* Allocate descriptors */
+ lmdesc = dmam_alloc_coherent(&pdev->dev,
+ sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
+ &channel->lmdesc.base_dma, GFP_KERNEL);
+ if (!lmdesc) {
+ dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
+ return -ENOMEM;
+ }
+ rz_lmdesc_setup(channel, lmdesc);
+
+ channel->vc.desc_free = rz_dmac_virt_desc_free;
+ vchan_init(&channel->vc, &dmac->engine);
+ INIT_LIST_HEAD(&channel->ld_free);
+
+ /* Initialize register for each channel */
+ rz_dmac_disable_hw(channel);
/* Request the channel interrupt. */
scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
@@ -995,42 +1335,10 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac,
ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler,
rz_dmac_irq_handler_thread, 0,
irqname, channel);
- if (ret) {
+ if (ret)
dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret);
- return ret;
- }
- /* Set io base address for each channel */
- if (index < 8) {
- channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
- EACH_CHANNEL_OFFSET * index;
- channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
- } else {
- channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
- EACH_CHANNEL_OFFSET * (index - 8);
- channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
- }
-
- /* Allocate descriptors */
- lmdesc = dma_alloc_coherent(&pdev->dev,
- sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
- &channel->lmdesc.base_dma, GFP_KERNEL);
- if (!lmdesc) {
- dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
- return -ENOMEM;
- }
- rz_lmdesc_setup(channel, lmdesc);
-
- /* Initialize register for each channel */
- rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
-
- channel->vc.desc_free = rz_dmac_virt_desc_free;
- vchan_init(&channel->vc, &dmac->engine);
- INIT_LIST_HEAD(&channel->ld_queue);
- INIT_LIST_HEAD(&channel->ld_free);
- INIT_LIST_HEAD(&channel->ld_active);
-
- return 0;
+ return ret;
}
static void rz_dmac_put_device(void *_dev)
@@ -1099,7 +1407,6 @@ static int rz_dmac_probe(struct platform_device *pdev)
const char *irqname = "error";
struct dma_device *engine;
struct rz_dmac *dmac;
- int channel_num;
int ret;
int irq;
u8 i;
@@ -1132,18 +1439,6 @@ static int rz_dmac_probe(struct platform_device *pdev)
return PTR_ERR(dmac->ext_base);
}
- /* Register interrupt handler for error */
- irq = platform_get_irq_byname_optional(pdev, irqname);
- if (irq > 0) {
- ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
- irqname, NULL);
- if (ret) {
- dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
- irq, ret);
- return ret;
- }
- }
-
/* Initialize the channels. */
INIT_LIST_HEAD(&dmac->engine.channels);
@@ -1152,6 +1447,7 @@ static int rz_dmac_probe(struct platform_device *pdev)
return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),
"failed to get resets\n");
+ pm_runtime_irq_safe(&pdev->dev);
pm_runtime_enable(&pdev->dev);
ret = pm_runtime_resume_and_get(&pdev->dev);
if (ret < 0) {
@@ -1169,6 +1465,18 @@ static int rz_dmac_probe(struct platform_device *pdev)
goto err;
}
+ /* Register interrupt handler for error */
+ irq = platform_get_irq_byname_optional(pdev, irqname);
+ if (irq > 0) {
+ ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
+ irqname, NULL);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
+ irq, ret);
+ goto err;
+ }
+ }
+
/* Register the DMAC as a DMA provider for DT. */
ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
NULL);
@@ -1179,6 +1487,8 @@ static int rz_dmac_probe(struct platform_device *pdev)
engine = &dmac->engine;
dma_cap_set(DMA_SLAVE, engine->cap_mask);
dma_cap_set(DMA_MEMCPY, engine->cap_mask);
+ dma_cap_set(DMA_CYCLIC, engine->cap_mask);
+ engine->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
@@ -1190,6 +1500,7 @@ static int rz_dmac_probe(struct platform_device *pdev)
engine->device_tx_status = rz_dmac_tx_status;
engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
+ engine->device_prep_dma_cyclic = rz_dmac_prep_dma_cyclic;
engine->device_config = rz_dmac_config;
engine->device_terminate_all = rz_dmac_terminate_all;
engine->device_issue_pending = rz_dmac_issue_pending;
@@ -1210,16 +1521,6 @@ static int rz_dmac_probe(struct platform_device *pdev)
dma_register_err:
of_dma_controller_free(pdev->dev.of_node);
err:
- channel_num = i ? i - 1 : 0;
- for (i = 0; i < channel_num; i++) {
- struct rz_dmac_chan *channel = &dmac->channels[i];
-
- dma_free_coherent(&pdev->dev,
- sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
- channel->lmdesc.base,
- channel->lmdesc.base_dma);
- }
-
reset_control_assert(dmac->rstc);
err_pm_runtime_put:
pm_runtime_put(&pdev->dev);
@@ -1232,25 +1533,135 @@ static int rz_dmac_probe(struct platform_device *pdev)
static void rz_dmac_remove(struct platform_device *pdev)
{
struct rz_dmac *dmac = platform_get_drvdata(pdev);
- unsigned int i;
dma_async_device_unregister(&dmac->engine);
of_dma_controller_free(pdev->dev.of_node);
- for (i = 0; i < dmac->n_channels; i++) {
- struct rz_dmac_chan *channel = &dmac->channels[i];
-
- dma_free_coherent(&pdev->dev,
- sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
- channel->lmdesc.base,
- channel->lmdesc.base_dma);
- }
reset_control_assert(dmac->rstc);
pm_runtime_put(&pdev->dev);
pm_runtime_disable(&pdev->dev);
}
+static void rz_dmac_suspend_recover(struct rz_dmac *dmac)
+{
+ int ret;
+
+ PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret)
+ return;
+
+ for (unsigned int i = 0; i < dmac->n_channels; i++) {
+ struct rz_dmac_chan *channel = &dmac->channels[i];
+
+ guard(spinlock_irqsave)(&channel->vc.lock);
+
+ if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)))
+ continue;
+
+ rz_dmac_device_resume_internal(channel);
+ }
+}
+
+static int rz_dmac_suspend(struct device *dev)
+{
+ struct rz_dmac *dmac = dev_get_drvdata(dev);
+ int ret = 0;
+
+ for (unsigned int i = 0; i < dmac->n_channels; i++) {
+ struct rz_dmac_chan *channel = &dmac->channels[i];
+
+ guard(spinlock_irqsave)(&channel->vc.lock);
+
+ if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)))
+ continue;
+
+ ret = rz_dmac_device_pause_internal(channel);
+ if (ret) {
+ dev_err(dev, "Failed to suspend channel %s\n",
+ dma_chan_name(&channel->vc.chan));
+ break;
+ }
+
+ channel->pm_state.nxla = rz_dmac_ch_readl(channel, NXLA, 1);
+ }
+
+ if (ret)
+ goto suspend_recover;
+
+ ret = reset_control_assert(dmac->rstc);
+ if (ret)
+ goto suspend_recover;
+
+ ret = pm_runtime_put_sync(dev);
+ if (ret < 0)
+ goto reset_deassert;
+
+ return 0;
+
+reset_deassert:
+ reset_control_deassert(dmac->rstc);
+suspend_recover:
+ rz_dmac_suspend_recover(dmac);
+ return ret;
+}
+
+static int rz_dmac_resume(struct device *dev)
+{
+ struct rz_dmac *dmac = dev_get_drvdata(dev);
+ int errors = 0, ret;
+
+ ret = pm_runtime_resume_and_get(dev);
+ if (ret)
+ return ret;
+
+ ret = reset_control_deassert(dmac->rstc);
+ if (ret) {
+ /*
+ * Do not put runtime PM here and keep the same state as in
+ * probe. As subsequent suspend/resume cycles may follow, leave
+ * the runtime PM as is, here, to avoid imbalances.
+ */
+ return ret;
+ }
+
+ rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
+ rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
+
+ for (unsigned int i = 0; i < dmac->n_channels; i++) {
+ struct rz_dmac_chan *channel = &dmac->channels[i];
+
+ guard(spinlock_irqsave)(&channel->vc.lock);
+
+ rz_dmac_disable_hw(&dmac->channels[i]);
+
+ if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)))
+ continue;
+
+ rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
+ rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack);
+
+ rz_dmac_ch_writel(channel, channel->pm_state.nxla, NXLA, 1);
+ rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
+ rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
+ rz_dmac_ch_writel(channel, channel->chctrl, CHCTRL, 1);
+
+ ret = rz_dmac_device_resume_internal(channel);
+ if (ret) {
+ errors = ret;
+ dev_err(dev, "Failed to resume channel %s, ret=%d\n",
+ dma_chan_name(&channel->vc.chan), ret);
+ }
+ }
+
+ return errors ? : 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(rz_dmac_pm_ops, rz_dmac_suspend, rz_dmac_resume);
+
static const struct rz_dmac_info rz_dmac_v2h_info = {
.icu_register_dma_req = rzv2h_icu_register_dma_req,
+ .icu_register_dma_ack = rzv2h_icu_register_dma_ack,
+ .default_dma_ack_no = RZV2H_ICU_DMAC_ACK_NO_DEFAULT,
.default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT,
};
@@ -1275,6 +1686,7 @@ static struct platform_driver rz_dmac_driver = {
.driver = {
.name = "rz-dmac",
.of_match_table = of_rz_dmac_match,
+ .pm = pm_ptr(&rz_dmac_pm_ops),
},
.probe = rz_dmac_probe,
.remove = rz_dmac_remove,
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 9b803c0a..0d9ffa3 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -602,7 +602,6 @@ struct d40_base {
struct dma_device dma_both;
struct dma_device dma_slave;
struct dma_device dma_memcpy;
- struct d40_chan *phy_chans;
struct d40_chan *log_chans;
struct d40_chan **lookup_log_chans;
struct d40_chan **lookup_phy_chans;
@@ -621,6 +620,7 @@ struct d40_base {
u32 *regs_interrupt;
u16 gcc_pwr_off_mask;
struct d40_gen_dmac gen_dmac;
+ struct d40_chan phy_chans[];
};
static struct device *chan2dev(struct d40_chan *d40c)
@@ -3128,6 +3128,7 @@ static int __init d40_hw_detect_init(struct platform_device *pdev,
struct clk *clk;
void __iomem *virtbase;
struct d40_base *base;
+ size_t alloc_size;
int num_log_chans;
int num_phy_chans;
int num_memcpy_chans;
@@ -3185,22 +3186,24 @@ static int __init d40_hw_detect_init(struct platform_device *pdev,
else
num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
+ num_phy_chans = min(num_phy_chans, STEDMA40_MAX_PHYS);
+
/* The number of channels used for memcpy */
if (plat_data->num_of_memcpy_chans)
num_memcpy_chans = plat_data->num_of_memcpy_chans;
else
num_memcpy_chans = ARRAY_SIZE(dma40_memcpy_channels);
+ num_memcpy_chans = min(num_memcpy_chans, D40_MEMCPY_MAX_CHANS);
num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY;
dev_info(dev,
"hardware rev: %d with %d physical and %d logical channels\n",
rev, num_phy_chans, num_log_chans);
- base = devm_kzalloc(dev,
- ALIGN(sizeof(struct d40_base), 4) +
- (num_phy_chans + num_log_chans + num_memcpy_chans) *
- sizeof(struct d40_chan), GFP_KERNEL);
+ alloc_size = struct_size(base, phy_chans, num_phy_chans);
+ alloc_size += sizeof(*base->log_chans) * (num_log_chans + num_memcpy_chans);
+ base = devm_kzalloc(dev, alloc_size, GFP_KERNEL);
if (!base)
return -ENOMEM;
@@ -3213,7 +3216,6 @@ static int __init d40_hw_detect_init(struct platform_device *pdev,
base->virtbase = virtbase;
base->plat_data = plat_data;
base->dev = dev;
- base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
base->log_chans = &base->phy_chans[num_phy_chans];
if (base->plat_data->num_of_phy_chans == 14) {
diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c
index 5948fbf..64cedef 100644
--- a/drivers/dma/tegra186-gpc-dma.c
+++ b/drivers/dma/tegra186-gpc-dma.c
@@ -15,6 +15,7 @@
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
+#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/reset.h>
#include <linux/slab.h>
@@ -22,7 +23,6 @@
#include "virt-dma.h"
/* CSR register */
-#define TEGRA_GPCDMA_CHAN_CSR 0x00
#define TEGRA_GPCDMA_CSR_ENB BIT(31)
#define TEGRA_GPCDMA_CSR_IE_EOC BIT(30)
#define TEGRA_GPCDMA_CSR_ONCE BIT(27)
@@ -58,7 +58,6 @@
#define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10)
/* STATUS register */
-#define TEGRA_GPCDMA_CHAN_STATUS 0x004
#define TEGRA_GPCDMA_STATUS_BUSY BIT(31)
#define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30)
#define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28)
@@ -70,22 +69,13 @@
#define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21)
#define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20)
-#define TEGRA_GPCDMA_CHAN_CSRE 0x008
#define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31)
-/* Source address */
-#define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C
-
-/* Destination address */
-#define TEGRA_GPCDMA_CHAN_DST_PTR 0x010
-
/* High address pointer */
-#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x014
#define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0)
#define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16)
/* MC sequence register */
-#define TEGRA_GPCDMA_CHAN_MCSEQ 0x18
#define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31)
#define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25)
#define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23)
@@ -101,7 +91,6 @@
#define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0)
/* MMIO sequence register */
-#define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c
#define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31)
#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28)
#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \
@@ -120,17 +109,7 @@
#define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16)
#define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7)
-/* Channel WCOUNT */
-#define TEGRA_GPCDMA_CHAN_WCOUNT 0x20
-
-/* Transfer count */
-#define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24
-
-/* DMA byte count status */
-#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28
-
/* Error Status Register */
-#define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30
#define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8
#define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF
#define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \
@@ -143,16 +122,6 @@
#define TEGRA_DMA_MC_SLAVE_ERR 0xB
#define TEGRA_DMA_MMIO_SLAVE_ERR 0xA
-/* Fixed Pattern */
-#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34
-
-#define TEGRA_GPCDMA_CHAN_TZ 0x38
-#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0)
-#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1)
-
-#define TEGRA_GPCDMA_CHAN_SPARE 0x3c
-#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16)
-
/*
* If any burst is in flight and DMA paused then this is the time to complete
* on-flight burst and update DMA status register.
@@ -178,21 +147,30 @@ struct tegra_dma_channel;
*/
struct tegra_dma_chip_data {
bool hw_support_pause;
+ unsigned int addr_bits;
unsigned int nr_channels;
unsigned int channel_reg_size;
unsigned int max_dma_count;
+ const struct tegra_dma_channel_regs *channel_regs;
int (*terminate)(struct tegra_dma_channel *tdc);
};
/* DMA channel registers */
struct tegra_dma_channel_regs {
u32 csr;
- u32 src_ptr;
- u32 dst_ptr;
- u32 high_addr_ptr;
+ u32 status;
+ u32 csre;
+ u32 src;
+ u32 dst;
+ u32 high_addr;
+ u32 src_high;
+ u32 dst_high;
u32 mc_seq;
u32 mmio_seq;
u32 wcount;
+ u32 wxfer;
+ u32 wstatus;
+ u32 err_status;
u32 fixed_pattern;
};
@@ -205,7 +183,13 @@ struct tegra_dma_channel_regs {
*/
struct tegra_dma_sg_req {
unsigned int len;
- struct tegra_dma_channel_regs ch_regs;
+ dma_addr_t src;
+ dma_addr_t dst;
+ u32 csr;
+ u32 mc_seq;
+ u32 mmio_seq;
+ u32 wcount;
+ u32 fixed_pattern;
};
/*
@@ -228,19 +212,20 @@ struct tegra_dma_desc {
* tegra_dma_channel: Channel specific information
*/
struct tegra_dma_channel {
- bool config_init;
- char name[30];
- enum dma_transfer_direction sid_dir;
- enum dma_status status;
- int id;
- int irq;
- int slave_id;
+ const struct tegra_dma_channel_regs *regs;
struct tegra_dma *tdma;
struct virt_dma_chan vc;
struct tegra_dma_desc *dma_desc;
struct dma_slave_config dma_sconfig;
+ enum dma_transfer_direction sid_dir;
+ enum dma_status status;
unsigned int stream_id;
unsigned long chan_base_offset;
+ bool config_init;
+ char name[30];
+ int id;
+ int irq;
+ int slave_id;
};
/*
@@ -284,26 +269,55 @@ static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
return tdc->vc.chan.device->dev;
}
+static void tegra_dma_program_addr(struct tegra_dma_channel *tdc,
+ struct tegra_dma_sg_req *sg_req)
+{
+ tdc_write(tdc, tdc->regs->src, lower_32_bits(sg_req->src));
+ tdc_write(tdc, tdc->regs->dst, lower_32_bits(sg_req->dst));
+
+ if (tdc->tdma->chip_data->addr_bits > 39) {
+ tdc_write(tdc, tdc->regs->src_high, upper_32_bits(sg_req->src));
+ tdc_write(tdc, tdc->regs->dst_high, upper_32_bits(sg_req->dst));
+ } else {
+ u32 src_high = FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR,
+ upper_32_bits(sg_req->src));
+ u32 dst_high = FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR,
+ upper_32_bits(sg_req->dst));
+
+ tdc_write(tdc, tdc->regs->high_addr, src_high | dst_high);
+ }
+}
+
static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc)
{
dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n",
tdc->id, tdc->name);
- dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n",
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR)
- );
- dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n",
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT),
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS)
- );
+ dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x\n",
+ tdc_read(tdc, tdc->regs->csr),
+ tdc_read(tdc, tdc->regs->status),
+ tdc_read(tdc, tdc->regs->csre));
+
+ if (tdc->tdma->chip_data->addr_bits > 39) {
+ dev_dbg(tdc2dev(tdc), "SRC %x SRC HI %x DST %x DST HI %x\n",
+ tdc_read(tdc, tdc->regs->src),
+ tdc_read(tdc, tdc->regs->src_high),
+ tdc_read(tdc, tdc->regs->dst),
+ tdc_read(tdc, tdc->regs->dst_high));
+ } else {
+ dev_dbg(tdc2dev(tdc), "SRC %x DST %x HI ADDR %x\n",
+ tdc_read(tdc, tdc->regs->src),
+ tdc_read(tdc, tdc->regs->dst),
+ tdc_read(tdc, tdc->regs->high_addr));
+ }
+
+ dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x WSTA %x\n",
+ tdc_read(tdc, tdc->regs->mc_seq),
+ tdc_read(tdc, tdc->regs->mmio_seq),
+ tdc_read(tdc, tdc->regs->wcount),
+ tdc_read(tdc, tdc->regs->wxfer),
+ tdc_read(tdc, tdc->regs->wstatus));
dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n",
- tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS));
+ tdc_read(tdc, tdc->regs->err_status));
}
static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc,
@@ -377,13 +391,13 @@ static int tegra_dma_pause(struct tegra_dma_channel *tdc)
int ret;
u32 val;
- val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
+ val = tdc_read(tdc, tdc->regs->csre);
val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
+ tdc_write(tdc, tdc->regs->csre, val);
/* Wait until busy bit is de-asserted */
ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
- tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
+ tdc->chan_base_offset + tdc->regs->status,
val,
!(val & TEGRA_GPCDMA_STATUS_BUSY),
TEGRA_GPCDMA_BURST_COMPLETE_TIME,
@@ -419,9 +433,9 @@ static void tegra_dma_resume(struct tegra_dma_channel *tdc)
{
u32 val;
- val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
+ val = tdc_read(tdc, tdc->regs->csre);
val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
+ tdc_write(tdc, tdc->regs->csre, val);
tdc->status = DMA_IN_PROGRESS;
}
@@ -456,27 +470,27 @@ static void tegra_dma_disable(struct tegra_dma_channel *tdc)
{
u32 csr, status;
- csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
+ csr = tdc_read(tdc, tdc->regs->csr);
/* Disable interrupts */
csr &= ~TEGRA_GPCDMA_CSR_IE_EOC;
/* Disable DMA */
csr &= ~TEGRA_GPCDMA_CSR_ENB;
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
+ tdc_write(tdc, tdc->regs->csr, csr);
/* Clear interrupt status if it is there */
- status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
+ status = tdc_read(tdc, tdc->regs->status);
if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) {
dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status);
+ tdc_write(tdc, tdc->regs->status, status);
}
}
static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)
{
struct tegra_dma_desc *dma_desc = tdc->dma_desc;
- struct tegra_dma_channel_regs *ch_regs;
+ struct tegra_dma_sg_req *sg_req;
int ret;
u32 val;
@@ -488,29 +502,27 @@ static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)
/* Configure next transfer immediately after DMA is busy */
ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
- tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
+ tdc->chan_base_offset + tdc->regs->status,
val,
(val & TEGRA_GPCDMA_STATUS_BUSY), 0,
TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
if (ret)
return;
- ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
+ sg_req = &dma_desc->sg_req[dma_desc->sg_idx];
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
+ tdc_write(tdc, tdc->regs->wcount, sg_req->wcount);
+ tegra_dma_program_addr(tdc, sg_req);
/* Start DMA */
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
- ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
+ tdc_write(tdc, tdc->regs->csr,
+ sg_req->csr | TEGRA_GPCDMA_CSR_ENB);
}
static void tegra_dma_start(struct tegra_dma_channel *tdc)
{
struct tegra_dma_desc *dma_desc = tdc->dma_desc;
- struct tegra_dma_channel_regs *ch_regs;
+ struct tegra_dma_sg_req *sg_req;
struct virt_dma_desc *vdesc;
if (!dma_desc) {
@@ -526,21 +538,19 @@ static void tegra_dma_start(struct tegra_dma_channel *tdc)
tegra_dma_resume(tdc);
}
- ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
+ sg_req = &dma_desc->sg_req[dma_desc->sg_idx];
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr);
+ tegra_dma_program_addr(tdc, sg_req);
+ tdc_write(tdc, tdc->regs->wcount, sg_req->wcount);
+ tdc_write(tdc, tdc->regs->csr, 0);
+ tdc_write(tdc, tdc->regs->fixed_pattern, sg_req->fixed_pattern);
+ tdc_write(tdc, tdc->regs->mmio_seq, sg_req->mmio_seq);
+ tdc_write(tdc, tdc->regs->mc_seq, sg_req->mc_seq);
+ tdc_write(tdc, tdc->regs->csr, sg_req->csr);
/* Start DMA */
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
- ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
+ tdc_write(tdc, tdc->regs->csr,
+ sg_req->csr | TEGRA_GPCDMA_CSR_ENB);
}
static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc)
@@ -601,19 +611,19 @@ static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
u32 status;
/* Check channel error status register */
- status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS);
+ status = tdc_read(tdc, tdc->regs->err_status);
if (status) {
tegra_dma_chan_decode_error(tdc, status);
tegra_dma_dump_chan_regs(tdc);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF);
+ tdc_write(tdc, tdc->regs->err_status, 0xFFFFFFFF);
}
spin_lock(&tdc->vc.lock);
- status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
+ status = tdc_read(tdc, tdc->regs->status);
if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC))
goto irq_done;
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS,
+ tdc_write(tdc, tdc->regs->status,
TEGRA_GPCDMA_STATUS_ISE_EOC);
if (!dma_desc)
@@ -673,10 +683,10 @@ static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)
* to stop DMA engine from starting any more bursts for
* the given client and wait for in flight bursts to complete
*/
- csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
+ csr = tdc_read(tdc, tdc->regs->csr);
csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK);
csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED;
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
+ tdc_write(tdc, tdc->regs->csr, csr);
/* Wait for in flight data transfer to finish */
udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME);
@@ -687,7 +697,7 @@ static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)
ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
tdc->chan_base_offset +
- TEGRA_GPCDMA_CHAN_STATUS,
+ tdc->regs->status,
status,
!(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX |
TEGRA_GPCDMA_STATUS_CHANNEL_RX)),
@@ -739,14 +749,14 @@ static int tegra_dma_get_residual(struct tegra_dma_channel *tdc)
unsigned int bytes_xfer, residual;
u32 wcount = 0, status;
- wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT);
+ wcount = tdc_read(tdc, tdc->regs->wxfer);
/*
* Set wcount = 0 if EOC bit is set. The transfer would have
* already completed and the CHAN_XFER_COUNT could have updated
* for the next transfer, specifically in case of cyclic transfers.
*/
- status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
+ status = tdc_read(tdc, tdc->regs->status);
if (status & TEGRA_GPCDMA_STATUS_ISE_EOC)
wcount = 0;
@@ -825,6 +835,13 @@ static unsigned int get_burst_size(struct tegra_dma_channel *tdc,
* len to calculate the optimum burst size
*/
burst_byte = burst_size ? burst_size * slave_bw : len;
+
+ /*
+ * Find the largest burst size that evenly divides the transfer length.
+ * The hardware requires the transfer length to be a multiple of the
+ * burst size - partial bursts are not supported.
+ */
+ burst_byte = min(burst_byte, 1U << __ffs(len));
burst_mmio_width = burst_byte / 4;
if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN)
@@ -837,7 +854,7 @@ static unsigned int get_burst_size(struct tegra_dma_channel *tdc,
static int get_transfer_param(struct tegra_dma_channel *tdc,
enum dma_transfer_direction direction,
- u32 *apb_addr,
+ dma_addr_t *apb_addr,
u32 *mmio_seq,
u32 *csr,
unsigned int *burst_size,
@@ -893,7 +910,7 @@ tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,
/* Configure default priority weight for the channel */
csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
- mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
+ mc_seq = tdc_read(tdc, tdc->regs->mc_seq);
/* retain stream-id and clean rest */
mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
@@ -915,17 +932,15 @@ tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,
dma_desc->bytes_req = len;
dma_desc->sg_count = 1;
sg_req = dma_desc->sg_req;
+ sg_req[0].src = 0;
+ sg_req[0].dst = dest;
- sg_req[0].ch_regs.src_ptr = 0;
- sg_req[0].ch_regs.dst_ptr = dest;
- sg_req[0].ch_regs.high_addr_ptr =
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
- sg_req[0].ch_regs.fixed_pattern = value;
+ sg_req[0].fixed_pattern = value;
/* Word count reg takes value as (N +1) words */
- sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
- sg_req[0].ch_regs.csr = csr;
- sg_req[0].ch_regs.mmio_seq = 0;
- sg_req[0].ch_regs.mc_seq = mc_seq;
+ sg_req[0].wcount = ((len - 4) >> 2);
+ sg_req[0].csr = csr;
+ sg_req[0].mmio_seq = 0;
+ sg_req[0].mc_seq = mc_seq;
sg_req[0].len = len;
dma_desc->cyclic = false;
@@ -961,7 +976,7 @@ tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,
/* Configure default priority weight for the channel */
csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
- mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
+ mc_seq = tdc_read(tdc, tdc->regs->mc_seq);
/* retain stream-id and clean rest */
mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) |
(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
@@ -985,17 +1000,14 @@ tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,
dma_desc->sg_count = 1;
sg_req = dma_desc->sg_req;
- sg_req[0].ch_regs.src_ptr = src;
- sg_req[0].ch_regs.dst_ptr = dest;
- sg_req[0].ch_regs.high_addr_ptr =
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32));
- sg_req[0].ch_regs.high_addr_ptr |=
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
+ sg_req[0].src = src;
+ sg_req[0].dst = dest;
+
/* Word count reg takes value as (N +1) words */
- sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
- sg_req[0].ch_regs.csr = csr;
- sg_req[0].ch_regs.mmio_seq = 0;
- sg_req[0].ch_regs.mc_seq = mc_seq;
+ sg_req[0].wcount = ((len - 4) >> 2);
+ sg_req[0].csr = csr;
+ sg_req[0].mmio_seq = 0;
+ sg_req[0].mc_seq = mc_seq;
sg_req[0].len = len;
dma_desc->cyclic = false;
@@ -1010,7 +1022,8 @@ tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
- u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0;
+ u32 csr, mc_seq, mmio_seq = 0;
+ dma_addr_t apb_ptr = 0;
struct tegra_dma_sg_req *sg_req;
struct tegra_dma_desc *dma_desc;
struct scatterlist *sg;
@@ -1049,7 +1062,7 @@ tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
if (flags & DMA_PREP_INTERRUPT)
csr |= TEGRA_GPCDMA_CSR_IE_EOC;
- mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
+ mc_seq = tdc_read(tdc, tdc->regs->mc_seq);
/* retain stream-id and clean rest */
mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
@@ -1096,25 +1109,21 @@ tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
dma_desc->bytes_req += len;
if (direction == DMA_MEM_TO_DEV) {
- sg_req[i].ch_regs.src_ptr = mem;
- sg_req[i].ch_regs.dst_ptr = apb_ptr;
- sg_req[i].ch_regs.high_addr_ptr =
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
+ sg_req[i].src = mem;
+ sg_req[i].dst = apb_ptr;
} else if (direction == DMA_DEV_TO_MEM) {
- sg_req[i].ch_regs.src_ptr = apb_ptr;
- sg_req[i].ch_regs.dst_ptr = mem;
- sg_req[i].ch_regs.high_addr_ptr =
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
+ sg_req[i].src = apb_ptr;
+ sg_req[i].dst = mem;
}
/*
* Word count register takes input in words. Writing a value
* of N into word count register means a req of (N+1) words.
*/
- sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
- sg_req[i].ch_regs.csr = csr;
- sg_req[i].ch_regs.mmio_seq = mmio_seq;
- sg_req[i].ch_regs.mc_seq = mc_seq;
+ sg_req[i].wcount = ((len - 4) >> 2);
+ sg_req[i].csr = csr;
+ sg_req[i].mmio_seq = mmio_seq;
+ sg_req[i].mc_seq = mc_seq;
sg_req[i].len = len;
}
@@ -1128,7 +1137,8 @@ tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_l
unsigned long flags)
{
enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
- u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size;
+ u32 csr, mc_seq, mmio_seq = 0, burst_size;
+ dma_addr_t apb_ptr = 0;
unsigned int max_dma_count, len, period_count, i;
struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
struct tegra_dma_desc *dma_desc;
@@ -1186,7 +1196,7 @@ tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_l
mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
- mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
+ mc_seq = tdc_read(tdc, tdc->regs->mc_seq);
/* retain stream-id and clean rest */
mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
@@ -1217,24 +1227,20 @@ tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_l
for (i = 0; i < period_count; i++) {
mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
if (direction == DMA_MEM_TO_DEV) {
- sg_req[i].ch_regs.src_ptr = mem;
- sg_req[i].ch_regs.dst_ptr = apb_ptr;
- sg_req[i].ch_regs.high_addr_ptr =
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
+ sg_req[i].src = mem;
+ sg_req[i].dst = apb_ptr;
} else if (direction == DMA_DEV_TO_MEM) {
- sg_req[i].ch_regs.src_ptr = apb_ptr;
- sg_req[i].ch_regs.dst_ptr = mem;
- sg_req[i].ch_regs.high_addr_ptr =
- FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
+ sg_req[i].src = apb_ptr;
+ sg_req[i].dst = mem;
}
/*
* Word count register takes input in words. Writing a value
* of N into word count register means a req of (N+1) words.
*/
- sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
- sg_req[i].ch_regs.csr = csr;
- sg_req[i].ch_regs.mmio_seq = mmio_seq;
- sg_req[i].ch_regs.mc_seq = mc_seq;
+ sg_req[i].wcount = ((len - 4) >> 2);
+ sg_req[i].csr = csr;
+ sg_req[i].mmio_seq = mmio_seq;
+ sg_req[i].mc_seq = mc_seq;
sg_req[i].len = len;
mem += len;
@@ -1304,27 +1310,76 @@ static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
return chan;
}
+static const struct tegra_dma_channel_regs tegra186_reg_offsets = {
+ .csr = 0x0,
+ .status = 0x4,
+ .csre = 0x8,
+ .src = 0xc,
+ .dst = 0x10,
+ .high_addr = 0x14,
+ .mc_seq = 0x18,
+ .mmio_seq = 0x1c,
+ .wcount = 0x20,
+ .wxfer = 0x24,
+ .wstatus = 0x28,
+ .err_status = 0x30,
+ .fixed_pattern = 0x34,
+};
+
+static const struct tegra_dma_channel_regs tegra264_reg_offsets = {
+ .csr = 0x0,
+ .status = 0x4,
+ .csre = 0x8,
+ .src = 0xc,
+ .dst = 0x10,
+ .src_high = 0x14,
+ .dst_high = 0x18,
+ .mc_seq = 0x1c,
+ .mmio_seq = 0x20,
+ .wcount = 0x24,
+ .wxfer = 0x28,
+ .wstatus = 0x2c,
+ .err_status = 0x34,
+ .fixed_pattern = 0x38,
+};
+
static const struct tegra_dma_chip_data tegra186_dma_chip_data = {
.nr_channels = 32,
+ .addr_bits = 39,
.channel_reg_size = SZ_64K,
.max_dma_count = SZ_1G,
.hw_support_pause = false,
+ .channel_regs = &tegra186_reg_offsets,
.terminate = tegra_dma_stop_client,
};
static const struct tegra_dma_chip_data tegra194_dma_chip_data = {
.nr_channels = 32,
+ .addr_bits = 39,
.channel_reg_size = SZ_64K,
.max_dma_count = SZ_1G,
.hw_support_pause = true,
+ .channel_regs = &tegra186_reg_offsets,
.terminate = tegra_dma_pause,
};
static const struct tegra_dma_chip_data tegra234_dma_chip_data = {
.nr_channels = 32,
+ .addr_bits = 39,
.channel_reg_size = SZ_64K,
.max_dma_count = SZ_1G,
.hw_support_pause = true,
+ .channel_regs = &tegra186_reg_offsets,
+ .terminate = tegra_dma_pause_noerr,
+};
+
+static const struct tegra_dma_chip_data tegra264_dma_chip_data = {
+ .nr_channels = 32,
+ .addr_bits = 41,
+ .channel_reg_size = SZ_64K,
+ .max_dma_count = SZ_1G,
+ .hw_support_pause = true,
+ .channel_regs = &tegra264_reg_offsets,
.terminate = tegra_dma_pause_noerr,
};
@@ -1339,13 +1394,16 @@ static const struct of_device_id tegra_dma_of_match[] = {
.compatible = "nvidia,tegra234-gpcdma",
.data = &tegra234_dma_chip_data,
}, {
+ .compatible = "nvidia,tegra264-gpcdma",
+ .data = &tegra264_dma_chip_data,
+ }, {
},
};
MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)
{
- unsigned int reg_val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
+ unsigned int reg_val = tdc_read(tdc, tdc->regs->mc_seq);
reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK);
reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
@@ -1353,16 +1411,20 @@ static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)
reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id);
reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id);
- tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val);
+ tdc_write(tdc, tdc->regs->mc_seq, reg_val);
return 0;
}
static int tegra_dma_probe(struct platform_device *pdev)
{
const struct tegra_dma_chip_data *cdata = NULL;
+ struct tegra_dma_channel *tdc;
+ struct tegra_dma *tdma;
+ struct dma_chan *chan;
+ struct device *chdev;
+ bool use_iommu_map = false;
unsigned int i;
u32 stream_id;
- struct tegra_dma *tdma;
int ret;
cdata = of_device_get_match_data(&pdev->dev);
@@ -1381,18 +1443,19 @@ static int tegra_dma_probe(struct platform_device *pdev)
if (IS_ERR(tdma->base_addr))
return PTR_ERR(tdma->base_addr);
- tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma");
+ tdma->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "gpcdma");
if (IS_ERR(tdma->rst)) {
return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst),
- "Missing controller reset\n");
+ "Failed to get controller reset\n");
}
reset_control_reset(tdma->rst);
tdma->dma_dev.dev = &pdev->dev;
- if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) {
- dev_err(&pdev->dev, "Missing iommu stream-id\n");
- return -EINVAL;
+ use_iommu_map = of_property_present(pdev->dev.of_node, "iommu-map");
+ if (!use_iommu_map) {
+ if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id))
+ return dev_err_probe(&pdev->dev, -EINVAL, "Missing iommu stream-id\n");
}
ret = device_property_read_u32(&pdev->dev, "dma-channel-mask",
@@ -1404,9 +1467,10 @@ static int tegra_dma_probe(struct platform_device *pdev)
tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK;
}
+ /* Initialize vchan for each channel and populate the channels list */
INIT_LIST_HEAD(&tdma->dma_dev.channels);
for (i = 0; i < cdata->nr_channels; i++) {
- struct tegra_dma_channel *tdc = &tdma->channels[i];
+ tdc = &tdma->channels[i];
/* Check for channel mask */
if (!(tdma->chan_mask & BIT(i)))
@@ -1419,18 +1483,17 @@ static int tegra_dma_probe(struct platform_device *pdev)
tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET +
i * cdata->channel_reg_size;
snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i);
+ tdc->regs = cdata->channel_regs;
tdc->tdma = tdma;
tdc->id = i;
tdc->slave_id = -1;
vchan_init(&tdc->vc, &tdma->dma_dev);
tdc->vc.desc_free = tegra_dma_desc_free;
-
- /* program stream-id for this channel */
- tegra_dma_program_sid(tdc, stream_id);
- tdc->stream_id = stream_id;
}
+ dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(cdata->addr_bits));
+
dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask);
@@ -1460,37 +1523,59 @@ static int tegra_dma_probe(struct platform_device *pdev)
tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize;
tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
- ret = dma_async_device_register(&tdma->dma_dev);
+ /* Register the DMA device and the channels */
+ ret = dmaenginem_async_device_register(&tdma->dma_dev);
if (ret < 0) {
dev_err_probe(&pdev->dev, ret,
"GPC DMA driver registration failed\n");
return ret;
}
- ret = of_dma_controller_register(pdev->dev.of_node,
- tegra_dma_of_xlate, tdma);
+ /*
+ * Configure stream ID for each channel from the channels registered
+ * above. This is done in a separate iteration to ensure that only
+ * the channels available and registered for the DMA device are used.
+ */
+ list_for_each_entry(chan, &tdma->dma_dev.channels, device_node) {
+ chdev = &chan->dev->device;
+ tdc = to_tegra_dma_chan(chan);
+
+ if (use_iommu_map) {
+ chdev->bus = pdev->dev.bus;
+ dma_coerce_mask_and_coherent(chdev, DMA_BIT_MASK(cdata->addr_bits));
+
+ ret = of_dma_configure_id(chdev, pdev->dev.of_node,
+ true, &tdc->id);
+ if (ret)
+ return dev_err_probe(chdev, ret,
+ "Failed to configure IOMMU for channel %d\n", tdc->id);
+
+ if (!tegra_dev_iommu_get_stream_id(chdev, &stream_id))
+ return dev_err_probe(chdev, -EINVAL,
+ "Failed to get stream ID for channel %d\n", tdc->id);
+
+ chan->dev->chan_dma_dev = true;
+ }
+
+ /* program stream-id for this channel */
+ tegra_dma_program_sid(tdc, stream_id);
+ tdc->stream_id = stream_id;
+ }
+
+ ret = devm_of_dma_controller_register(&pdev->dev, pdev->dev.of_node,
+ tegra_dma_of_xlate, tdma);
if (ret < 0) {
dev_err_probe(&pdev->dev, ret,
"GPC DMA OF registration failed\n");
-
- dma_async_device_unregister(&tdma->dma_dev);
return ret;
}
- dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n",
+ dev_info(&pdev->dev, "GPC DMA driver registered %lu channels\n",
hweight_long(tdma->chan_mask));
return 0;
}
-static void tegra_dma_remove(struct platform_device *pdev)
-{
- struct tegra_dma *tdma = platform_get_drvdata(pdev);
-
- of_dma_controller_free(pdev->dev.of_node);
- dma_async_device_unregister(&tdma->dma_dev);
-}
-
static int __maybe_unused tegra_dma_pm_suspend(struct device *dev)
{
struct tegra_dma *tdma = dev_get_drvdata(dev);
@@ -1541,7 +1626,6 @@ static struct platform_driver tegra_dma_driver = {
.of_match_table = tegra_dma_of_match,
},
.probe = tegra_dma_probe,
- .remove = tegra_dma_remove,
};
module_platform_driver(tegra_dma_driver);
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index 14e0c408..ceaee1e 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -335,8 +335,16 @@ static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc,
struct tegra_adma *tdma = tdc->tdma;
unsigned int sreq_index = tdc->sreq_index;
- if (tdc->sreq_reserved)
- return tdc->sreq_dir == direction ? 0 : -EINVAL;
+ if (tdc->sreq_reserved) {
+ if (tdc->sreq_dir != direction) {
+ dev_err(tdma->dev,
+ "DMA request direction mismatch: reserved=%s, requested=%s\n",
+ dmaengine_get_direction_text(tdc->sreq_dir),
+ dmaengine_get_direction_text(direction));
+ return -EINVAL;
+ }
+ return 0;
+ }
if (sreq_index > tdma->cdata->ch_req_max) {
dev_err(tdma->dev, "invalid DMA request\n");
@@ -665,8 +673,11 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;
unsigned int burst_size, adma_dir, fifo_size_shift;
- if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)
+ if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) {
+ dev_err(tdc2dev(tdc), "invalid DMA periods %zu (max %u)\n",
+ desc->num_periods, ADMA_CH_CONFIG_MAX_BUFS);
return -EINVAL;
+ }
switch (direction) {
case DMA_MEM_TO_DEV:
@@ -1029,8 +1040,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
cdata = of_device_get_match_data(&pdev->dev);
if (!cdata) {
- dev_err(&pdev->dev, "device match data not found\n");
- return -ENODEV;
+ return dev_err_probe(&pdev->dev, -ENODEV,
+ "device match data not found\n");
}
tdma = devm_kzalloc(&pdev->dev,
@@ -1056,7 +1067,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
unsigned int ch_base_offset;
if (res_page->start < res_base->start)
- return -EINVAL;
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "invalid page/global resource order\n");
page_offset = res_page->start - res_base->start;
ch_base_offset = cdata->ch_base_offset;
if (!ch_base_offset)
@@ -1064,7 +1076,9 @@ static int tegra_adma_probe(struct platform_device *pdev)
page_no = div_u64(page_offset, ch_base_offset);
if (!page_no || page_no > INT_MAX)
- return -EINVAL;
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "invalid page number %llu\n",
+ (unsigned long long)page_no);
tdma->ch_page_no = page_no - 1;
tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base);
@@ -1079,7 +1093,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
if (IS_ERR(tdma->base_addr))
return PTR_ERR(tdma->base_addr);
} else {
- return -ENODEV;
+ return dev_err_probe(&pdev->dev, -ENODEV,
+ "failed to get memory resource\n");
}
tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset;
@@ -1087,8 +1102,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio");
if (IS_ERR(tdma->ahub_clk)) {
- dev_err(&pdev->dev, "Error: Missing ahub controller clock\n");
- return PTR_ERR(tdma->ahub_clk);
+ return dev_err_probe(&pdev->dev, PTR_ERR(tdma->ahub_clk),
+ "failed to get ahub clock\n");
}
tdma->dma_chan_mask = devm_kzalloc(&pdev->dev,
@@ -1104,8 +1119,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
(u32 *)tdma->dma_chan_mask,
BITS_TO_U32(tdma->nr_channels));
if (ret < 0 && (ret != -EINVAL)) {
- dev_err(&pdev->dev, "dma-channel-mask is not complete.\n");
- return ret;
+ return dev_err_probe(&pdev->dev, ret,
+ "dma-channel-mask is not complete.\n");
}
INIT_LIST_HEAD(&tdma->dma_dev.channels);
@@ -1127,11 +1142,13 @@ static int tegra_adma_probe(struct platform_device *pdev)
cdata->global_ch_config_base + (4 * i);
}
- tdc->irq = of_irq_get(pdev->dev.of_node, i);
- if (tdc->irq <= 0) {
- ret = tdc->irq ?: -ENXIO;
+ ret = of_irq_get(pdev->dev.of_node, i);
+ if (ret <= 0) {
+ ret = dev_err_probe(&pdev->dev, ret ?: -ENXIO,
+ "failed to get IRQ for channel %d\n", i);
goto irq_dispose;
}
+ tdc->irq = ret;
vchan_init(&tdc->vc, &tdma->dma_dev);
tdc->vc.desc_free = tegra_adma_desc_free;
@@ -1141,12 +1158,18 @@ static int tegra_adma_probe(struct platform_device *pdev)
pm_runtime_enable(&pdev->dev);
ret = pm_runtime_resume_and_get(&pdev->dev);
- if (ret < 0)
+ if (ret < 0) {
+ ret = dev_err_probe(&pdev->dev, ret,
+ "runtime PM resume failed\n");
goto rpm_disable;
+ }
ret = tegra_adma_init(tdma);
- if (ret)
+ if (ret) {
+ ret = dev_err_probe(&pdev->dev, ret,
+ "failed to initialize ADMA\n");
goto rpm_put;
+ }
dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
@@ -1172,14 +1195,16 @@ static int tegra_adma_probe(struct platform_device *pdev)
ret = dma_async_device_register(&tdma->dma_dev);
if (ret < 0) {
- dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret);
+ ret = dev_err_probe(&pdev->dev, ret,
+ "ADMA registration failed\n");
goto rpm_put;
}
ret = of_dma_controller_register(pdev->dev.of_node,
tegra_dma_of_xlate, tdma);
if (ret < 0) {
- dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret);
+ ret = dev_err_probe(&pdev->dev, ret,
+ "ADMA OF registration failed\n");
goto dma_remove;
}
diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c
index 31c543c..971ac83 100644
--- a/drivers/irqchip/irq-renesas-rzv2h.c
+++ b/drivers/irqchip/irq-renesas-rzv2h.c
@@ -151,6 +151,12 @@ struct rzv2h_hw_info {
#define ICU_DMAC_PREP_DMAREQ(sel, up) (FIELD_PREP(ICU_DMAC_DkRQ_SEL_MASK, (sel)) \
<< ICU_DMAC_DMAREQ_SHIFT(up))
+/* DMAC ACK routing - 4 x 7-bit fields per 32-bit register, 8-bit spacing */
+#define ICU_DMAC_DACK_SEL_MASK GENMASK(6, 0)
+#define ICU_DMAC_DACK_SHIFT(n) ((n) * 8)
+#define ICU_DMAC_DACK_FIELD_MASK(n) (ICU_DMAC_DACK_SEL_MASK << ICU_DMAC_DACK_SHIFT(n))
+#define ICU_DMAC_PREP_DACK(val, n) (((val) & ICU_DMAC_DACK_SEL_MASK) << ICU_DMAC_DACK_SHIFT(n))
+
/**
* struct rzv2h_icu_priv - Interrupt Control Unit controller private data structure.
* @base: Controller's base address
@@ -188,6 +194,40 @@ void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index,
}
EXPORT_SYMBOL_GPL(rzv2h_icu_register_dma_req);
+/**
+ * rzv2h_icu_register_dma_ack - Configure DMA ACK signal routing
+ * @icu_dev: ICU platform device
+ * @dmac_index: DMAC instance index (0-4)
+ * @dmac_channel: DMAC channel number (0-15), or RZV2H_ICU_DMAC_ACK_NO_DEFAULT
+ * to disconnect routing for a given ack_no
+ * @ack_no: Peripheral ACK number (0-88) per RZ/G3E manual Table 4.6-28,
+ * used as index into ICU_DMACKSELk
+ *
+ * Routes the ACK signal of the peripheral identified by @ack_no to DMAC
+ * channel @dmac_channel of instance @dmac_index. When @dmac_channel is
+ * RZV2H_ICU_DMAC_ACK_NO_DEFAULT the field is reset, disconnecting any
+ * previously configured routing for that peripheral.
+ */
+void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index,
+ u8 dmac_channel, u16 ack_no)
+{
+ struct rzv2h_icu_priv *priv = platform_get_drvdata(icu_dev);
+ u8 reg_idx = ack_no / 4;
+ u8 field_idx = ack_no & 0x3;
+ u8 dmac_ack_src = (dmac_channel == RZV2H_ICU_DMAC_ACK_NO_DEFAULT) ?
+ RZV2H_ICU_DMAC_ACK_NO_DEFAULT :
+ (dmac_index * 16 + dmac_channel);
+ u32 val;
+
+ guard(raw_spinlock_irqsave)(&priv->lock);
+
+ val = readl(priv->base + ICU_DMACKSELk(reg_idx));
+ val &= ~ICU_DMAC_DACK_FIELD_MASK(field_idx);
+ val |= ICU_DMAC_PREP_DACK(dmac_ack_src, field_idx);
+ writel(val, priv->base + ICU_DMACKSELk(reg_idx));
+}
+EXPORT_SYMBOL_GPL(rzv2h_icu_register_dma_ack);
+
static inline struct rzv2h_icu_priv *irq_data_to_priv(struct irq_data *data)
{
return data->domain->host_data;
diff --git a/include/linux/irqchip/irq-renesas-rzv2h.h b/include/linux/irqchip/irq-renesas-rzv2h.h
index 618a60d..4ffa898 100644
--- a/include/linux/irqchip/irq-renesas-rzv2h.h
+++ b/include/linux/irqchip/irq-renesas-rzv2h.h
@@ -11,13 +11,18 @@
#include <linux/platform_device.h>
#define RZV2H_ICU_DMAC_REQ_NO_DEFAULT 0x3ff
+#define RZV2H_ICU_DMAC_ACK_NO_DEFAULT 0x7f
#ifdef CONFIG_RENESAS_RZV2H_ICU
void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel,
u16 req_no);
+void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index,
+ u8 dmac_channel, u16 ack_no);
#else
static inline void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index,
u8 dmac_channel, u16 req_no) { }
+static inline void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index,
+ u8 dmac_channel, u16 ack_no) { }
#endif
#endif /* __LINUX_IRQ_RENESAS_RZV2H */
diff --git a/include/linux/platform_data/dma-iop32x.h b/include/linux/platform_data/dma-iop32x.h
deleted file mode 100644
index ac83cff..0000000
--- a/include/linux/platform_data/dma-iop32x.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright © 2006, Intel Corporation.
- */
-#ifndef IOP_ADMA_H
-#define IOP_ADMA_H
-#include <linux/types.h>
-#include <linux/dmaengine.h>
-#include <linux/interrupt.h>
-
-#define IOP_ADMA_SLOT_SIZE 32
-#define IOP_ADMA_THRESHOLD 4
-#ifdef DEBUG
-#define IOP_PARANOIA 1
-#else
-#define IOP_PARANOIA 0
-#endif
-#define iop_paranoia(x) BUG_ON(IOP_PARANOIA && (x))
-
-#define DMA0_ID 0
-#define DMA1_ID 1
-#define AAU_ID 2
-
-/**
- * struct iop_adma_device - internal representation of an ADMA device
- * @pdev: Platform device
- * @id: HW ADMA Device selector
- * @dma_desc_pool: base of DMA descriptor region (DMA address)
- * @dma_desc_pool_virt: base of DMA descriptor region (CPU address)
- * @common: embedded struct dma_device
- */
-struct iop_adma_device {
- struct platform_device *pdev;
- int id;
- dma_addr_t dma_desc_pool;
- void *dma_desc_pool_virt;
- struct dma_device common;
-};
-
-/**
- * struct iop_adma_chan - internal representation of an ADMA device
- * @pending: allows batching of hardware operations
- * @lock: serializes enqueue/dequeue operations to the slot pool
- * @mmr_base: memory mapped register base
- * @chain: device chain view of the descriptors
- * @device: parent device
- * @common: common dmaengine channel object members
- * @last_used: place holder for allocation to continue from where it left off
- * @all_slots: complete domain of slots usable by the channel
- * @slots_allocated: records the actual size of the descriptor slot pool
- * @irq_tasklet: bottom half where iop_adma_slot_cleanup runs
- */
-struct iop_adma_chan {
- int pending;
- spinlock_t lock; /* protects the descriptor slot pool */
- void __iomem *mmr_base;
- struct list_head chain;
- struct iop_adma_device *device;
- struct dma_chan common;
- struct iop_adma_desc_slot *last_used;
- struct list_head all_slots;
- int slots_allocated;
- struct tasklet_struct irq_tasklet;
-};
-
-/**
- * struct iop_adma_desc_slot - IOP-ADMA software descriptor
- * @slot_node: node on the iop_adma_chan.all_slots list
- * @chain_node: node on the op_adma_chan.chain list
- * @hw_desc: virtual address of the hardware descriptor chain
- * @phys: hardware address of the hardware descriptor chain
- * @group_head: first operation in a transaction
- * @slot_cnt: total slots used in an transaction (group of operations)
- * @slots_per_op: number of slots per operation
- * @idx: pool index
- * @tx_list: list of descriptors that are associated with one operation
- * @async_tx: support for the async_tx api
- * @group_list: list of slots that make up a multi-descriptor transaction
- * for example transfer lengths larger than the supported hw max
- * @xor_check_result: result of zero sum
- * @crc32_result: result crc calculation
- */
-struct iop_adma_desc_slot {
- struct list_head slot_node;
- struct list_head chain_node;
- void *hw_desc;
- struct iop_adma_desc_slot *group_head;
- u16 slot_cnt;
- u16 slots_per_op;
- u16 idx;
- struct list_head tx_list;
- struct dma_async_tx_descriptor async_tx;
- union {
- u32 *xor_check_result;
- u32 *crc32_result;
- u32 *pq_check_result;
- };
-};
-
-struct iop_adma_platform_data {
- int hw_id;
- dma_cap_mask_t cap_mask;
- size_t pool_size;
-};
-
-#define to_iop_sw_desc(addr_hw_desc) \
- container_of(addr_hw_desc, struct iop_adma_desc_slot, hw_desc)
-#define iop_hw_desc_slot_idx(hw_desc, idx) \
- ( (void *) (((unsigned long) hw_desc) + ((idx) << 5)) )
-#endif
diff --git a/sound/soc/renesas/Kconfig b/sound/soc/renesas/Kconfig
index 11c2027..6520217 100644
--- a/sound/soc/renesas/Kconfig
+++ b/sound/soc/renesas/Kconfig
@@ -56,6 +56,7 @@
config SND_SOC_RZ
tristate "RZ/G2L series SSIF-2 support"
depends on ARCH_RZG2L || COMPILE_TEST
+ select SND_SOC_GENERIC_DMAENGINE_PCM
help
This option enables RZ/G2L SSIF-2 sound support.
diff --git a/sound/soc/renesas/rz-ssi.c b/sound/soc/renesas/rz-ssi.c
index 71e434c..9fe8a63 100644
--- a/sound/soc/renesas/rz-ssi.c
+++ b/sound/soc/renesas/rz-ssi.c
@@ -13,6 +13,8 @@
#include <linux/module.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
+#include <sound/dmaengine_pcm.h>
+#include <sound/pcm.h>
#include <sound/pcm_params.h>
#include <sound/soc.h>
@@ -87,8 +89,6 @@ struct rz_ssi_stream {
struct rz_ssi_priv *priv;
struct snd_pcm_substream *substream;
int fifo_sample_size; /* sample capacity of SSI FIFO */
- int dma_buffer_pos; /* The address for the next DMA descriptor */
- int completed_dma_buf_pos; /* The address of the last completed DMA descriptor. */
int period_counter; /* for keeping track of periods transferred */
int buffer_pos; /* current frame position in the buffer */
int running; /* 0=stopped, 1=running */
@@ -96,8 +96,6 @@ struct rz_ssi_stream {
int uerr_num;
int oerr_num;
- struct dma_chan *dma_ch;
-
int (*transfer)(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm);
};
@@ -108,7 +106,6 @@ struct rz_ssi_priv {
struct clk *sfr_clk;
struct clk *clk;
- phys_addr_t phys;
int irq_int;
int irq_tx;
int irq_rx;
@@ -148,9 +145,10 @@ struct rz_ssi_priv {
unsigned int sample_width;
unsigned int sample_bits;
} hw_params_cache;
-};
-static void rz_ssi_dma_complete(void *data);
+ struct snd_dmaengine_dai_dma_data dma_dais[SNDRV_PCM_STREAM_LAST + 1];
+ struct dma_chan *dmas[SNDRV_PCM_STREAM_LAST + 1];
+};
static void rz_ssi_reg_writel(struct rz_ssi_priv *priv, uint reg, u32 data)
{
@@ -172,11 +170,6 @@ static void rz_ssi_reg_mask_setl(struct rz_ssi_priv *priv, uint reg,
writel(val, (priv->base + reg));
}
-static inline bool rz_ssi_stream_is_play(struct snd_pcm_substream *substream)
-{
- return substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
-}
-
static inline struct rz_ssi_stream *
rz_ssi_stream_get(struct rz_ssi_priv *ssi, struct snd_pcm_substream *substream)
{
@@ -185,7 +178,7 @@ rz_ssi_stream_get(struct rz_ssi_priv *ssi, struct snd_pcm_substream *substream)
static inline bool rz_ssi_is_dma_enabled(struct rz_ssi_priv *ssi)
{
- return (ssi->playback.dma_ch && (ssi->dma_rt || ssi->capture.dma_ch));
+ return !ssi->playback.transfer && !ssi->capture.transfer;
}
static void rz_ssi_set_substream(struct rz_ssi_stream *strm,
@@ -215,8 +208,6 @@ static void rz_ssi_stream_init(struct rz_ssi_stream *strm,
struct snd_pcm_substream *substream)
{
rz_ssi_set_substream(strm, substream);
- strm->dma_buffer_pos = 0;
- strm->completed_dma_buf_pos = 0;
strm->period_counter = 0;
strm->buffer_pos = 0;
@@ -242,12 +233,13 @@ static void rz_ssi_stream_quit(struct rz_ssi_priv *ssi,
dev_info(dev, "underrun = %d\n", strm->uerr_num);
}
-static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate,
- unsigned int channels)
+static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, struct snd_pcm_substream *substream,
+ unsigned int rate, unsigned int channels)
{
static u8 ckdv[] = { 1, 2, 4, 8, 16, 32, 64, 128, 6, 12, 24, 48, 96 };
unsigned int channel_bits = 32; /* System Word Length */
unsigned long bclk_rate = rate * channels * channel_bits;
+ struct snd_dmaengine_dai_dma_data *dma_dai;
unsigned int div;
unsigned int i;
u32 ssicr = 0;
@@ -290,6 +282,8 @@ static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate,
return -EINVAL;
}
+ dma_dai = &ssi->dma_dais[substream->stream];
+
/*
* DWL: Data Word Length = {16, 24, 32} bits
* SWL: System Word Length = 32 bits
@@ -298,12 +292,15 @@ static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate,
switch (ssi->hw_params_cache.sample_width) {
case 16:
ssicr |= SSICR_DWL(1);
+ dma_dai->addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
break;
case 24:
ssicr |= SSICR_DWL(5) | SSICR_PDTA;
+ dma_dai->addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
break;
case 32:
ssicr |= SSICR_DWL(6);
+ dma_dai->addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
break;
default:
dev_err(ssi->dev, "Not support %u data width",
@@ -344,7 +341,7 @@ static void rz_ssi_set_idle(struct rz_ssi_priv *ssi)
static int rz_ssi_start(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
{
- bool is_play = rz_ssi_stream_is_play(strm->substream);
+ bool is_play = strm->substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
bool is_full_duplex;
u32 ssicr, ssifcr;
@@ -423,14 +420,6 @@ static int rz_ssi_stop(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
/* Disable TX/RX */
rz_ssi_reg_mask_setl(ssi, SSICR, SSICR_TEN | SSICR_REN, 0);
- /* Cancel all remaining DMA transactions */
- if (rz_ssi_is_dma_enabled(ssi)) {
- if (ssi->playback.dma_ch)
- dmaengine_terminate_async(ssi->playback.dma_ch);
- if (ssi->capture.dma_ch)
- dmaengine_terminate_async(ssi->capture.dma_ch);
- }
-
rz_ssi_set_idle(ssi);
return 0;
@@ -458,10 +447,6 @@ static void rz_ssi_pointer_update(struct rz_ssi_stream *strm, int frames)
snd_pcm_period_elapsed(strm->substream);
strm->period_counter = current_period;
}
-
- strm->completed_dma_buf_pos += runtime->period_size;
- if (strm->completed_dma_buf_pos >= runtime->buffer_size)
- strm->completed_dma_buf_pos = 0;
}
static int rz_ssi_pio_recv(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
@@ -606,12 +591,6 @@ static irqreturn_t rz_ssi_interrupt(int irq, void *data)
if (irq == ssi->irq_int) { /* error or idle */
bool is_stopped = !!(ssisr & (SSISR_RUIRQ | SSISR_ROIRQ |
SSISR_TUIRQ | SSISR_TOIRQ));
- int i, count;
-
- if (rz_ssi_is_dma_enabled(ssi))
- count = 4;
- else
- count = 1;
if (ssi->capture.substream && is_stopped) {
if (ssisr & SSISR_RUIRQ)
@@ -631,19 +610,41 @@ static irqreturn_t rz_ssi_interrupt(int irq, void *data)
rz_ssi_stop(ssi, strm_playback);
}
+ if (!rz_ssi_is_stream_running(&ssi->playback) &&
+ !rz_ssi_is_stream_running(&ssi->capture) &&
+ rz_ssi_is_dma_enabled(ssi) && is_stopped) {
+ if (ssi->playback.substream &&
+ ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK])
+ dmaengine_pause(ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK]);
+ if (ssi->capture.substream &&
+ ssi->dmas[SNDRV_PCM_STREAM_CAPTURE] &&
+ /* Avoid calling pause twice in case of half duplex. */
+ ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK] !=
+ ssi->dmas[SNDRV_PCM_STREAM_CAPTURE])
+ dmaengine_pause(ssi->dmas[SNDRV_PCM_STREAM_CAPTURE]);
+ }
+
/* Clear all flags */
rz_ssi_reg_mask_setl(ssi, SSISR, SSISR_TOIRQ | SSISR_TUIRQ |
SSISR_ROIRQ | SSISR_RUIRQ, 0);
/* Add/remove more data */
if (ssi->capture.substream && is_stopped) {
- for (i = 0; i < count; i++)
+ if (rz_ssi_is_dma_enabled(ssi)) {
+ if (ssi->dmas[SNDRV_PCM_STREAM_CAPTURE])
+ dmaengine_resume(ssi->dmas[SNDRV_PCM_STREAM_CAPTURE]);
+ } else {
strm_capture->transfer(ssi, strm_capture);
+ }
}
if (ssi->playback.substream && is_stopped) {
- for (i = 0; i < count; i++)
+ if (rz_ssi_is_dma_enabled(ssi)) {
+ if (ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK])
+ dmaengine_resume(ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK]);
+ } else {
strm_playback->transfer(ssi, strm_playback);
+ }
}
/* Resume */
@@ -679,153 +680,11 @@ static irqreturn_t rz_ssi_interrupt(int irq, void *data)
return IRQ_HANDLED;
}
-static int rz_ssi_dma_slave_config(struct rz_ssi_priv *ssi,
- struct dma_chan *dma_ch, bool is_play)
-{
- struct dma_slave_config cfg;
-
- memset(&cfg, 0, sizeof(cfg));
-
- cfg.direction = is_play ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
- cfg.dst_addr = ssi->phys + SSIFTDR;
- cfg.src_addr = ssi->phys + SSIFRDR;
- if (ssi->hw_params_cache.sample_width == 16) {
- cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
- cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
- } else {
- cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
- cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
- }
-
- return dmaengine_slave_config(dma_ch, &cfg);
-}
-
-static int rz_ssi_dma_transfer(struct rz_ssi_priv *ssi,
- struct rz_ssi_stream *strm)
-{
- struct snd_pcm_substream *substream = strm->substream;
- struct dma_async_tx_descriptor *desc;
- struct snd_pcm_runtime *runtime;
- enum dma_transfer_direction dir;
- u32 dma_paddr, dma_size;
- int amount;
-
- if (!rz_ssi_stream_is_valid(ssi, strm))
- return -EINVAL;
-
- runtime = substream->runtime;
- if (runtime->state == SNDRV_PCM_STATE_DRAINING)
- /*
- * Stream is ending, so do not queue up any more DMA
- * transfers otherwise we play partial sound clips
- * because we can't shut off the DMA quick enough.
- */
- return 0;
-
- dir = rz_ssi_stream_is_play(substream) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
-
- /* Always transfer 1 period */
- amount = runtime->period_size;
-
- /* DMA physical address and size */
- dma_paddr = runtime->dma_addr + frames_to_bytes(runtime,
- strm->dma_buffer_pos);
- dma_size = frames_to_bytes(runtime, amount);
- desc = dmaengine_prep_slave_single(strm->dma_ch, dma_paddr, dma_size,
- dir,
- DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
- if (!desc) {
- dev_err(ssi->dev, "dmaengine_prep_slave_single() fail\n");
- return -ENOMEM;
- }
-
- desc->callback = rz_ssi_dma_complete;
- desc->callback_param = strm;
-
- if (dmaengine_submit(desc) < 0) {
- dev_err(ssi->dev, "dmaengine_submit() fail\n");
- return -EIO;
- }
-
- /* Update DMA pointer */
- strm->dma_buffer_pos += amount;
- if (strm->dma_buffer_pos >= runtime->buffer_size)
- strm->dma_buffer_pos = 0;
-
- /* Start DMA */
- dma_async_issue_pending(strm->dma_ch);
-
- return 0;
-}
-
-static void rz_ssi_dma_complete(void *data)
-{
- struct rz_ssi_stream *strm = (struct rz_ssi_stream *)data;
-
- if (!strm->running || !strm->substream || !strm->substream->runtime)
- return;
-
- /* Note that next DMA transaction has probably already started */
- rz_ssi_pointer_update(strm, strm->substream->runtime->period_size);
-
- /* Queue up another DMA transaction */
- rz_ssi_dma_transfer(strm->priv, strm);
-}
-
-static void rz_ssi_release_dma_channels(struct rz_ssi_priv *ssi)
-{
- if (ssi->playback.dma_ch) {
- dma_release_channel(ssi->playback.dma_ch);
- ssi->playback.dma_ch = NULL;
- if (ssi->dma_rt)
- ssi->dma_rt = false;
- }
-
- if (ssi->capture.dma_ch) {
- dma_release_channel(ssi->capture.dma_ch);
- ssi->capture.dma_ch = NULL;
- }
-}
-
-static int rz_ssi_dma_request(struct rz_ssi_priv *ssi, struct device *dev)
-{
- ssi->playback.dma_ch = dma_request_chan(dev, "tx");
- if (IS_ERR(ssi->playback.dma_ch))
- ssi->playback.dma_ch = NULL;
-
- ssi->capture.dma_ch = dma_request_chan(dev, "rx");
- if (IS_ERR(ssi->capture.dma_ch))
- ssi->capture.dma_ch = NULL;
-
- if (!ssi->playback.dma_ch && !ssi->capture.dma_ch) {
- ssi->playback.dma_ch = dma_request_chan(dev, "rt");
- if (IS_ERR(ssi->playback.dma_ch)) {
- ssi->playback.dma_ch = NULL;
- goto no_dma;
- }
-
- ssi->dma_rt = true;
- }
-
- if (!rz_ssi_is_dma_enabled(ssi))
- goto no_dma;
-
- return 0;
-
-no_dma:
- rz_ssi_release_dma_channels(ssi);
-
- return -ENODEV;
-}
-
static int rz_ssi_trigger_resume(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
{
struct snd_pcm_substream *substream = strm->substream;
- struct snd_pcm_runtime *runtime = substream->runtime;
int ret;
- strm->dma_buffer_pos = strm->completed_dma_buf_pos + runtime->period_size;
-
if (rz_ssi_is_stream_running(&ssi->playback) ||
rz_ssi_is_stream_running(&ssi->capture))
return 0;
@@ -834,7 +693,7 @@ static int rz_ssi_trigger_resume(struct rz_ssi_priv *ssi, struct rz_ssi_stream *
if (ret)
return ret;
- return rz_ssi_clk_setup(ssi, ssi->hw_params_cache.rate,
+ return rz_ssi_clk_setup(ssi, substream, ssi->hw_params_cache.rate,
ssi->hw_params_cache.channels);
}
@@ -843,10 +702,11 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd,
{
struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai);
struct rz_ssi_stream *strm = rz_ssi_stream_get(ssi, substream);
- int ret = 0, i, num_transfer = 1;
+ int ret = 0;
switch (cmd) {
case SNDRV_PCM_TRIGGER_RESUME:
+ case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
ret = rz_ssi_trigger_resume(ssi, strm);
if (ret)
return ret;
@@ -857,28 +717,7 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd,
if (cmd == SNDRV_PCM_TRIGGER_START)
rz_ssi_stream_init(strm, substream);
- if (rz_ssi_is_dma_enabled(ssi)) {
- bool is_playback = rz_ssi_stream_is_play(substream);
-
- if (ssi->dma_rt)
- ret = rz_ssi_dma_slave_config(ssi, ssi->playback.dma_ch,
- is_playback);
- else
- ret = rz_ssi_dma_slave_config(ssi, strm->dma_ch,
- is_playback);
-
- /* Fallback to pio */
- if (ret < 0) {
- ssi->playback.transfer = rz_ssi_pio_send;
- ssi->capture.transfer = rz_ssi_pio_recv;
- rz_ssi_release_dma_channels(ssi);
- } else {
- /* For DMA, queue up multiple DMA descriptors */
- num_transfer = 4;
- }
- }
-
- for (i = 0; i < num_transfer; i++) {
+ if (!rz_ssi_is_dma_enabled(ssi)) {
ret = strm->transfer(ssi, strm);
if (ret)
return ret;
@@ -888,6 +727,7 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd,
break;
case SNDRV_PCM_TRIGGER_SUSPEND:
+ case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
rz_ssi_stop(ssi, strm);
break;
@@ -973,6 +813,8 @@ static void rz_ssi_shutdown(struct snd_pcm_substream *substream,
ssi->dup.tx_active = false;
else
ssi->dup.rx_active = false;
+
+ ssi->dmas[substream->stream] = NULL;
}
static bool rz_ssi_is_valid_hw_params(struct rz_ssi_priv *ssi, unsigned int rate,
@@ -1024,6 +866,12 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream,
return -EINVAL;
}
+ /* Save the DMA channels for recovery. */
+ if (rz_ssi_is_dma_enabled(ssi))
+ ssi->dmas[substream->stream] = snd_dmaengine_pcm_get_chan(substream);
+ else
+ ssi->dmas[substream->stream] = NULL;
+
if (rz_ssi_is_stream_running(&ssi->playback) ||
rz_ssi_is_stream_running(&ssi->capture)) {
if (rz_ssi_is_valid_hw_params(ssi, rate, channels, sample_width, sample_bits))
@@ -1039,10 +887,21 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream,
if (ret)
return ret;
- return rz_ssi_clk_setup(ssi, rate, channels);
+ return rz_ssi_clk_setup(ssi, substream, rate, channels);
+}
+
+static int rz_ssi_dai_probe(struct snd_soc_dai *dai)
+{
+ struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai);
+
+ snd_soc_dai_init_dma_data(dai, &ssi->dma_dais[SNDRV_PCM_STREAM_PLAYBACK],
+ &ssi->dma_dais[SNDRV_PCM_STREAM_CAPTURE]);
+
+ return 0;
}
static const struct snd_soc_dai_ops rz_ssi_dai_ops = {
+ .probe = rz_ssi_dai_probe,
.startup = rz_ssi_startup,
.shutdown = rz_ssi_shutdown,
.trigger = rz_ssi_dai_trigger,
@@ -1054,10 +913,11 @@ static const struct snd_pcm_hardware rz_ssi_pcm_hardware = {
.info = SNDRV_PCM_INFO_INTERLEAVED |
SNDRV_PCM_INFO_MMAP |
SNDRV_PCM_INFO_MMAP_VALID |
- SNDRV_PCM_INFO_RESUME,
- .buffer_bytes_max = PREALLOC_BUFFER,
+ SNDRV_PCM_INFO_RESUME |
+ SNDRV_PCM_INFO_PAUSE,
+ .buffer_bytes_max = 192 * 1024,
.period_bytes_min = 32,
- .period_bytes_max = 8192,
+ .period_bytes_max = 48 * 1024,
.channels_min = SSI_CHAN_MIN,
.channels_max = SSI_CHAN_MAX,
.periods_min = 1,
@@ -1065,8 +925,8 @@ static const struct snd_pcm_hardware rz_ssi_pcm_hardware = {
.fifo_size = 32 * 2,
};
-static int rz_ssi_pcm_open(struct snd_soc_component *component,
- struct snd_pcm_substream *substream)
+static int rz_ssi_pcm_open_pio(struct snd_soc_component *component,
+ struct snd_pcm_substream *substream)
{
snd_soc_set_runtime_hwparams(substream, &rz_ssi_pcm_hardware);
@@ -1074,6 +934,13 @@ static int rz_ssi_pcm_open(struct snd_soc_component *component,
SNDRV_PCM_HW_PARAM_PERIODS);
}
+static int rz_ssi_pcm_open_dma(struct snd_soc_component *component,
+ struct snd_pcm_substream *substream)
+{
+ return snd_pcm_hw_constraint_integer(substream->runtime,
+ SNDRV_PCM_HW_PARAM_PERIODS);
+}
+
static snd_pcm_uframes_t rz_ssi_pcm_pointer(struct snd_soc_component *component,
struct snd_pcm_substream *substream)
{
@@ -1090,7 +957,8 @@ static int rz_ssi_pcm_new(struct snd_soc_component *component,
{
snd_pcm_set_managed_buffer_all(rtd->pcm, SNDRV_DMA_TYPE_DEV,
rtd->card->snd_card->dev,
- PREALLOC_BUFFER, PREALLOC_BUFFER_MAX);
+ rz_ssi_pcm_hardware.buffer_bytes_max,
+ rz_ssi_pcm_hardware.buffer_bytes_max);
return 0;
}
@@ -1113,16 +981,30 @@ static struct snd_soc_dai_driver rz_ssi_soc_dai[] = {
},
};
-static const struct snd_soc_component_driver rz_ssi_soc_component = {
+static const struct snd_soc_component_driver rz_ssi_soc_component_pio = {
.name = "rz-ssi",
- .open = rz_ssi_pcm_open,
+ .open = rz_ssi_pcm_open_pio,
.pointer = rz_ssi_pcm_pointer,
.pcm_new = rz_ssi_pcm_new,
.legacy_dai_naming = 1,
};
+static const struct snd_soc_component_driver rz_ssi_soc_component_dma = {
+ .name = "rz-ssi",
+ .open = rz_ssi_pcm_open_dma,
+ .legacy_dai_naming = 1,
+};
+
+static const struct snd_dmaengine_pcm_config rz_ssi_dmaengine_pcm_conf = {
+ .pcm_hardware = &rz_ssi_pcm_hardware,
+ .prealloc_buffer_size = 192 * 1024,
+ .prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
+};
+
static int rz_ssi_probe(struct platform_device *pdev)
{
+ const struct snd_soc_component_driver *component_driver;
+ struct device_node *np = pdev->dev.of_node;
struct device *dev = &pdev->dev;
struct rz_ssi_priv *ssi;
struct clk *audio_clk;
@@ -1138,7 +1020,6 @@ static int rz_ssi_probe(struct platform_device *pdev)
if (IS_ERR(ssi->base))
return PTR_ERR(ssi->base);
- ssi->phys = res->start;
ssi->clk = devm_clk_get(dev, "ssi");
if (IS_ERR(ssi->clk))
return PTR_ERR(ssi->clk);
@@ -1162,16 +1043,43 @@ static int rz_ssi_probe(struct platform_device *pdev)
ssi->audio_mck = ssi->audio_clk_1 ? ssi->audio_clk_1 : ssi->audio_clk_2;
- /* Detect DMA support */
- ret = rz_ssi_dma_request(ssi, dev);
- if (ret < 0) {
+ ssi->dma_dais[SNDRV_PCM_STREAM_PLAYBACK].addr = (dma_addr_t)res->start + SSIFTDR;
+ ssi->dma_dais[SNDRV_PCM_STREAM_CAPTURE].addr = (dma_addr_t)res->start + SSIFRDR;
+
+ if (of_property_present(np, "dma-names")) {
+ struct snd_dmaengine_pcm_config *config;
+ unsigned int flags = 0;
+
+ config = devm_kzalloc(dev, sizeof(*config), GFP_KERNEL);
+ if (!config)
+ return -ENOMEM;
+
+ config->pcm_hardware = rz_ssi_dmaengine_pcm_conf.pcm_hardware;
+ config->prealloc_buffer_size = rz_ssi_dmaengine_pcm_conf.prealloc_buffer_size;
+ config->prepare_slave_config = rz_ssi_dmaengine_pcm_conf.prepare_slave_config;
+
+ if (of_property_match_string(np, "dma-names", "rt") == 0) {
+ flags = SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX;
+ config->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = "rt";
+ } else {
+ config->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = "tx";
+ config->chan_names[SNDRV_PCM_STREAM_CAPTURE] = "rx";
+ }
+ ret = devm_snd_dmaengine_pcm_register(&pdev->dev, config, flags);
+ } else {
+ ret = -ENODEV;
+ }
+
+ if (ret == -EPROBE_DEFER) {
+ return ret;
+ } else if (ret) {
dev_warn(dev, "DMA not available, using PIO\n");
ssi->playback.transfer = rz_ssi_pio_send;
ssi->capture.transfer = rz_ssi_pio_recv;
+ component_driver = &rz_ssi_soc_component_pio;
} else {
- dev_info(dev, "DMA enabled");
- ssi->playback.transfer = rz_ssi_dma_transfer;
- ssi->capture.transfer = rz_ssi_dma_transfer;
+ dev_info(dev, "DMA enabled\n");
+ component_driver = &rz_ssi_soc_component_dma;
}
ssi->playback.priv = ssi;
@@ -1182,17 +1090,13 @@ static int rz_ssi_probe(struct platform_device *pdev)
/* Error Interrupt */
ssi->irq_int = platform_get_irq_byname(pdev, "int_req");
- if (ssi->irq_int < 0) {
- ret = ssi->irq_int;
- goto err_release_dma_chs;
- }
+ if (ssi->irq_int < 0)
+ return ssi->irq_int;
ret = devm_request_irq(dev, ssi->irq_int, rz_ssi_interrupt,
0, dev_name(dev), ssi);
- if (ret < 0) {
- dev_err_probe(dev, ret, "irq request error (int_req)\n");
- goto err_release_dma_chs;
- }
+ if (ret < 0)
+ return dev_err_probe(dev, ret, "irq request error (int_req)\n");
if (!rz_ssi_is_dma_enabled(ssi)) {
/* Tx and Rx interrupts (pio only) */
@@ -1233,43 +1137,19 @@ static int rz_ssi_probe(struct platform_device *pdev)
}
ssi->rstc = devm_reset_control_get_exclusive(dev, NULL);
- if (IS_ERR(ssi->rstc)) {
- ret = PTR_ERR(ssi->rstc);
- goto err_release_dma_chs;
- }
+ if (IS_ERR(ssi->rstc))
+ return dev_err_probe(dev, PTR_ERR(ssi->rstc), "Failed to get reset\n");
/* Default 0 for power saving. Can be overridden via sysfs. */
pm_runtime_set_autosuspend_delay(dev, 0);
pm_runtime_use_autosuspend(dev);
ret = devm_pm_runtime_enable(dev);
- if (ret < 0) {
- dev_err(dev, "Failed to enable runtime PM!\n");
- goto err_release_dma_chs;
- }
+ if (ret < 0)
+ return dev_err_probe(dev, ret, "Failed to enable runtime PM!\n");
- ret = devm_snd_soc_register_component(dev, &rz_ssi_soc_component,
- rz_ssi_soc_dai,
- ARRAY_SIZE(rz_ssi_soc_dai));
- if (ret < 0) {
- dev_err(dev, "failed to register snd component\n");
- goto err_release_dma_chs;
- }
-
- return 0;
-
-err_release_dma_chs:
- rz_ssi_release_dma_channels(ssi);
-
- return ret;
-}
-
-static void rz_ssi_remove(struct platform_device *pdev)
-{
- struct rz_ssi_priv *ssi = dev_get_drvdata(&pdev->dev);
-
- rz_ssi_release_dma_channels(ssi);
-
- reset_control_assert(ssi->rstc);
+ return devm_snd_soc_register_component(dev, component_driver,
+ rz_ssi_soc_dai,
+ ARRAY_SIZE(rz_ssi_soc_dai));
}
static const struct of_device_id rz_ssi_of_match[] = {
@@ -1304,7 +1184,6 @@ static struct platform_driver rz_ssi_driver = {
.pm = pm_ptr(&rz_ssi_pm_ops),
},
.probe = rz_ssi_probe,
- .remove = rz_ssi_remove,
};
module_platform_driver(rz_ssi_driver);
