From 5febf432df1cfa5b25d99b54c32103fafbdd0eb9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 16 Aug 2026 15:32:33 +0200 Subject: kbuild: rust: keep Rust objects out of Clang LTO with inline helpers Under `CONFIG_LTO_CLANG` + `CONFIG_RUST_INLINE_HELPERS`, one may hit `objtool` errors such as: vmlinux.o: error: objtool: _R..._3Gsp4boot+0xd6a: can't find jump dest instruction at .text._R..._3Gsp4boot+0x1dfd The reason is that in such builds, the Clang invocation that compiles the combined Rust plus helpers bitcode emits LLVM bitcode (again) -- the final code generation happens in the linker's LTO step, which the `-mllvm` trap options passed to Clang do not reach. This, in turn, means that unreachable traps are missing, and the impossible paths do not merely fallthrough to the next symbol, but past the end of their own section, since LTO builds place each function in its own section. Thus filter `CC_FLAGS_LTO` out of the Clang invocation, so that it always emits machine code directly, with the traps in place. Assisted-by: LLM Cc: Gary Guo Cc: Boqun Feng Cc: Alice Ryhl Cc: Matthew Maurer Cc: Josh Poimboeuf Cc: Peter Zijlstra Cc: stable@vger.kernel.org Fixes: 3a2486cc1da5 ("kbuild: rust: provide an option to inline C helpers into Rust") Acked-by: Gary Guo Link: https://patch.msgid.link/20260816133233.197500-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- drivers/dma/dw/Kconfig | 36 ++ drivers/dma/dw/Makefile | 13 + drivers/dma/dw/acpi.c | 57 ++ drivers/dma/dw/core.c | 1406 ++++++++++++++++++++++++++++++++++++++++++ drivers/dma/dw/dw.c | 145 +++++ drivers/dma/dw/idma32.c | 294 +++++++++ drivers/dma/dw/internal.h | 101 +++ drivers/dma/dw/of.c | 122 ++++ drivers/dma/dw/pci.c | 143 +++++ drivers/dma/dw/platform.c | 211 +++++++ drivers/dma/dw/regs.h | 408 ++++++++++++ drivers/dma/dw/rzn1-dmamux.c | 169 +++++ 12 files changed, 3105 insertions(+) create mode 100644 drivers/dma/dw/Kconfig create mode 100644 drivers/dma/dw/Makefile create mode 100644 drivers/dma/dw/acpi.c create mode 100644 drivers/dma/dw/core.c create mode 100644 drivers/dma/dw/dw.c create mode 100644 drivers/dma/dw/idma32.c create mode 100644 drivers/dma/dw/internal.h create mode 100644 drivers/dma/dw/of.c create mode 100644 drivers/dma/dw/pci.c create mode 100644 drivers/dma/dw/platform.c create mode 100644 drivers/dma/dw/regs.h create mode 100644 drivers/dma/dw/rzn1-dmamux.c (limited to 'drivers/dma/dw') diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig new file mode 100644 index 000000000..a9828ddd6 --- /dev/null +++ b/drivers/dma/dw/Kconfig @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0 + +# +# DMA engine configuration for dw +# + +config DW_DMAC_CORE + tristate + select DMA_ENGINE + +config DW_DMAC + tristate "Synopsys DesignWare AHB DMA platform driver" + depends on HAS_IOMEM + select DW_DMAC_CORE + help + Support the Synopsys DesignWare AHB DMA controller. This + can be integrated in chips such as the Intel Cherrytrail. + +config RZN1_DMAMUX + tristate "Renesas RZ/N1 DMAMUX driver" + depends on DW_DMAC + depends on ARCH_RZN1 || COMPILE_TEST + help + Support the Renesas RZ/N1 DMAMUX which is located in front of + the Synopsys DesignWare AHB DMA controller located on Renesas + SoCs. + +config DW_DMAC_PCI + tristate "Synopsys DesignWare AHB DMA PCI driver" + depends on PCI + depends on HAS_IOMEM + select DW_DMAC_CORE + help + Support the Synopsys DesignWare AHB DMA controller on the + platforms that enumerate it as a PCI device. For example, + Intel Medfield has integrated this GPDMA controller. diff --git a/drivers/dma/dw/Makefile b/drivers/dma/dw/Makefile new file mode 100644 index 000000000..e1796015f --- /dev/null +++ b/drivers/dma/dw/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_DW_DMAC_CORE) += dw_dmac_core.o +dw_dmac_core-y := core.o dw.o idma32.o +dw_dmac_core-$(CONFIG_ACPI) += acpi.o + +obj-$(CONFIG_DW_DMAC) += dw_dmac.o +dw_dmac-y := platform.o +dw_dmac-$(CONFIG_OF) += of.o + +obj-$(CONFIG_DW_DMAC_PCI) += dw_dmac_pci.o +dw_dmac_pci-y := pci.o + +obj-$(CONFIG_RZN1_DMAMUX) += rzn1-dmamux.o diff --git a/drivers/dma/dw/acpi.c b/drivers/dma/dw/acpi.c new file mode 100644 index 000000000..b6452fffa --- /dev/null +++ b/drivers/dma/dw/acpi.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2013,2019 Intel Corporation + +#include +#include + +#include "internal.h" + +static bool dw_dma_acpi_filter(struct dma_chan *chan, void *param) +{ + struct dw_dma *dw = to_dw_dma(chan->device); + struct dw_dma_chip_pdata *data = dev_get_drvdata(dw->dma.dev); + struct acpi_dma_spec *dma_spec = param; + struct dw_dma_slave slave = { + .dma_dev = dma_spec->dev, + .src_id = dma_spec->slave_id, + .dst_id = dma_spec->slave_id, + .m_master = data->m_master, + .p_master = data->p_master, + }; + + return dw_dma_filter(chan, &slave); +} + +void dw_dma_acpi_controller_register(struct dw_dma *dw) +{ + struct device *dev = dw->dma.dev; + struct acpi_dma_filter_info *info; + int ret; + + if (!has_acpi_companion(dev)) + return; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return; + + dma_cap_zero(info->dma_cap); + dma_cap_set(DMA_SLAVE, info->dma_cap); + info->filter_fn = dw_dma_acpi_filter; + + ret = acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info); + if (ret) + dev_err(dev, "could not register acpi_dma_controller\n"); +} +EXPORT_SYMBOL_GPL(dw_dma_acpi_controller_register); + +void dw_dma_acpi_controller_free(struct dw_dma *dw) +{ + struct device *dev = dw->dma.dev; + + if (!has_acpi_companion(dev)) + return; + + acpi_dma_controller_free(dev); +} +EXPORT_SYMBOL_GPL(dw_dma_acpi_controller_free); diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c new file mode 100644 index 000000000..dd75f97a3 --- /dev/null +++ b/drivers/dma/dw/core.c @@ -0,0 +1,1406 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Core driver for the Synopsys DesignWare DMA Controller + * + * Copyright (C) 2007-2008 Atmel Corporation + * Copyright (C) 2010-2011 ST Microelectronics + * Copyright (C) 2013 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" +#include "internal.h" + +/* + * This supports the Synopsys "DesignWare AHB Central DMA Controller", + * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all + * of which use ARM any more). See the "Databook" from Synopsys for + * information beyond what licensees probably provide. + */ + +/* The set of bus widths supported by the DMA controller */ +#define DW_DMA_BUSWIDTHS \ + BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) + +/*----------------------------------------------------------------------*/ + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc) +{ + return to_dw_desc(dwc->active_list.next); +} + +static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct dw_desc *desc = txd_to_dw_desc(tx); + struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan); + dma_cookie_t cookie; + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + cookie = dma_cookie_assign(tx); + + /* + * REVISIT: We should attempt to chain as many descriptors as + * possible, perhaps even appending to those already submitted + * for DMA. But this is hard to do in a race-free manner. + */ + + list_add_tail(&desc->desc_node, &dwc->queue); + spin_unlock_irqrestore(&dwc->lock, flags); + dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", + __func__, desc->txd.cookie); + + return cookie; +} + +static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + struct dw_desc *desc; + dma_addr_t phys; + + desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys); + if (!desc) + return NULL; + + dwc->descs_allocated++; + INIT_LIST_HEAD(&desc->tx_list); + dma_async_tx_descriptor_init(&desc->txd, &dwc->chan); + desc->txd.tx_submit = dwc_tx_submit; + desc->txd.flags = DMA_CTRL_ACK; + desc->txd.phys = phys; + return desc; +} + +static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + struct dw_desc *child, *_next; + + if (unlikely(!desc)) + return; + + list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) { + list_del(&child->desc_node); + dma_pool_free(dw->desc_pool, child, child->txd.phys); + dwc->descs_allocated--; + } + + dma_pool_free(dw->desc_pool, desc, desc->txd.phys); + dwc->descs_allocated--; +} + +static void dwc_initialize(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + + dw->initialize_chan(dwc); + + /* Enable interrupts */ + channel_set_bit(dw, MASK.XFER, dwc->mask); + channel_set_bit(dw, MASK.ERROR, dwc->mask); +} + +/*----------------------------------------------------------------------*/ + +static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc) +{ + dev_err(chan2dev(&dwc->chan), + " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n", + channel_readl(dwc, SAR), + channel_readl(dwc, DAR), + channel_readl(dwc, LLP), + channel_readl(dwc, CTL_HI), + channel_readl(dwc, CTL_LO)); +} + +static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc) +{ + channel_clear_bit(dw, CH_EN, dwc->mask); + while (dma_readl(dw, CH_EN) & dwc->mask) + cpu_relax(); +} + +/*----------------------------------------------------------------------*/ + +/* Perform single block transfer */ +static inline void dwc_do_single_block(struct dw_dma_chan *dwc, + struct dw_desc *desc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + u32 ctllo; + + /* + * Software emulation of LLP mode relies on interrupts to continue + * multi block transfer. + */ + ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN; + + channel_writel(dwc, SAR, lli_read(desc, sar)); + channel_writel(dwc, DAR, lli_read(desc, dar)); + channel_writel(dwc, CTL_LO, ctllo); + channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi)); + channel_set_bit(dw, CH_EN, dwc->mask); + + /* Move pointer to next descriptor */ + dwc->tx_node_active = dwc->tx_node_active->next; +} + +/* Called with dwc->lock held and bh disabled */ +static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + u8 lms = DWC_LLP_LMS(dwc->dws.m_master); + unsigned long was_soft_llp; + + /* ASSERT: channel is idle */ + if (dma_readl(dw, CH_EN) & dwc->mask) { + dev_err(chan2dev(&dwc->chan), + "%s: BUG: Attempted to start non-idle channel\n", + __func__); + dwc_dump_chan_regs(dwc); + + /* The tasklet will hopefully advance the queue... */ + return; + } + + if (dwc->nollp) { + was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP, + &dwc->flags); + if (was_soft_llp) { + dev_err(chan2dev(&dwc->chan), + "BUG: Attempted to start new LLP transfer inside ongoing one\n"); + return; + } + + dwc_initialize(dwc); + + first->residue = first->total_len; + dwc->tx_node_active = &first->tx_list; + + /* Submit first block */ + dwc_do_single_block(dwc, first); + + return; + } + + dwc_initialize(dwc); + + channel_writel(dwc, LLP, first->txd.phys | lms); + channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN); + channel_writel(dwc, CTL_HI, 0); + channel_set_bit(dw, CH_EN, dwc->mask); +} + +static void dwc_dostart_first_queued(struct dw_dma_chan *dwc) +{ + struct dw_desc *desc; + + if (list_empty(&dwc->queue)) + return; + + list_move(dwc->queue.next, &dwc->active_list); + desc = dwc_first_active(dwc); + dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie); + dwc_dostart(dwc, desc); +} + +/*----------------------------------------------------------------------*/ + +static void +dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc, + bool callback_required) +{ + struct dma_async_tx_descriptor *txd = &desc->txd; + struct dw_desc *child; + unsigned long flags; + struct dmaengine_desc_callback cb; + + dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie); + + spin_lock_irqsave(&dwc->lock, flags); + dma_cookie_complete(txd); + if (callback_required) + dmaengine_desc_get_callback(txd, &cb); + else + memset(&cb, 0, sizeof(cb)); + + /* async_tx_ack */ + list_for_each_entry(child, &desc->tx_list, desc_node) + async_tx_ack(&child->txd); + async_tx_ack(&desc->txd); + dwc_desc_put(dwc, desc); + spin_unlock_irqrestore(&dwc->lock, flags); + + dmaengine_desc_callback_invoke(&cb, NULL); +} + +static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc) +{ + struct dw_desc *desc, *_desc; + LIST_HEAD(list); + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + if (dma_readl(dw, CH_EN) & dwc->mask) { + dev_err(chan2dev(&dwc->chan), + "BUG: XFER bit set, but channel not idle!\n"); + + /* Try to continue after resetting the channel... */ + dwc_chan_disable(dw, dwc); + } + + /* + * Submit queued descriptors ASAP, i.e. before we go through + * the completed ones. + */ + list_splice_init(&dwc->active_list, &list); + dwc_dostart_first_queued(dwc); + + spin_unlock_irqrestore(&dwc->lock, flags); + + list_for_each_entry_safe(desc, _desc, &list, desc_node) + dwc_descriptor_complete(dwc, desc, true); +} + +/* Returns how many bytes were already received from source */ +static inline u32 dwc_get_sent(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + u32 ctlhi = channel_readl(dwc, CTL_HI); + u32 ctllo = channel_readl(dwc, CTL_LO); + + return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7); +} + +static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) +{ + dma_addr_t llp; + struct dw_desc *desc, *_desc; + struct dw_desc *child; + u32 status_xfer; + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + llp = channel_readl(dwc, LLP); + status_xfer = dma_readl(dw, RAW.XFER); + + if (status_xfer & dwc->mask) { + /* Everything we've submitted is done */ + dma_writel(dw, CLEAR.XFER, dwc->mask); + + if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { + struct list_head *head, *active = dwc->tx_node_active; + + /* + * We are inside first active descriptor. + * Otherwise something is really wrong. + */ + desc = dwc_first_active(dwc); + + head = &desc->tx_list; + if (active != head) { + /* Update residue to reflect last sent descriptor */ + if (active == head->next) + desc->residue -= desc->len; + else + desc->residue -= to_dw_desc(active->prev)->len; + + child = to_dw_desc(active); + + /* Submit next block */ + dwc_do_single_block(dwc, child); + + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + + /* We are done here */ + clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); + } + + spin_unlock_irqrestore(&dwc->lock, flags); + + dwc_complete_all(dw, dwc); + return; + } + + if (list_empty(&dwc->active_list)) { + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + + if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { + dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__); + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + + dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp); + + list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) { + /* Initial residue value */ + desc->residue = desc->total_len; + + /* Check first descriptors addr */ + if (desc->txd.phys == DWC_LLP_LOC(llp)) { + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + + /* Check first descriptors llp */ + if (lli_read(desc, llp) == llp) { + /* This one is currently in progress */ + desc->residue -= dwc_get_sent(dwc); + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + + desc->residue -= desc->len; + list_for_each_entry(child, &desc->tx_list, desc_node) { + if (lli_read(child, llp) == llp) { + /* Currently in progress */ + desc->residue -= dwc_get_sent(dwc); + spin_unlock_irqrestore(&dwc->lock, flags); + return; + } + desc->residue -= child->len; + } + + /* + * No descriptors so far seem to be in progress, i.e. + * this one must be done. + */ + spin_unlock_irqrestore(&dwc->lock, flags); + dwc_descriptor_complete(dwc, desc, true); + spin_lock_irqsave(&dwc->lock, flags); + } + + dev_err(chan2dev(&dwc->chan), + "BUG: All descriptors done, but channel not idle!\n"); + + /* Try to continue after resetting the channel... */ + dwc_chan_disable(dw, dwc); + + dwc_dostart_first_queued(dwc); + spin_unlock_irqrestore(&dwc->lock, flags); +} + +static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc) +{ + dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n", + lli_read(desc, sar), + lli_read(desc, dar), + lli_read(desc, llp), + lli_read(desc, ctlhi), + lli_read(desc, ctllo)); +} + +static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc) +{ + struct dw_desc *bad_desc; + struct dw_desc *child; + unsigned long flags; + + dwc_scan_descriptors(dw, dwc); + + spin_lock_irqsave(&dwc->lock, flags); + + /* + * The descriptor currently at the head of the active list is + * borked. Since we don't have any way to report errors, we'll + * just have to scream loudly and try to carry on. + */ + bad_desc = dwc_first_active(dwc); + list_del_init(&bad_desc->desc_node); + list_move(dwc->queue.next, dwc->active_list.prev); + + /* Clear the error flag and try to restart the controller */ + dma_writel(dw, CLEAR.ERROR, dwc->mask); + if (!list_empty(&dwc->active_list)) + dwc_dostart(dwc, dwc_first_active(dwc)); + + /* + * WARN may seem harsh, but since this only happens + * when someone submits a bad physical address in a + * descriptor, we should consider ourselves lucky that the + * controller flagged an error instead of scribbling over + * random memory locations. + */ + dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n" + " cookie: %d\n", bad_desc->txd.cookie); + dwc_dump_lli(dwc, bad_desc); + list_for_each_entry(child, &bad_desc->tx_list, desc_node) + dwc_dump_lli(dwc, child); + + spin_unlock_irqrestore(&dwc->lock, flags); + + /* Pretend the descriptor completed successfully */ + dwc_descriptor_complete(dwc, bad_desc, true); +} + +static void dw_dma_tasklet(struct tasklet_struct *t) +{ + struct dw_dma *dw = from_tasklet(dw, t, tasklet); + struct dw_dma_chan *dwc; + u32 status_xfer; + u32 status_err; + unsigned int i; + + status_xfer = dma_readl(dw, RAW.XFER); + status_err = dma_readl(dw, RAW.ERROR); + + dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err); + + for (i = 0; i < dw->dma.chancnt; i++) { + dwc = &dw->chan[i]; + if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) + dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n"); + else if (status_err & (1 << i)) + dwc_handle_error(dw, dwc); + else if (status_xfer & (1 << i)) + dwc_scan_descriptors(dw, dwc); + } + + /* Re-enable interrupts */ + channel_set_bit(dw, MASK.XFER, dw->all_chan_mask); + channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask); +} + +static irqreturn_t dw_dma_interrupt(int irq, void *dev_id) +{ + struct dw_dma *dw = dev_id; + u32 status; + + /* Check if we have any interrupt from the DMAC which is not in use */ + if (!dw->in_use) + return IRQ_NONE; + + status = dma_readl(dw, STATUS_INT); + dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status); + + /* Check if we have any interrupt from the DMAC */ + if (!status) + return IRQ_NONE; + + /* + * Just disable the interrupts. We'll turn them back on in the + * softirq handler. + */ + channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask); + channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask); + channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask); + + status = dma_readl(dw, STATUS_INT); + if (status) { + dev_err(dw->dma.dev, + "BUG: Unexpected interrupts pending: 0x%x\n", + status); + + /* Try to recover */ + channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1); + channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1); + channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1); + channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1); + channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1); + } + + tasklet_schedule(&dw->tasklet); + + return IRQ_HANDLED; +} + +/*----------------------------------------------------------------------*/ + +static struct dma_async_tx_descriptor * +dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + struct dw_desc *desc; + struct dw_desc *first; + struct dw_desc *prev; + size_t xfer_count; + size_t offset; + u8 m_master = dwc->dws.m_master; + unsigned int src_width; + unsigned int dst_width; + unsigned int data_width = dw->pdata->data_width[m_master]; + u32 ctllo, ctlhi; + u8 lms = DWC_LLP_LMS(m_master); + + dev_vdbg(chan2dev(chan), + "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__, + &dest, &src, len, flags); + + if (unlikely(!len)) { + dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); + return NULL; + } + + dwc->direction = DMA_MEM_TO_MEM; + + src_width = dst_width = __ffs(data_width | src | dest | len); + + ctllo = dw->prepare_ctllo(dwc) + | DWC_CTLL_DST_WIDTH(dst_width) + | DWC_CTLL_SRC_WIDTH(src_width) + | DWC_CTLL_DST_INC + | DWC_CTLL_SRC_INC + | DWC_CTLL_FC_M2M; + prev = first = NULL; + + for (offset = 0; offset < len; offset += xfer_count) { + desc = dwc_desc_get(dwc); + if (!desc) + goto err_desc_get; + + ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count); + + lli_write(desc, sar, src + offset); + lli_write(desc, dar, dest + offset); + lli_write(desc, ctllo, ctllo); + lli_write(desc, ctlhi, ctlhi); + desc->len = xfer_count; + + if (!first) { + first = desc; + } else { + lli_write(prev, llp, desc->txd.phys | lms); + list_add_tail(&desc->desc_node, &first->tx_list); + } + prev = desc; + } + + if (flags & DMA_PREP_INTERRUPT) + /* Trigger interrupt after last block */ + lli_set(prev, ctllo, DWC_CTLL_INT_EN); + + prev->lli.llp = 0; + lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN); + first->txd.flags = flags; + first->total_len = len; + + return &first->txd; + +err_desc_get: + dwc_desc_put(dwc, first); + return NULL; +} + +static struct dma_async_tx_descriptor * +dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + struct dma_slave_config *sconfig = &dwc->dma_sconfig; + struct dw_desc *prev; + struct dw_desc *first; + u32 ctllo, ctlhi; + u8 lms = DWC_LLP_LMS(dwc->dws.m_master); + dma_addr_t reg; + unsigned int reg_width; + unsigned int mem_width; + unsigned int i; + struct scatterlist *sg; + size_t total_len = 0; + + dev_vdbg(chan2dev(chan), "%s\n", __func__); + + if (unlikely(!is_slave_direction(direction) || !sg_len)) + return NULL; + + dwc->direction = direction; + + prev = first = NULL; + + switch (direction) { + case DMA_MEM_TO_DEV: + reg_width = __ffs(sconfig->dst_addr_width); + reg = sconfig->dst_addr; + ctllo = dw->prepare_ctllo(dwc) + | DWC_CTLL_DST_WIDTH(reg_width) + | DWC_CTLL_DST_FIX + | DWC_CTLL_SRC_INC; + + ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) : + DWC_CTLL_FC(DW_DMA_FC_D_M2P); + + for_each_sg(sgl, sg, sg_len, i) { + struct dw_desc *desc; + u32 len, mem; + size_t dlen; + + mem = sg_dma_address(sg); + len = sg_dma_len(sg); + + mem_width = __ffs(sconfig->src_addr_width | mem | len); + +slave_sg_todev_fill_desc: + desc = dwc_desc_get(dwc); + if (!desc) + goto err_desc_get; + + ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen); + + lli_write(desc, sar, mem); + lli_write(desc, dar, reg); + lli_write(desc, ctlhi, ctlhi); + lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width)); + desc->len = dlen; + + if (!first) { + first = desc; + } else { + lli_write(prev, llp, desc->txd.phys | lms); + list_add_tail(&desc->desc_node, &first->tx_list); + } + prev = desc; + + mem += dlen; + len -= dlen; + total_len += dlen; + + if (len) + goto slave_sg_todev_fill_desc; + } + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(sconfig->src_addr_width); + reg = sconfig->src_addr; + ctllo = dw->prepare_ctllo(dwc) + | DWC_CTLL_SRC_WIDTH(reg_width) + | DWC_CTLL_DST_INC + | DWC_CTLL_SRC_FIX; + + ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) : + DWC_CTLL_FC(DW_DMA_FC_D_P2M); + + for_each_sg(sgl, sg, sg_len, i) { + struct dw_desc *desc; + u32 len, mem; + size_t dlen; + + mem = sg_dma_address(sg); + len = sg_dma_len(sg); + +slave_sg_fromdev_fill_desc: + desc = dwc_desc_get(dwc); + if (!desc) + goto err_desc_get; + + ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen); + + lli_write(desc, sar, reg); + lli_write(desc, dar, mem); + lli_write(desc, ctlhi, ctlhi); + mem_width = __ffs(sconfig->dst_addr_width | mem); + lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); + desc->len = dlen; + + if (!first) { + first = desc; + } else { + lli_write(prev, llp, desc->txd.phys | lms); + list_add_tail(&desc->desc_node, &first->tx_list); + } + prev = desc; + + mem += dlen; + len -= dlen; + total_len += dlen; + + if (len) + goto slave_sg_fromdev_fill_desc; + } + break; + default: + return NULL; + } + + if (flags & DMA_PREP_INTERRUPT) + /* Trigger interrupt after last block */ + lli_set(prev, ctllo, DWC_CTLL_INT_EN); + + prev->lli.llp = 0; + lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN); + first->total_len = total_len; + + return &first->txd; + +err_desc_get: + dev_err(chan2dev(chan), + "not enough descriptors available. Direction %d\n", direction); + dwc_desc_put(dwc, first); + return NULL; +} + +bool dw_dma_filter(struct dma_chan *chan, void *param) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma_slave *dws = param; + + if (dws->dma_dev != chan->device->dev) + return false; + + /* permit channels in accordance with the channels mask */ + if (dws->channels && !(dws->channels & dwc->mask)) + return false; + + /* We have to copy data since dws can be temporary storage */ + memcpy(&dwc->dws, dws, sizeof(struct dw_dma_slave)); + + return true; +} +EXPORT_SYMBOL_GPL(dw_dma_filter); + +static int dwc_verify_maxburst(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + + dwc->dma_sconfig.src_maxburst = + clamp(dwc->dma_sconfig.src_maxburst, 1U, dwc->max_burst); + dwc->dma_sconfig.dst_maxburst = + clamp(dwc->dma_sconfig.dst_maxburst, 1U, dwc->max_burst); + + dwc->dma_sconfig.src_maxburst = + rounddown_pow_of_two(dwc->dma_sconfig.src_maxburst); + dwc->dma_sconfig.dst_maxburst = + rounddown_pow_of_two(dwc->dma_sconfig.dst_maxburst); + + return 0; +} + +static int dwc_verify_p_buswidth(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + u32 reg_width, max_width; + + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) + reg_width = dwc->dma_sconfig.dst_addr_width; + else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) + reg_width = dwc->dma_sconfig.src_addr_width; + else /* DMA_MEM_TO_MEM */ + return 0; + + max_width = dw->pdata->data_width[dwc->dws.p_master]; + + /* Fall-back to 1-byte transfer width if undefined */ + if (reg_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + reg_width = DMA_SLAVE_BUSWIDTH_1_BYTE; + else if (!is_power_of_2(reg_width) || reg_width > max_width) + return -EINVAL; + else /* bus width is valid */ + return 0; + + /* Update undefined addr width value */ + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) + dwc->dma_sconfig.dst_addr_width = reg_width; + else /* DMA_DEV_TO_MEM */ + dwc->dma_sconfig.src_addr_width = reg_width; + + return 0; +} + +static int dwc_verify_m_buswidth(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + u32 reg_width, reg_burst, mem_width; + + mem_width = dw->pdata->data_width[dwc->dws.m_master]; + + /* + * It's possible to have a data portion locked in the DMA FIFO in case + * of the channel suspension. Subsequent channel disabling will cause + * that data silent loss. In order to prevent that maintain the src and + * dst transfer widths coherency by means of the relation: + * (CTLx.SRC_TR_WIDTH * CTLx.SRC_MSIZE >= CTLx.DST_TR_WIDTH) + * Look for the details in the commit message that brings this change. + * + * Note the DMA configs utilized in the calculations below must have + * been verified to have correct values by this method call. + */ + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) { + reg_width = dwc->dma_sconfig.dst_addr_width; + if (mem_width < reg_width) + return -EINVAL; + + dwc->dma_sconfig.src_addr_width = mem_width; + } else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) { + reg_width = dwc->dma_sconfig.src_addr_width; + reg_burst = dwc->dma_sconfig.src_maxburst; + + dwc->dma_sconfig.dst_addr_width = min(mem_width, reg_width * reg_burst); + } + + return 0; +} + +static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + int ret; + + memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); + + ret = dwc_verify_maxburst(chan); + if (ret) + return ret; + + ret = dwc_verify_p_buswidth(chan); + if (ret) + return ret; + + ret = dwc_verify_m_buswidth(chan); + if (ret) + return ret; + + return 0; +} + +static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + unsigned int count = 20; /* timeout iterations */ + + dw->suspend_chan(dwc, drain); + + while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--) + udelay(2); + + set_bit(DW_DMA_IS_PAUSED, &dwc->flags); +} + +static int dwc_pause(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + dwc_chan_pause(dwc, false); + spin_unlock_irqrestore(&dwc->lock, flags); + + return 0; +} + +static inline void dwc_chan_resume(struct dw_dma_chan *dwc, bool drain) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + + dw->resume_chan(dwc, drain); + + clear_bit(DW_DMA_IS_PAUSED, &dwc->flags); +} + +static int dwc_resume(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + + if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) + dwc_chan_resume(dwc, false); + + spin_unlock_irqrestore(&dwc->lock, flags); + + return 0; +} + +static int dwc_terminate_all(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + struct dw_desc *desc, *_desc; + unsigned long flags; + LIST_HEAD(list); + + spin_lock_irqsave(&dwc->lock, flags); + + clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); + + dwc_chan_pause(dwc, true); + + dwc_chan_disable(dw, dwc); + + dwc_chan_resume(dwc, true); + + /* active_list entries will end up before queued entries */ + list_splice_init(&dwc->queue, &list); + list_splice_init(&dwc->active_list, &list); + + spin_unlock_irqrestore(&dwc->lock, flags); + + /* Flush all pending and queued descriptors */ + list_for_each_entry_safe(desc, _desc, &list, desc_node) + dwc_descriptor_complete(dwc, desc, false); + + return 0; +} + +static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c) +{ + struct dw_desc *desc; + + list_for_each_entry(desc, &dwc->active_list, desc_node) + if (desc->txd.cookie == c) + return desc; + + return NULL; +} + +static u32 dwc_get_residue_and_status(struct dw_dma_chan *dwc, dma_cookie_t cookie, + enum dma_status *status) +{ + struct dw_desc *desc; + unsigned long flags; + u32 residue; + + spin_lock_irqsave(&dwc->lock, flags); + + desc = dwc_find_desc(dwc, cookie); + if (desc) { + if (desc == dwc_first_active(dwc)) { + residue = desc->residue; + if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue) + residue -= dwc_get_sent(dwc); + if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) + *status = DMA_PAUSED; + } else { + residue = desc->total_len; + } + } else { + residue = 0; + } + + spin_unlock_irqrestore(&dwc->lock, flags); + return residue; +} + +static enum dma_status +dwc_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + dwc_scan_descriptors(to_dw_dma(chan->device), dwc); + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + dma_set_residue(txstate, dwc_get_residue_and_status(dwc, cookie, &ret)); + return ret; +} + +static void dwc_issue_pending(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + if (list_empty(&dwc->active_list)) + dwc_dostart_first_queued(dwc); + spin_unlock_irqrestore(&dwc->lock, flags); +} + +/*----------------------------------------------------------------------*/ + +void do_dw_dma_off(struct dw_dma *dw) +{ + dma_writel(dw, CFG, 0); + + channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask); + channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask); + channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask); + channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask); + channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask); + + while (dma_readl(dw, CFG) & DW_CFG_DMA_EN) + cpu_relax(); +} + +void do_dw_dma_on(struct dw_dma *dw) +{ + dma_writel(dw, CFG, DW_CFG_DMA_EN); +} + +static int dwc_alloc_chan_resources(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + + dev_vdbg(chan2dev(chan), "%s\n", __func__); + + /* ASSERT: channel is idle */ + if (dma_readl(dw, CH_EN) & dwc->mask) { + dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); + return -EIO; + } + + dma_cookie_init(chan); + + /* + * NOTE: some controllers may have additional features that we + * need to initialize here, like "scatter-gather" (which + * doesn't mean what you think it means), and status writeback. + */ + + /* + * We need controller-specific data to set up slave transfers. + */ + if (chan->private && !dw_dma_filter(chan, chan->private)) { + dev_warn(chan2dev(chan), "Wrong controller-specific data\n"); + return -EINVAL; + } + + /* Enable controller here if needed */ + if (!dw->in_use) + do_dw_dma_on(dw); + dw->in_use |= dwc->mask; + + return 0; +} + +static void dwc_free_chan_resources(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + unsigned long flags; + + dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__, + dwc->descs_allocated); + + /* ASSERT: channel is idle */ + BUG_ON(!list_empty(&dwc->active_list)); + BUG_ON(!list_empty(&dwc->queue)); + BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask); + + spin_lock_irqsave(&dwc->lock, flags); + + /* Clear custom channel configuration */ + memset(&dwc->dws, 0, sizeof(struct dw_dma_slave)); + + /* Disable interrupts */ + channel_clear_bit(dw, MASK.XFER, dwc->mask); + channel_clear_bit(dw, MASK.BLOCK, dwc->mask); + channel_clear_bit(dw, MASK.ERROR, dwc->mask); + + spin_unlock_irqrestore(&dwc->lock, flags); + + /* Disable controller in case it was a last user */ + dw->in_use &= ~dwc->mask; + if (!dw->in_use) + do_dw_dma_off(dw); + + dev_vdbg(chan2dev(chan), "%s: done\n", __func__); +} + +static void dwc_caps(struct dma_chan *chan, struct dma_slave_caps *caps) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + + caps->max_burst = dwc->max_burst; + + /* + * It might be crucial for some devices to have the hardware + * accelerated multi-block transfers supported, aka LLPs in DW DMAC + * notation. So if LLPs are supported then max_sg_burst is set to + * zero which means unlimited number of SG entries can be handled in a + * single DMA transaction, otherwise it's just one SG entry. + */ + if (dwc->nollp) + caps->max_sg_burst = 1; + else + caps->max_sg_burst = 0; +} + +int do_dma_probe(struct dw_dma_chip *chip) +{ + struct dw_dma *dw = chip->dw; + struct dw_dma_platform_data *pdata; + bool autocfg = false; + unsigned int dw_params; + unsigned int i; + int ret; + + dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL); + if (!dw->pdata) + return -ENOMEM; + + dw->regs = chip->regs; + + pm_runtime_get_sync(chip->dev); + + if (!chip->pdata) { + dw_params = dma_readl(dw, DW_PARAMS); + dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params); + + autocfg = dw_params >> DW_PARAMS_EN & 1; + if (!autocfg) { + ret = -EINVAL; + goto err_pdata; + } + + /* Reassign the platform data pointer */ + pdata = dw->pdata; + + /* Get hardware configuration parameters */ + pdata->nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 7) + 1; + pdata->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1; + for (i = 0; i < pdata->nr_masters; i++) { + pdata->data_width[i] = + 4 << (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3); + } + pdata->block_size = dma_readl(dw, MAX_BLK_SIZE); + + /* Fill platform data with the default values */ + pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; + pdata->chan_priority = CHAN_PRIORITY_ASCENDING; + } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { + ret = -EINVAL; + goto err_pdata; + } else { + memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata)); + + /* Reassign the platform data pointer */ + pdata = dw->pdata; + } + + dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan), + GFP_KERNEL); + if (!dw->chan) { + ret = -ENOMEM; + goto err_pdata; + } + + /* Calculate all channel mask before DMA setup */ + dw->all_chan_mask = (1 << pdata->nr_channels) - 1; + + /* Force dma off, just in case */ + dw->disable(dw); + + /* Device and instance ID for IRQ and DMA pool */ + dw->set_device_name(dw, chip->id); + + /* Create a pool of consistent memory blocks for hardware descriptors */ + dw->desc_pool = dmam_pool_create(dw->name, chip->dev, + sizeof(struct dw_desc), 4, 0); + if (!dw->desc_pool) { + dev_err(chip->dev, "No memory for descriptors dma pool\n"); + ret = -ENOMEM; + goto err_pdata; + } + + tasklet_setup(&dw->tasklet, dw_dma_tasklet); + + ret = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, + dw->name, dw); + if (ret) + goto err_pdata; + + INIT_LIST_HEAD(&dw->dma.channels); + for (i = 0; i < pdata->nr_channels; i++) { + struct dw_dma_chan *dwc = &dw->chan[i]; + + dwc->chan.device = &dw->dma; + dma_cookie_init(&dwc->chan); + if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING) + list_add_tail(&dwc->chan.device_node, + &dw->dma.channels); + else + list_add(&dwc->chan.device_node, &dw->dma.channels); + + /* 7 is highest priority & 0 is lowest. */ + if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING) + dwc->priority = pdata->nr_channels - i - 1; + else + dwc->priority = i; + + dwc->ch_regs = &__dw_regs(dw)->CHAN[i]; + spin_lock_init(&dwc->lock); + dwc->mask = 1 << i; + + INIT_LIST_HEAD(&dwc->active_list); + INIT_LIST_HEAD(&dwc->queue); + + channel_clear_bit(dw, CH_EN, dwc->mask); + + dwc->direction = DMA_TRANS_NONE; + + /* Hardware configuration */ + if (autocfg) { + unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1; + void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r]; + unsigned int dwc_params = readl(addr); + + dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i, + dwc_params); + + /* + * Decode maximum block size for given channel. The + * stored 4 bit value represents blocks from 0x00 for 3 + * up to 0x0a for 4095. + */ + dwc->block_size = + (4 << ((pdata->block_size >> 4 * i) & 0xf)) - 1; + + /* + * According to the DW DMA databook the true scatter- + * gether LLPs aren't available if either multi-block + * config is disabled (CHx_MULTI_BLK_EN == 0) or the + * LLP register is hard-coded to zeros + * (CHx_HC_LLP == 1). + */ + dwc->nollp = + (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0 || + (dwc_params >> DWC_PARAMS_HC_LLP & 0x1) == 1; + dwc->max_burst = + (0x4 << (dwc_params >> DWC_PARAMS_MSIZE & 0x7)); + } else { + dwc->block_size = pdata->block_size; + dwc->nollp = !pdata->multi_block[i]; + dwc->max_burst = pdata->max_burst[i] ?: DW_DMA_MAX_BURST; + } + } + + /* Clear all interrupts on all channels. */ + dma_writel(dw, CLEAR.XFER, dw->all_chan_mask); + dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask); + dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask); + dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask); + dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask); + + /* Set capabilities */ + dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); + dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask); + dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); + + dw->dma.dev = chip->dev; + dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources; + dw->dma.device_free_chan_resources = dwc_free_chan_resources; + + dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy; + dw->dma.device_prep_slave_sg = dwc_prep_slave_sg; + + dw->dma.device_caps = dwc_caps; + dw->dma.device_config = dwc_config; + dw->dma.device_pause = dwc_pause; + dw->dma.device_resume = dwc_resume; + dw->dma.device_terminate_all = dwc_terminate_all; + + dw->dma.device_tx_status = dwc_tx_status; + dw->dma.device_issue_pending = dwc_issue_pending; + + /* DMA capabilities */ + dw->dma.min_burst = DW_DMA_MIN_BURST; + dw->dma.max_burst = DW_DMA_MAX_BURST; + dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS; + dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS; + dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | + BIT(DMA_MEM_TO_MEM); + dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + /* + * For now there is no hardware with non uniform maximum block size + * across all of the device channels, so we set the maximum segment + * size as the block size found for the very first channel. + */ + dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size); + + ret = dma_async_device_register(&dw->dma); + if (ret) + goto err_dma_register; + + dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", + pdata->nr_channels); + + pm_runtime_put_sync_suspend(chip->dev); + + return 0; + +err_dma_register: + free_irq(chip->irq, dw); +err_pdata: + pm_runtime_put_sync_suspend(chip->dev); + return ret; +} + +int do_dma_remove(struct dw_dma_chip *chip) +{ + struct dw_dma *dw = chip->dw; + struct dw_dma_chan *dwc, *_dwc; + + pm_runtime_get_sync(chip->dev); + + do_dw_dma_off(dw); + dma_async_device_unregister(&dw->dma); + + free_irq(chip->irq, dw); + tasklet_kill(&dw->tasklet); + + list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels, + chan.device_node) { + list_del(&dwc->chan.device_node); + channel_clear_bit(dw, CH_EN, dwc->mask); + } + + pm_runtime_put_sync_suspend(chip->dev); + return 0; +} + +int do_dw_dma_disable(struct dw_dma_chip *chip) +{ + struct dw_dma *dw = chip->dw; + + dw->disable(dw); + return 0; +} +EXPORT_SYMBOL_GPL(do_dw_dma_disable); + +int do_dw_dma_enable(struct dw_dma_chip *chip) +{ + struct dw_dma *dw = chip->dw; + + dw->enable(dw); + return 0; +} +EXPORT_SYMBOL_GPL(do_dw_dma_enable); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver"); +MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); +MODULE_AUTHOR("Viresh Kumar "); diff --git a/drivers/dma/dw/dw.c b/drivers/dma/dw/dw.c new file mode 100644 index 000000000..676614288 --- /dev/null +++ b/drivers/dma/dw/dw.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2007-2008 Atmel Corporation +// Copyright (C) 2010-2011 ST Microelectronics +// Copyright (C) 2013,2018 Intel Corporation + +#include +#include +#include +#include +#include + +#include "internal.h" + +static void dw_dma_initialize_chan(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + u32 cfghi = is_slave_direction(dwc->direction) ? 0 : DWC_CFGH_FIFO_MODE; + u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority); + bool hs_polarity = dwc->dws.hs_polarity; + + cfghi |= DWC_CFGH_DST_PER(dwc->dws.dst_id); + cfghi |= DWC_CFGH_SRC_PER(dwc->dws.src_id); + cfghi |= DWC_CFGH_PROTCTL(dw->pdata->protctl); + + /* Set polarity of handshake interface */ + cfglo |= hs_polarity ? DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL : 0; + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void dw_dma_suspend_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP); +} + +static void dw_dma_resume_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP); +} + +static u32 dw_dma_bytes2block(struct dw_dma_chan *dwc, + size_t bytes, unsigned int width, size_t *len) +{ + u32 block; + + if ((bytes >> width) > dwc->block_size) { + block = dwc->block_size; + *len = dwc->block_size << width; + } else { + block = bytes >> width; + *len = bytes; + } + + return block; +} + +static size_t dw_dma_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) +{ + return DWC_CTLH_BLOCK_TS(block) << width; +} + +static inline u8 dw_dma_encode_maxburst(u32 maxburst) +{ + /* + * Fix burst size according to dw_dmac. We need to convert them as: + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. + */ + return maxburst > 1 ? fls(maxburst) - 2 : 0; +} + +static u32 dw_dma_prepare_ctllo(struct dw_dma_chan *dwc) +{ + struct dma_slave_config *sconfig = &dwc->dma_sconfig; + u8 smsize = 0, dmsize = 0; + u8 sms, dms; + + if (dwc->direction == DMA_MEM_TO_DEV) { + sms = dwc->dws.m_master; + dms = dwc->dws.p_master; + dmsize = dw_dma_encode_maxburst(sconfig->dst_maxburst); + } else if (dwc->direction == DMA_DEV_TO_MEM) { + sms = dwc->dws.p_master; + dms = dwc->dws.m_master; + smsize = dw_dma_encode_maxburst(sconfig->src_maxburst); + } else /* DMA_MEM_TO_MEM */ { + sms = dwc->dws.m_master; + dms = dwc->dws.m_master; + } + + return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | + DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize) | + DWC_CTLL_DMS(dms) | DWC_CTLL_SMS(sms); +} + +static void dw_dma_set_device_name(struct dw_dma *dw, int id) +{ + snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", id); +} + +static void dw_dma_disable(struct dw_dma *dw) +{ + do_dw_dma_off(dw); +} + +static void dw_dma_enable(struct dw_dma *dw) +{ + do_dw_dma_on(dw); +} + +int dw_dma_probe(struct dw_dma_chip *chip) +{ + struct dw_dma *dw; + + dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + /* Channel operations */ + dw->initialize_chan = dw_dma_initialize_chan; + dw->suspend_chan = dw_dma_suspend_chan; + dw->resume_chan = dw_dma_resume_chan; + dw->prepare_ctllo = dw_dma_prepare_ctllo; + dw->bytes2block = dw_dma_bytes2block; + dw->block2bytes = dw_dma_block2bytes; + + /* Device operations */ + dw->set_device_name = dw_dma_set_device_name; + dw->disable = dw_dma_disable; + dw->enable = dw_dma_enable; + + chip->dw = dw; + return do_dma_probe(chip); +} +EXPORT_SYMBOL_GPL(dw_dma_probe); + +int dw_dma_remove(struct dw_dma_chip *chip) +{ + return do_dma_remove(chip); +} +EXPORT_SYMBOL_GPL(dw_dma_remove); diff --git a/drivers/dma/dw/idma32.c b/drivers/dma/dw/idma32.c new file mode 100644 index 000000000..dac617c18 --- /dev/null +++ b/drivers/dma/dw/idma32.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2013,2018,2020-2021 Intel Corporation + +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define DMA_CTL_CH(x) (0x1000 + (x) * 4) +#define DMA_SRC_ADDR_FILLIN(x) (0x1100 + (x) * 4) +#define DMA_DST_ADDR_FILLIN(x) (0x1200 + (x) * 4) +#define DMA_XBAR_SEL(x) (0x1300 + (x) * 4) +#define DMA_REGACCESS_CHID_CFG (0x1400) + +#define CTL_CH_TRANSFER_MODE_MASK GENMASK(1, 0) +#define CTL_CH_TRANSFER_MODE_S2S 0 +#define CTL_CH_TRANSFER_MODE_S2D 1 +#define CTL_CH_TRANSFER_MODE_D2S 2 +#define CTL_CH_TRANSFER_MODE_D2D 3 +#define CTL_CH_RD_RS_MASK GENMASK(4, 3) +#define CTL_CH_WR_RS_MASK GENMASK(6, 5) +#define CTL_CH_RD_NON_SNOOP_BIT BIT(8) +#define CTL_CH_WR_NON_SNOOP_BIT BIT(9) + +#define XBAR_SEL_DEVID_MASK GENMASK(15, 0) +#define XBAR_SEL_RX_TX_BIT BIT(16) +#define XBAR_SEL_RX_TX_SHIFT 16 + +#define REGACCESS_CHID_MASK GENMASK(2, 0) + +static unsigned int idma32_get_slave_devfn(struct dw_dma_chan *dwc) +{ + struct device *slave = dwc->chan.slave; + + if (!slave || !dev_is_pci(slave)) + return 0; + + return to_pci_dev(slave)->devfn; +} + +static void idma32_initialize_chan_xbar(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + void __iomem *misc = __dw_regs(dw); + u32 cfghi = 0, cfglo = 0; + u8 dst_id, src_id; + u32 value; + + /* DMA Channel ID Configuration register must be programmed first */ + value = readl(misc + DMA_REGACCESS_CHID_CFG); + + value &= ~REGACCESS_CHID_MASK; + value |= dwc->chan.chan_id; + + writel(value, misc + DMA_REGACCESS_CHID_CFG); + + /* Configure channel attributes */ + value = readl(misc + DMA_CTL_CH(dwc->chan.chan_id)); + + value &= ~(CTL_CH_RD_NON_SNOOP_BIT | CTL_CH_WR_NON_SNOOP_BIT); + value &= ~(CTL_CH_RD_RS_MASK | CTL_CH_WR_RS_MASK); + value &= ~CTL_CH_TRANSFER_MODE_MASK; + + switch (dwc->direction) { + case DMA_MEM_TO_DEV: + value |= CTL_CH_TRANSFER_MODE_D2S; + value |= CTL_CH_WR_NON_SNOOP_BIT; + break; + case DMA_DEV_TO_MEM: + value |= CTL_CH_TRANSFER_MODE_S2D; + value |= CTL_CH_RD_NON_SNOOP_BIT; + break; + default: + /* + * Memory-to-Memory and Device-to-Device are ignored for now. + * + * For Memory-to-Memory transfers we would need to set mode + * and disable snooping on both sides. + */ + return; + } + + writel(value, misc + DMA_CTL_CH(dwc->chan.chan_id)); + + /* Configure crossbar selection */ + value = readl(misc + DMA_XBAR_SEL(dwc->chan.chan_id)); + + /* DEVFN selection */ + value &= ~XBAR_SEL_DEVID_MASK; + value |= idma32_get_slave_devfn(dwc); + + switch (dwc->direction) { + case DMA_MEM_TO_DEV: + value |= XBAR_SEL_RX_TX_BIT; + break; + case DMA_DEV_TO_MEM: + value &= ~XBAR_SEL_RX_TX_BIT; + break; + default: + /* Memory-to-Memory and Device-to-Device are ignored for now */ + return; + } + + writel(value, misc + DMA_XBAR_SEL(dwc->chan.chan_id)); + + /* Configure DMA channel low and high registers */ + switch (dwc->direction) { + case DMA_MEM_TO_DEV: + dst_id = dwc->chan.chan_id; + src_id = dwc->dws.src_id; + break; + case DMA_DEV_TO_MEM: + dst_id = dwc->dws.dst_id; + src_id = dwc->chan.chan_id; + break; + default: + /* Memory-to-Memory and Device-to-Device are ignored for now */ + return; + } + + /* Set default burst alignment */ + cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN; + + /* Low 4 bits of the request lines */ + cfghi |= IDMA32C_CFGH_DST_PER(dst_id & 0xf); + cfghi |= IDMA32C_CFGH_SRC_PER(src_id & 0xf); + + /* Request line extension (2 bits) */ + cfghi |= IDMA32C_CFGH_DST_PER_EXT(dst_id >> 4 & 0x3); + cfghi |= IDMA32C_CFGH_SRC_PER_EXT(src_id >> 4 & 0x3); + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void idma32_initialize_chan_generic(struct dw_dma_chan *dwc) +{ + u32 cfghi = 0; + u32 cfglo = 0; + + /* Set default burst alignment */ + cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN; + + /* Low 4 bits of the request lines */ + cfghi |= IDMA32C_CFGH_DST_PER(dwc->dws.dst_id & 0xf); + cfghi |= IDMA32C_CFGH_SRC_PER(dwc->dws.src_id & 0xf); + + /* Request line extension (2 bits) */ + cfghi |= IDMA32C_CFGH_DST_PER_EXT(dwc->dws.dst_id >> 4 & 0x3); + cfghi |= IDMA32C_CFGH_SRC_PER_EXT(dwc->dws.src_id >> 4 & 0x3); + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void idma32_suspend_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + if (drain) + cfglo |= IDMA32C_CFGL_CH_DRAIN; + + channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP); +} + +static void idma32_resume_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + if (drain) + cfglo &= ~IDMA32C_CFGL_CH_DRAIN; + + channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP); +} + +static u32 idma32_bytes2block(struct dw_dma_chan *dwc, + size_t bytes, unsigned int width, size_t *len) +{ + u32 block; + + if (bytes > dwc->block_size) { + block = dwc->block_size; + *len = dwc->block_size; + } else { + block = bytes; + *len = bytes; + } + + return block; +} + +static size_t idma32_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) +{ + return IDMA32C_CTLH_BLOCK_TS(block); +} + +static inline u8 idma32_encode_maxburst(u32 maxburst) +{ + return maxburst > 1 ? fls(maxburst) - 1 : 0; +} + +static u32 idma32_prepare_ctllo(struct dw_dma_chan *dwc) +{ + struct dma_slave_config *sconfig = &dwc->dma_sconfig; + u8 smsize = 0, dmsize = 0; + + if (dwc->direction == DMA_MEM_TO_DEV) + dmsize = idma32_encode_maxburst(sconfig->dst_maxburst); + else if (dwc->direction == DMA_DEV_TO_MEM) + smsize = idma32_encode_maxburst(sconfig->src_maxburst); + + return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | + DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize); +} + +static void idma32_set_device_name(struct dw_dma *dw, int id) +{ + snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", id); +} + +/* + * Program FIFO size of channels. + * + * By default full FIFO (512 bytes) is assigned to channel 0. Here we + * slice FIFO on equal parts between channels. + */ +static void idma32_fifo_partition(struct dw_dma *dw) +{ + u64 value = IDMA32C_FP_PSIZE_CH0(64) | IDMA32C_FP_PSIZE_CH1(64) | + IDMA32C_FP_UPDATE; + u64 fifo_partition = 0; + + /* Fill FIFO_PARTITION low bits (Channels 0..1, 4..5) */ + fifo_partition |= value << 0; + + /* Fill FIFO_PARTITION high bits (Channels 2..3, 6..7) */ + fifo_partition |= value << 32; + + /* Program FIFO Partition registers - 64 bytes per channel */ + idma32_writeq(dw, FIFO_PARTITION1, fifo_partition); + idma32_writeq(dw, FIFO_PARTITION0, fifo_partition); +} + +static void idma32_disable(struct dw_dma *dw) +{ + do_dw_dma_off(dw); + idma32_fifo_partition(dw); +} + +static void idma32_enable(struct dw_dma *dw) +{ + idma32_fifo_partition(dw); + do_dw_dma_on(dw); +} + +int idma32_dma_probe(struct dw_dma_chip *chip) +{ + struct dw_dma *dw; + + dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + /* Channel operations */ + if (chip->pdata->quirks & DW_DMA_QUIRK_XBAR_PRESENT) + dw->initialize_chan = idma32_initialize_chan_xbar; + else + dw->initialize_chan = idma32_initialize_chan_generic; + dw->suspend_chan = idma32_suspend_chan; + dw->resume_chan = idma32_resume_chan; + dw->prepare_ctllo = idma32_prepare_ctllo; + dw->bytes2block = idma32_bytes2block; + dw->block2bytes = idma32_block2bytes; + + /* Device operations */ + dw->set_device_name = idma32_set_device_name; + dw->disable = idma32_disable; + dw->enable = idma32_enable; + + chip->dw = dw; + return do_dma_probe(chip); +} +EXPORT_SYMBOL_GPL(idma32_dma_probe); + +int idma32_dma_remove(struct dw_dma_chip *chip) +{ + return do_dma_remove(chip); +} +EXPORT_SYMBOL_GPL(idma32_dma_remove); diff --git a/drivers/dma/dw/internal.h b/drivers/dma/dw/internal.h new file mode 100644 index 000000000..f1bd06a20 --- /dev/null +++ b/drivers/dma/dw/internal.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Driver for the Synopsys DesignWare DMA Controller + * + * Copyright (C) 2013 Intel Corporation + */ + +#ifndef _DMA_DW_INTERNAL_H +#define _DMA_DW_INTERNAL_H + +#include + +#include "regs.h" + +int do_dma_probe(struct dw_dma_chip *chip); +int do_dma_remove(struct dw_dma_chip *chip); + +void do_dw_dma_on(struct dw_dma *dw); +void do_dw_dma_off(struct dw_dma *dw); + +int do_dw_dma_disable(struct dw_dma_chip *chip); +int do_dw_dma_enable(struct dw_dma_chip *chip); + +extern bool dw_dma_filter(struct dma_chan *chan, void *param); + +#ifdef CONFIG_ACPI +void dw_dma_acpi_controller_register(struct dw_dma *dw); +void dw_dma_acpi_controller_free(struct dw_dma *dw); +#else /* !CONFIG_ACPI */ +static inline void dw_dma_acpi_controller_register(struct dw_dma *dw) {} +static inline void dw_dma_acpi_controller_free(struct dw_dma *dw) {} +#endif /* !CONFIG_ACPI */ + +struct platform_device; + +#ifdef CONFIG_OF +struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev); +void dw_dma_of_controller_register(struct dw_dma *dw); +void dw_dma_of_controller_free(struct dw_dma *dw); +#else +static inline struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev) +{ + return NULL; +} +static inline void dw_dma_of_controller_register(struct dw_dma *dw) {} +static inline void dw_dma_of_controller_free(struct dw_dma *dw) {} +#endif + +struct dw_dma_chip_pdata { + const struct dw_dma_platform_data *pdata; + int (*probe)(struct dw_dma_chip *chip); + int (*remove)(struct dw_dma_chip *chip); + struct dw_dma_chip *chip; + u8 m_master; + u8 p_master; +}; + +static __maybe_unused const struct dw_dma_chip_pdata dw_dma_chip_pdata = { + .probe = dw_dma_probe, + .remove = dw_dma_remove, + .m_master = 0, + .p_master = 1, +}; + +static const struct dw_dma_platform_data idma32_pdata = { + .nr_channels = 8, + .chan_allocation_order = CHAN_ALLOCATION_ASCENDING, + .chan_priority = CHAN_PRIORITY_ASCENDING, + .block_size = 131071, + .nr_masters = 1, + .data_width = {4}, + .multi_block = {1, 1, 1, 1, 1, 1, 1, 1}, +}; + +static __maybe_unused const struct dw_dma_chip_pdata idma32_chip_pdata = { + .pdata = &idma32_pdata, + .probe = idma32_dma_probe, + .remove = idma32_dma_remove, + .m_master = 0, + .p_master = 0, +}; + +static const struct dw_dma_platform_data xbar_pdata = { + .nr_channels = 8, + .chan_allocation_order = CHAN_ALLOCATION_ASCENDING, + .chan_priority = CHAN_PRIORITY_ASCENDING, + .block_size = 131071, + .nr_masters = 1, + .data_width = {4}, + .quirks = DW_DMA_QUIRK_XBAR_PRESENT, +}; + +static __maybe_unused const struct dw_dma_chip_pdata xbar_chip_pdata = { + .pdata = &xbar_pdata, + .probe = idma32_dma_probe, + .remove = idma32_dma_remove, + .m_master = 0, + .p_master = 0, +}; + +#endif /* _DMA_DW_INTERNAL_H */ diff --git a/drivers/dma/dw/of.c b/drivers/dma/dw/of.c new file mode 100644 index 000000000..523ca8068 --- /dev/null +++ b/drivers/dma/dw/of.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Platform driver for the Synopsys DesignWare DMA Controller + * + * Copyright (C) 2007-2008 Atmel Corporation + * Copyright (C) 2010-2011 ST Microelectronics + * Copyright (C) 2013 Intel Corporation + */ + +#include +#include +#include + +#include "internal.h" + +static struct dma_chan *dw_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct dw_dma *dw = ofdma->of_dma_data; + struct dw_dma_slave slave = { + .dma_dev = dw->dma.dev, + }; + dma_cap_mask_t cap; + + if (dma_spec->args_count < 3 || dma_spec->args_count > 4) + return NULL; + + slave.src_id = dma_spec->args[0]; + slave.dst_id = dma_spec->args[0]; + slave.m_master = dma_spec->args[1]; + slave.p_master = dma_spec->args[2]; + if (dma_spec->args_count >= 4) + slave.channels = dma_spec->args[3]; + + if (WARN_ON(slave.src_id >= DW_DMA_MAX_NR_REQUESTS || + slave.dst_id >= DW_DMA_MAX_NR_REQUESTS || + slave.m_master >= dw->pdata->nr_masters || + slave.p_master >= dw->pdata->nr_masters || + slave.channels >= BIT(dw->pdata->nr_channels))) + return NULL; + + dma_cap_zero(cap); + dma_cap_set(DMA_SLAVE, cap); + + /* TODO: there should be a simpler way to do this */ + return dma_request_channel(cap, dw_dma_filter, &slave); +} + +struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct dw_dma_platform_data *pdata; + u32 tmp, arr[DW_DMA_MAX_NR_MASTERS]; + u32 nr_masters; + u32 nr_channels; + + if (of_property_read_u32(np, "dma-masters", &nr_masters)) + return NULL; + if (nr_masters < 1 || nr_masters > DW_DMA_MAX_NR_MASTERS) + return NULL; + + if (of_property_read_u32(np, "dma-channels", &nr_channels)) + return NULL; + if (nr_channels > DW_DMA_MAX_NR_CHANNELS) + return NULL; + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return NULL; + + pdata->nr_masters = nr_masters; + pdata->nr_channels = nr_channels; + + of_property_read_u32(np, "chan_allocation_order", &pdata->chan_allocation_order); + of_property_read_u32(np, "chan_priority", &pdata->chan_priority); + + of_property_read_u32(np, "block_size", &pdata->block_size); + + /* Try deprecated property first */ + if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) { + for (tmp = 0; tmp < nr_masters; tmp++) + pdata->data_width[tmp] = BIT(arr[tmp] & 0x07); + } + + /* If "data_width" and "data-width" both provided use the latter one */ + of_property_read_u32_array(np, "data-width", pdata->data_width, nr_masters); + + memset32(pdata->multi_block, 1, nr_channels); + of_property_read_u32_array(np, "multi-block", pdata->multi_block, nr_channels); + + memset32(pdata->max_burst, DW_DMA_MAX_BURST, nr_channels); + of_property_read_u32_array(np, "snps,max-burst-len", pdata->max_burst, nr_channels); + + of_property_read_u32(np, "snps,dma-protection-control", &pdata->protctl); + if (pdata->protctl > CHAN_PROTCTL_MASK) + return NULL; + + return pdata; +} + +void dw_dma_of_controller_register(struct dw_dma *dw) +{ + struct device *dev = dw->dma.dev; + int ret; + + if (!dev->of_node) + return; + + ret = of_dma_controller_register(dev->of_node, dw_dma_of_xlate, dw); + if (ret) + dev_err(dev, "could not register of_dma_controller\n"); +} + +void dw_dma_of_controller_free(struct dw_dma *dw) +{ + struct device *dev = dw->dma.dev; + + if (!dev->of_node) + return; + + of_dma_controller_free(dev->of_node); +} diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c new file mode 100644 index 000000000..a3aae3d1c --- /dev/null +++ b/drivers/dma/dw/pci.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI driver for the Synopsys DesignWare DMA Controller + * + * Copyright (C) 2013 Intel Corporation + * Author: Andy Shevchenko + */ + +#include +#include +#include + +#include "internal.h" + +static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) +{ + const struct dw_dma_chip_pdata *drv_data = (void *)pid->driver_data; + struct dw_dma_chip_pdata *data; + struct dw_dma_chip *chip; + int ret; + + ret = pcim_enable_device(pdev); + if (ret) + return ret; + + ret = pcim_iomap_regions(pdev, 1 << 0, pci_name(pdev)); + if (ret) { + dev_err(&pdev->dev, "I/O memory remapping failed\n"); + return ret; + } + + pci_set_master(pdev); + pci_try_set_mwi(pdev); + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + data = devm_kmemdup(&pdev->dev, drv_data, sizeof(*drv_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + chip->dev = &pdev->dev; + chip->id = pdev->devfn; + chip->regs = pcim_iomap_table(pdev)[0]; + chip->irq = pdev->irq; + chip->pdata = data->pdata; + + data->chip = chip; + + ret = data->probe(chip); + if (ret) + return ret; + + pci_set_drvdata(pdev, data); + + dw_dma_acpi_controller_register(chip->dw); + + return 0; +} + +static void dw_pci_remove(struct pci_dev *pdev) +{ + struct dw_dma_chip_pdata *data = pci_get_drvdata(pdev); + struct dw_dma_chip *chip = data->chip; + int ret; + + dw_dma_acpi_controller_free(chip->dw); + + ret = data->remove(chip); + if (ret) + dev_warn(&pdev->dev, "can't remove device properly: %d\n", ret); +} + +static int dw_pci_suspend_late(struct device *dev) +{ + struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); + struct dw_dma_chip *chip = data->chip; + + return do_dw_dma_disable(chip); +}; + +static int dw_pci_resume_early(struct device *dev) +{ + struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); + struct dw_dma_chip *chip = data->chip; + + return do_dw_dma_enable(chip); +}; + +static const struct dev_pm_ops dw_pci_dev_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) +}; + +static const struct pci_device_id dw_pci_id_table[] = { + /* Medfield (GPDMA) */ + { PCI_VDEVICE(INTEL, 0x0827), (kernel_ulong_t)&dw_dma_chip_pdata }, + + /* BayTrail */ + { PCI_VDEVICE(INTEL, 0x0f06), (kernel_ulong_t)&dw_dma_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x0f40), (kernel_ulong_t)&dw_dma_chip_pdata }, + + /* Merrifield */ + { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&idma32_chip_pdata }, + + /* Braswell */ + { PCI_VDEVICE(INTEL, 0x2286), (kernel_ulong_t)&dw_dma_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_dma_chip_pdata }, + + /* Elkhart Lake iDMA 32-bit (PSE DMA) */ + { PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&xbar_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&xbar_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&xbar_chip_pdata }, + + /* Haswell */ + { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_dma_chip_pdata }, + + /* Broadwell */ + { PCI_VDEVICE(INTEL, 0x9ce0), (kernel_ulong_t)&dw_dma_chip_pdata }, + + { } +}; +MODULE_DEVICE_TABLE(pci, dw_pci_id_table); + +static struct pci_driver dw_pci_driver = { + .name = "dw_dmac_pci", + .id_table = dw_pci_id_table, + .probe = dw_pci_probe, + .remove = dw_pci_remove, + .driver = { + .pm = pm_sleep_ptr(&dw_pci_dev_pm_ops), + }, +}; + +module_pci_driver(dw_pci_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller PCI driver"); +MODULE_AUTHOR("Andy Shevchenko "); diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c new file mode 100644 index 000000000..c63fa5203 --- /dev/null +++ b/drivers/dma/dw/platform.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Platform driver for the Synopsys DesignWare DMA Controller + * + * Copyright (C) 2007-2008 Atmel Corporation + * Copyright (C) 2010-2011 ST Microelectronics + * Copyright (C) 2013 Intel Corporation + * + * Some parts of this driver are derived from the original dw_dmac. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static int dw_probe(struct platform_device *pdev) +{ + const struct dw_dma_chip_pdata *match; + struct dw_dma_chip_pdata *data; + struct dw_dma_chip *chip; + struct device *dev = &pdev->dev; + int ret; + + match = device_get_match_data(dev); + if (!match) + return -ENODEV; + + data = devm_kmemdup(&pdev->dev, match, sizeof(*match), GFP_KERNEL); + if (!data) + return -ENOMEM; + + chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + chip->irq = platform_get_irq(pdev, 0); + if (chip->irq < 0) + return chip->irq; + + chip->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(chip->regs)) + return PTR_ERR(chip->regs); + + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + if (!data->pdata) + data->pdata = dev_get_platdata(dev); + if (!data->pdata) + data->pdata = dw_dma_parse_dt(pdev); + + chip->dev = dev; + chip->id = pdev->id; + chip->pdata = data->pdata; + + data->chip = chip; + + chip->clk = devm_clk_get_optional(chip->dev, "hclk"); + if (IS_ERR(chip->clk)) + return PTR_ERR(chip->clk); + ret = clk_prepare_enable(chip->clk); + if (ret) + return ret; + + pm_runtime_enable(&pdev->dev); + + ret = data->probe(chip); + if (ret) + goto err_dw_dma_probe; + + platform_set_drvdata(pdev, data); + + dw_dma_of_controller_register(chip->dw); + + dw_dma_acpi_controller_register(chip->dw); + + return 0; + +err_dw_dma_probe: + pm_runtime_disable(&pdev->dev); + clk_disable_unprepare(chip->clk); + return ret; +} + +static void dw_remove(struct platform_device *pdev) +{ + struct dw_dma_chip_pdata *data = platform_get_drvdata(pdev); + struct dw_dma_chip *chip = data->chip; + int ret; + + dw_dma_acpi_controller_free(chip->dw); + + dw_dma_of_controller_free(chip->dw); + + ret = data->remove(chip); + if (ret) + dev_warn(chip->dev, "can't remove device properly: %d\n", ret); + + pm_runtime_disable(&pdev->dev); + clk_disable_unprepare(chip->clk); +} + +static void dw_shutdown(struct platform_device *pdev) +{ + struct dw_dma_chip_pdata *data = platform_get_drvdata(pdev); + struct dw_dma_chip *chip = data->chip; + + /* + * We have to call do_dw_dma_disable() to stop any ongoing transfer. On + * some platforms we can't do that since DMA device is powered off. + * Moreover we have no possibility to check if the platform is affected + * or not. That's why we call pm_runtime_get_sync() / pm_runtime_put() + * unconditionally. On the other hand we can't use + * pm_runtime_suspended() because runtime PM framework is not fully + * used by the driver. + */ + pm_runtime_get_sync(chip->dev); + do_dw_dma_disable(chip); + pm_runtime_put_sync_suspend(chip->dev); + + clk_disable_unprepare(chip->clk); +} + +#ifdef CONFIG_OF +static const struct of_device_id dw_dma_of_id_table[] = { + { .compatible = "snps,dma-spear1340", .data = &dw_dma_chip_pdata }, + { .compatible = "renesas,rzn1-dma", .data = &dw_dma_chip_pdata }, + {} +}; +MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); +#endif + +#ifdef CONFIG_ACPI +static const struct acpi_device_id dw_dma_acpi_id_table[] = { + { "INTL9C60", (kernel_ulong_t)&dw_dma_chip_pdata }, + { "80862286", (kernel_ulong_t)&dw_dma_chip_pdata }, + { "808622C0", (kernel_ulong_t)&dw_dma_chip_pdata }, + + /* Elkhart Lake iDMA 32-bit (PSE DMA) */ + { "80864BB4", (kernel_ulong_t)&xbar_chip_pdata }, + { "80864BB5", (kernel_ulong_t)&xbar_chip_pdata }, + { "80864BB6", (kernel_ulong_t)&xbar_chip_pdata }, + + { } +}; +MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table); +#endif + +static int dw_suspend_late(struct device *dev) +{ + struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); + struct dw_dma_chip *chip = data->chip; + + do_dw_dma_disable(chip); + clk_disable_unprepare(chip->clk); + + return 0; +} + +static int dw_resume_early(struct device *dev) +{ + struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); + struct dw_dma_chip *chip = data->chip; + int ret; + + ret = clk_prepare_enable(chip->clk); + if (ret) + return ret; + + return do_dw_dma_enable(chip); +} + +static const struct dev_pm_ops dw_dev_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) +}; + +static struct platform_driver dw_driver = { + .probe = dw_probe, + .remove = dw_remove, + .shutdown = dw_shutdown, + .driver = { + .name = "dw_dmac", + .pm = pm_sleep_ptr(&dw_dev_pm_ops), + .of_match_table = of_match_ptr(dw_dma_of_id_table), + .acpi_match_table = ACPI_PTR(dw_dma_acpi_id_table), + }, +}; + +static int __init dw_init(void) +{ + return platform_driver_register(&dw_driver); +} +subsys_initcall(dw_init); + +static void __exit dw_exit(void) +{ + platform_driver_unregister(&dw_driver); +} +module_exit(dw_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller platform driver"); diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h new file mode 100644 index 000000000..5969d9cc8 --- /dev/null +++ b/drivers/dma/dw/regs.h @@ -0,0 +1,408 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Driver for the Synopsys DesignWare AHB DMA Controller + * + * Copyright (C) 2005-2007 Atmel Corporation + * Copyright (C) 2010-2011 ST Microelectronics + * Copyright (C) 2016 Intel Corporation + */ + +#include +#include +#include + +#include + +#include "internal.h" + +#define DW_DMA_MAX_NR_REQUESTS 16 + +/* flow controller */ +enum dw_dma_fc { + DW_DMA_FC_D_M2M, + DW_DMA_FC_D_M2P, + DW_DMA_FC_D_P2M, + DW_DMA_FC_D_P2P, + DW_DMA_FC_P_P2M, + DW_DMA_FC_SP_P2P, + DW_DMA_FC_P_M2P, + DW_DMA_FC_DP_P2P, +}; + +/* + * Redefine this macro to handle differences between 32- and 64-bit + * addressing, big vs. little endian, etc. + */ +#define DW_REG(name) u32 name; u32 __pad_##name + +/* Hardware register definitions. */ +struct dw_dma_chan_regs { + DW_REG(SAR); /* Source Address Register */ + DW_REG(DAR); /* Destination Address Register */ + DW_REG(LLP); /* Linked List Pointer */ + u32 CTL_LO; /* Control Register Low */ + u32 CTL_HI; /* Control Register High */ + DW_REG(SSTAT); + DW_REG(DSTAT); + DW_REG(SSTATAR); + DW_REG(DSTATAR); + u32 CFG_LO; /* Configuration Register Low */ + u32 CFG_HI; /* Configuration Register High */ + DW_REG(SGR); + DW_REG(DSR); +}; + +struct dw_dma_irq_regs { + DW_REG(XFER); + DW_REG(BLOCK); + DW_REG(SRC_TRAN); + DW_REG(DST_TRAN); + DW_REG(ERROR); +}; + +struct dw_dma_regs { + /* per-channel registers */ + struct dw_dma_chan_regs CHAN[DW_DMA_MAX_NR_CHANNELS]; + + /* irq handling */ + struct dw_dma_irq_regs RAW; /* r */ + struct dw_dma_irq_regs STATUS; /* r (raw & mask) */ + struct dw_dma_irq_regs MASK; /* rw (set = irq enabled) */ + struct dw_dma_irq_regs CLEAR; /* w (ack, affects "raw") */ + + DW_REG(STATUS_INT); /* r */ + + /* software handshaking */ + DW_REG(REQ_SRC); + DW_REG(REQ_DST); + DW_REG(SGL_REQ_SRC); + DW_REG(SGL_REQ_DST); + DW_REG(LAST_SRC); + DW_REG(LAST_DST); + + /* miscellaneous */ + DW_REG(CFG); + DW_REG(CH_EN); + DW_REG(ID); + DW_REG(TEST); + + /* iDMA 32-bit support */ + DW_REG(CLASS_PRIORITY0); + DW_REG(CLASS_PRIORITY1); + + /* optional encoded params, 0x3c8..0x3f7 */ + u32 __reserved; + + /* per-channel configuration registers */ + u32 DWC_PARAMS[DW_DMA_MAX_NR_CHANNELS]; + u32 MULTI_BLK_TYPE; + u32 MAX_BLK_SIZE; + + /* top-level parameters */ + u32 DW_PARAMS; + + /* component ID */ + u32 COMP_TYPE; + u32 COMP_VERSION; + + /* iDMA 32-bit support */ + DW_REG(FIFO_PARTITION0); + DW_REG(FIFO_PARTITION1); + + DW_REG(SAI_ERR); + DW_REG(GLOBAL_CFG); +}; + +/* Bitfields in DW_PARAMS */ +#define DW_PARAMS_NR_CHAN 8 /* number of channels */ +#define DW_PARAMS_NR_MASTER 11 /* number of AHB masters */ +#define DW_PARAMS_DATA_WIDTH(n) (15 + 2 * (n)) +#define DW_PARAMS_DATA_WIDTH1 15 /* master 1 data width */ +#define DW_PARAMS_DATA_WIDTH2 17 /* master 2 data width */ +#define DW_PARAMS_DATA_WIDTH3 19 /* master 3 data width */ +#define DW_PARAMS_DATA_WIDTH4 21 /* master 4 data width */ +#define DW_PARAMS_EN 28 /* encoded parameters */ + +/* Bitfields in DWC_PARAMS */ +#define DWC_PARAMS_MBLK_EN 11 /* multi block transfer */ +#define DWC_PARAMS_HC_LLP 13 /* set LLP register to zero */ +#define DWC_PARAMS_MSIZE 16 /* max group transaction size */ + +/* bursts size */ +enum dw_dma_msize { + DW_DMA_MSIZE_1, + DW_DMA_MSIZE_4, + DW_DMA_MSIZE_8, + DW_DMA_MSIZE_16, + DW_DMA_MSIZE_32, + DW_DMA_MSIZE_64, + DW_DMA_MSIZE_128, + DW_DMA_MSIZE_256, +}; + +/* Bitfields in LLP */ +#define DWC_LLP_LMS(x) ((x) & 3) /* list master select */ +#define DWC_LLP_LOC(x) ((x) & ~3) /* next lli */ + +/* Bitfields in CTL_LO */ +#define DWC_CTLL_INT_EN (1 << 0) /* irqs enabled? */ +#define DWC_CTLL_DST_WIDTH(n) ((n)<<1) /* bytes per element */ +#define DWC_CTLL_SRC_WIDTH(n) ((n)<<4) +#define DWC_CTLL_DST_INC (0<<7) /* DAR update/not */ +#define DWC_CTLL_DST_DEC (1<<7) +#define DWC_CTLL_DST_FIX (2<<7) +#define DWC_CTLL_SRC_INC (0<<9) /* SAR update/not */ +#define DWC_CTLL_SRC_DEC (1<<9) +#define DWC_CTLL_SRC_FIX (2<<9) +#define DWC_CTLL_DST_MSIZE(n) ((n)<<11) /* burst, #elements */ +#define DWC_CTLL_SRC_MSIZE(n) ((n)<<14) +#define DWC_CTLL_S_GATH_EN (1 << 17) /* src gather, !FIX */ +#define DWC_CTLL_D_SCAT_EN (1 << 18) /* dst scatter, !FIX */ +#define DWC_CTLL_FC(n) ((n) << 20) +#define DWC_CTLL_FC_M2M (0 << 20) /* mem-to-mem */ +#define DWC_CTLL_FC_M2P (1 << 20) /* mem-to-periph */ +#define DWC_CTLL_FC_P2M (2 << 20) /* periph-to-mem */ +#define DWC_CTLL_FC_P2P (3 << 20) /* periph-to-periph */ +/* plus 4 transfer types for peripheral-as-flow-controller */ +#define DWC_CTLL_DMS(n) ((n)<<23) /* dst master select */ +#define DWC_CTLL_SMS(n) ((n)<<25) /* src master select */ +#define DWC_CTLL_LLP_D_EN (1 << 27) /* dest block chain */ +#define DWC_CTLL_LLP_S_EN (1 << 28) /* src block chain */ + +/* Bitfields in CTL_HI */ +#define DWC_CTLH_BLOCK_TS_MASK GENMASK(11, 0) +#define DWC_CTLH_BLOCK_TS(x) ((x) & DWC_CTLH_BLOCK_TS_MASK) +#define DWC_CTLH_DONE (1 << 12) + +/* Bitfields in CFG_LO */ +#define DWC_CFGL_CH_PRIOR_MASK (0x7 << 5) /* priority mask */ +#define DWC_CFGL_CH_PRIOR(x) ((x) << 5) /* priority */ +#define DWC_CFGL_CH_SUSP (1 << 8) /* pause xfer */ +#define DWC_CFGL_FIFO_EMPTY (1 << 9) /* pause xfer */ +#define DWC_CFGL_HS_DST (1 << 10) /* handshake w/dst */ +#define DWC_CFGL_HS_SRC (1 << 11) /* handshake w/src */ +#define DWC_CFGL_LOCK_CH_XFER (0 << 12) /* scope of LOCK_CH */ +#define DWC_CFGL_LOCK_CH_BLOCK (1 << 12) +#define DWC_CFGL_LOCK_CH_XACT (2 << 12) +#define DWC_CFGL_LOCK_BUS_XFER (0 << 14) /* scope of LOCK_BUS */ +#define DWC_CFGL_LOCK_BUS_BLOCK (1 << 14) +#define DWC_CFGL_LOCK_BUS_XACT (2 << 14) +#define DWC_CFGL_LOCK_CH (1 << 15) /* channel lockout */ +#define DWC_CFGL_LOCK_BUS (1 << 16) /* busmaster lockout */ +#define DWC_CFGL_HS_DST_POL (1 << 18) /* dst handshake active low */ +#define DWC_CFGL_HS_SRC_POL (1 << 19) /* src handshake active low */ +#define DWC_CFGL_MAX_BURST(x) ((x) << 20) +#define DWC_CFGL_RELOAD_SAR (1 << 30) +#define DWC_CFGL_RELOAD_DAR (1 << 31) + +/* Bitfields in CFG_HI */ +#define DWC_CFGH_FCMODE (1 << 0) +#define DWC_CFGH_FIFO_MODE (1 << 1) +#define DWC_CFGH_PROTCTL(x) ((x) << 2) +#define DWC_CFGH_PROTCTL_DATA (0 << 2) /* data access - always set */ +#define DWC_CFGH_PROTCTL_PRIV (1 << 2) /* privileged -> AHB HPROT[1] */ +#define DWC_CFGH_PROTCTL_BUFFER (2 << 2) /* bufferable -> AHB HPROT[2] */ +#define DWC_CFGH_PROTCTL_CACHE (4 << 2) /* cacheable -> AHB HPROT[3] */ +#define DWC_CFGH_DS_UPD_EN (1 << 5) +#define DWC_CFGH_SS_UPD_EN (1 << 6) +#define DWC_CFGH_SRC_PER(x) ((x) << 7) +#define DWC_CFGH_DST_PER(x) ((x) << 11) + +/* Bitfields in SGR */ +#define DWC_SGR_SGI(x) ((x) << 0) +#define DWC_SGR_SGC(x) ((x) << 20) + +/* Bitfields in DSR */ +#define DWC_DSR_DSI(x) ((x) << 0) +#define DWC_DSR_DSC(x) ((x) << 20) + +/* Bitfields in CFG */ +#define DW_CFG_DMA_EN (1 << 0) + +/* iDMA 32-bit support */ + +/* bursts size */ +enum idma32_msize { + IDMA32_MSIZE_1, + IDMA32_MSIZE_2, + IDMA32_MSIZE_4, + IDMA32_MSIZE_8, + IDMA32_MSIZE_16, + IDMA32_MSIZE_32, +}; + +/* Bitfields in CTL_HI */ +#define IDMA32C_CTLH_BLOCK_TS_MASK GENMASK(16, 0) +#define IDMA32C_CTLH_BLOCK_TS(x) ((x) & IDMA32C_CTLH_BLOCK_TS_MASK) +#define IDMA32C_CTLH_DONE (1 << 17) + +/* Bitfields in CFG_LO */ +#define IDMA32C_CFGL_DST_BURST_ALIGN (1 << 0) /* dst burst align */ +#define IDMA32C_CFGL_SRC_BURST_ALIGN (1 << 1) /* src burst align */ +#define IDMA32C_CFGL_CH_DRAIN (1 << 10) /* drain FIFO */ +#define IDMA32C_CFGL_DST_OPT_BL (1 << 20) /* optimize dst burst length */ +#define IDMA32C_CFGL_SRC_OPT_BL (1 << 21) /* optimize src burst length */ + +/* Bitfields in CFG_HI */ +#define IDMA32C_CFGH_SRC_PER(x) ((x) << 0) +#define IDMA32C_CFGH_DST_PER(x) ((x) << 4) +#define IDMA32C_CFGH_RD_ISSUE_THD(x) ((x) << 8) +#define IDMA32C_CFGH_RW_ISSUE_THD(x) ((x) << 18) +#define IDMA32C_CFGH_SRC_PER_EXT(x) ((x) << 28) /* src peripheral extension */ +#define IDMA32C_CFGH_DST_PER_EXT(x) ((x) << 30) /* dst peripheral extension */ + +/* Bitfields in FIFO_PARTITION */ +#define IDMA32C_FP_PSIZE_CH0(x) ((x) << 0) +#define IDMA32C_FP_PSIZE_CH1(x) ((x) << 13) +#define IDMA32C_FP_UPDATE (1 << 26) + +enum dw_dmac_flags { + DW_DMA_IS_CYCLIC = 0, + DW_DMA_IS_SOFT_LLP = 1, + DW_DMA_IS_PAUSED = 2, + DW_DMA_IS_INITIALIZED = 3, +}; + +struct dw_dma_chan { + struct dma_chan chan; + void __iomem *ch_regs; + u8 mask; + u8 priority; + enum dma_transfer_direction direction; + + /* software emulation of the LLP transfers */ + struct list_head *tx_node_active; + + spinlock_t lock; + + /* these other elements are all protected by lock */ + unsigned long flags; + struct list_head active_list; + struct list_head queue; + + unsigned int descs_allocated; + + /* hardware configuration */ + unsigned int block_size; + bool nollp; + u32 max_burst; + + /* custom slave configuration */ + struct dw_dma_slave dws; + + /* configuration passed via .device_config */ + struct dma_slave_config dma_sconfig; +}; + +static inline struct dw_dma_chan_regs __iomem * +__dwc_regs(struct dw_dma_chan *dwc) +{ + return dwc->ch_regs; +} + +#define channel_readl(dwc, name) \ + readl(&(__dwc_regs(dwc)->name)) +#define channel_writel(dwc, name, val) \ + writel((val), &(__dwc_regs(dwc)->name)) + +static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct dw_dma_chan, chan); +} + +struct dw_dma { + struct dma_device dma; + char name[20]; + void __iomem *regs; + struct dma_pool *desc_pool; + struct tasklet_struct tasklet; + + /* channels */ + struct dw_dma_chan *chan; + u8 all_chan_mask; + u8 in_use; + + /* Channel operations */ + void (*initialize_chan)(struct dw_dma_chan *dwc); + void (*suspend_chan)(struct dw_dma_chan *dwc, bool drain); + void (*resume_chan)(struct dw_dma_chan *dwc, bool drain); + u32 (*prepare_ctllo)(struct dw_dma_chan *dwc); + u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes, + unsigned int width, size_t *len); + size_t (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width); + + /* Device operations */ + void (*set_device_name)(struct dw_dma *dw, int id); + void (*disable)(struct dw_dma *dw); + void (*enable)(struct dw_dma *dw); + + /* platform data */ + struct dw_dma_platform_data *pdata; +}; + +static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw) +{ + return dw->regs; +} + +#define dma_readl(dw, name) \ + readl(&(__dw_regs(dw)->name)) +#define dma_writel(dw, name, val) \ + writel((val), &(__dw_regs(dw)->name)) + +#define idma32_readq(dw, name) \ + hi_lo_readq(&(__dw_regs(dw)->name)) +#define idma32_writeq(dw, name, val) \ + hi_lo_writeq((val), &(__dw_regs(dw)->name)) + +#define channel_set_bit(dw, reg, mask) \ + dma_writel(dw, reg, ((mask) << 8) | (mask)) +#define channel_clear_bit(dw, reg, mask) \ + dma_writel(dw, reg, ((mask) << 8) | 0) + +static inline struct dw_dma *to_dw_dma(struct dma_device *ddev) +{ + return container_of(ddev, struct dw_dma, dma); +} + +/* LLI == Linked List Item; a.k.a. DMA block descriptor */ +struct dw_lli { + /* values that are not changed by hardware */ + __le32 sar; + __le32 dar; + __le32 llp; /* chain to next lli */ + __le32 ctllo; + /* values that may get written back: */ + __le32 ctlhi; + /* sstat and dstat can snapshot peripheral register state. + * silicon config may discard either or both... + */ + __le32 sstat; + __le32 dstat; +}; + +struct dw_desc { + /* FIRST values the hardware uses */ + struct dw_lli lli; + +#define lli_set(d, reg, v) ((d)->lli.reg |= cpu_to_le32(v)) +#define lli_clear(d, reg, v) ((d)->lli.reg &= ~cpu_to_le32(v)) +#define lli_read(d, reg) le32_to_cpu((d)->lli.reg) +#define lli_write(d, reg, v) ((d)->lli.reg = cpu_to_le32(v)) + + /* THEN values for driver housekeeping */ + struct list_head desc_node; + struct list_head tx_list; + struct dma_async_tx_descriptor txd; + size_t len; + size_t total_len; + u32 residue; +}; + +#define to_dw_desc(h) list_entry(h, struct dw_desc, desc_node) + +static inline struct dw_desc * +txd_to_dw_desc(struct dma_async_tx_descriptor *txd) +{ + return container_of(txd, struct dw_desc, txd); +} diff --git a/drivers/dma/dw/rzn1-dmamux.c b/drivers/dma/dw/rzn1-dmamux.c new file mode 100644 index 000000000..fb495ee6e --- /dev/null +++ b/drivers/dma/dw/rzn1-dmamux.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Schneider-Electric + * Author: Miquel Raynal + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNZ1_DMAMUX_NCELLS 6 +#define RZN1_DMAMUX_MAX_LINES 64 +#define RZN1_DMAMUX_LINES_PER_CTLR 16 + +struct rzn1_dmamux_data { + struct dma_router dmarouter; + DECLARE_BITMAP(used_chans, 2 * RZN1_DMAMUX_LINES_PER_CTLR); +}; + +struct rzn1_dmamux_map { + unsigned int req_idx; +}; + +static void rzn1_dmamux_free(struct device *dev, void *route_data) +{ + struct rzn1_dmamux_data *dmamux = dev_get_drvdata(dev); + struct rzn1_dmamux_map *map = route_data; + + dev_dbg(dev, "Unmapping DMAMUX request %u\n", map->req_idx); + + clear_bit(map->req_idx, dmamux->used_chans); + + kfree(map); +} + +static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct rzn1_dmamux_data *dmamux = platform_get_drvdata(pdev); + struct rzn1_dmamux_map *map; + unsigned int dmac_idx, chan, val; + u32 mask; + int ret; + + if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS) { + ret = -EINVAL; + goto put_device; + } + + map = kzalloc_obj(*map); + if (!map) { + ret = -ENOMEM; + goto put_device; + } + + chan = dma_spec->args[0]; + map->req_idx = dma_spec->args[4]; + val = dma_spec->args[5]; + dma_spec->args_count -= 2; + + if (chan >= RZN1_DMAMUX_LINES_PER_CTLR) { + dev_err(&pdev->dev, "Invalid DMA request line: %u\n", chan); + ret = -EINVAL; + goto free_map; + } + + if (map->req_idx >= RZN1_DMAMUX_MAX_LINES || + (map->req_idx % RZN1_DMAMUX_LINES_PER_CTLR) != chan) { + dev_err(&pdev->dev, "Invalid MUX request line: %u\n", map->req_idx); + ret = -EINVAL; + goto free_map; + } + + dmac_idx = map->req_idx >= RZN1_DMAMUX_LINES_PER_CTLR ? 1 : 0; + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", dmac_idx); + if (!dma_spec->np) { + dev_err(&pdev->dev, "Can't get DMA master\n"); + ret = -EINVAL; + goto free_map; + } + + dev_dbg(&pdev->dev, "Mapping DMAMUX request %u to DMAC%u request %u\n", + map->req_idx, dmac_idx, chan); + + if (test_and_set_bit(map->req_idx, dmamux->used_chans)) { + ret = -EBUSY; + goto put_dma_spec_np; + } + + mask = BIT(map->req_idx); + ret = r9a06g032_sysctrl_set_dmamux(mask, val ? mask : 0); + if (ret) + goto clear_bitmap; + + put_device(&pdev->dev); + return map; + +clear_bitmap: + clear_bit(map->req_idx, dmamux->used_chans); +put_dma_spec_np: + of_node_put(dma_spec->np); +free_map: + kfree(map); +put_device: + put_device(&pdev->dev); + + return ERR_PTR(ret); +} + +#ifdef CONFIG_OF +static const struct of_device_id rzn1_dmac_match[] = { + { .compatible = "renesas,rzn1-dma" }, + {} +}; +#endif + +static int rzn1_dmamux_probe(struct platform_device *pdev) +{ + struct device_node *mux_node = pdev->dev.of_node; + const struct of_device_id *match; + struct device_node *dmac_node; + struct rzn1_dmamux_data *dmamux; + + dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL); + if (!dmamux) + return -ENOMEM; + + dmac_node = of_parse_phandle(mux_node, "dma-masters", 0); + if (!dmac_node) + return dev_err_probe(&pdev->dev, -ENODEV, "Can't get DMA master node\n"); + + match = of_match_node(rzn1_dmac_match, dmac_node); + of_node_put(dmac_node); + if (!match) + return dev_err_probe(&pdev->dev, -EINVAL, "DMA master is not supported\n"); + + dmamux->dmarouter.dev = &pdev->dev; + dmamux->dmarouter.route_free = rzn1_dmamux_free; + + platform_set_drvdata(pdev, dmamux); + + return of_dma_router_register(mux_node, rzn1_dmamux_route_allocate, + &dmamux->dmarouter); +} + +static const struct of_device_id rzn1_dmamux_match[] = { + { .compatible = "renesas,rzn1-dmamux" }, + {} +}; +MODULE_DEVICE_TABLE(of, rzn1_dmamux_match); + +static struct platform_driver rzn1_dmamux_driver = { + .driver = { + .name = "renesas,rzn1-dmamux", + .of_match_table = rzn1_dmamux_match, + }, + .probe = rzn1_dmamux_probe, +}; +module_platform_driver(rzn1_dmamux_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Miquel Raynal