selftests/bpf: Reject graph kptr use after RCU unlockgrafted

Add a sleepable verifier test that loads a graph-node local kptr in an
explicit RCU read-side critical section, then passes its node to
bpf_rbtree_remove() after the section ends.

Before the verifier fix, the stale NON_OWN_REF flag makes the node look like
a live borrowed reference and the program is accepted. After the fix, the
pointer is demoted without NON_OWN_REF and the graph kfunc argument is
rejected.

Also exercise a graph kptr loaded while a spin lock provides implicit RCU
protection. The pointer must be invalidated when the lock is released, which
guards the required ordering between non-owning-reference invalidation and
RCU demotion.

Update the existing fault-protected load test state description. The
post-unlock pointer no longer carries NON_OWN_REF, but remains readable
because the load is rewritten to use BPF_PROBE_MEM.

Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20260904084325.52250-7-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

5 files changed, 2121 insertions, 0 deletions

diff --git a/drivers/dma/loongson/Kconfig b/drivers/dma/loongson/Kconfig
new file mode 100644
index 000000000..c4e62dce5
--- /dev/null
+++ b/drivers/dma/loongson/Kconfig
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Loongson DMA controllers drivers
+#
+if MACH_LOONGSON32 || MACH_LOONGSON64 || COMPILE_TEST
+
+config LOONGSON1_APB_DMA
+	tristate "Loongson1 APB DMA support"
+	depends on MACH_LOONGSON32 || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  This selects support for the APB DMA controller in Loongson1 SoCs,
+	  which is required by Loongson1 NAND and audio support.
+
+config LOONGSON2_APB_DMA
+	tristate "Loongson2 APB DMA support"
+	depends on MACH_LOONGSON64 || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  Support for the Loongson2 APB DMA controller driver. The
+	  DMA controller is having single DMA channel which can be
+	  configured for different peripherals like audio, nand, sdio
+	  etc which is in APB bus.
+
+	  This DMA controller transfers data from memory to peripheral fifo.
+	  It does not support memory to memory data transfer.
+
+config LOONGSON2_APB_CMC_DMA
+	tristate "Loongson2 Chain Multi-Channel DMA support"
+	depends on MACH_LOONGSON64 || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  Support for the Loongson Chain Multi-Channel DMA controller driver.
+	  It is discovered on the Loongson-2K chip (Loongson-2K0300/Loongson-2K3000),
+	  which has 4/8 channels internally, enabling bidirectional data transfer
+	  between devices and memory.
+
+endif
diff --git a/drivers/dma/loongson/Makefile b/drivers/dma/loongson/Makefile
new file mode 100644
index 000000000..48c19781e
--- /dev/null
+++ b/drivers/dma/loongson/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o
+obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o
+obj-$(CONFIG_LOONGSON2_APB_CMC_DMA) += loongson2-apb-cmc-dma.o
diff --git a/drivers/dma/loongson/loongson1-apb-dma.c b/drivers/dma/loongson/loongson1-apb-dma.c
new file mode 100644
index 000000000..89786cbd2
--- /dev/null
+++ b/drivers/dma/loongson/loongson1-apb-dma.c
@@ -0,0 +1,660 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Loongson-1 APB DMA Controller
+ *
+ * Copyright (C) 2015-2024 Keguang Zhang <keguang.zhang@gmail.com>
+ */
+
+#include <linux/dmapool.h>
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+/* Loongson-1 DMA Control Register */
+#define LS1X_DMA_CTRL		0x0
+
+/* DMA Control Register Bits */
+#define LS1X_DMA_STOP		BIT(4)
+#define LS1X_DMA_START		BIT(3)
+#define LS1X_DMA_ASK_VALID	BIT(2)
+
+/* DMA Next Field Bits */
+#define LS1X_DMA_NEXT_VALID	BIT(0)
+
+/* DMA Command Field Bits */
+#define LS1X_DMA_RAM2DEV	BIT(12)
+#define LS1X_DMA_INT		BIT(1)
+#define LS1X_DMA_INT_MASK	BIT(0)
+
+#define LS1X_DMA_LLI_ALIGNMENT	64
+#define LS1X_DMA_LLI_ADDR_MASK	GENMASK(31, __ffs(LS1X_DMA_LLI_ALIGNMENT))
+#define LS1X_DMA_MAX_CHANNELS	3
+
+enum ls1x_dmadesc_offsets {
+	LS1X_DMADESC_NEXT = 0,
+	LS1X_DMADESC_SADDR,
+	LS1X_DMADESC_DADDR,
+	LS1X_DMADESC_LENGTH,
+	LS1X_DMADESC_STRIDE,
+	LS1X_DMADESC_CYCLES,
+	LS1X_DMADESC_CMD,
+	LS1X_DMADESC_SIZE
+};
+
+struct ls1x_dma_lli {
+	unsigned int hw[LS1X_DMADESC_SIZE];
+	dma_addr_t phys;
+	struct list_head node;
+} __aligned(LS1X_DMA_LLI_ALIGNMENT);
+
+struct ls1x_dma_desc {
+	struct virt_dma_desc vd;
+	struct list_head lli_list;
+};
+
+struct ls1x_dma_chan {
+	struct virt_dma_chan vc;
+	struct dma_pool *lli_pool;
+	phys_addr_t src_addr;
+	phys_addr_t dst_addr;
+	enum dma_slave_buswidth src_addr_width;
+	enum dma_slave_buswidth dst_addr_width;
+	unsigned int bus_width;
+	void __iomem *reg_base;
+	int irq;
+	bool is_cyclic;
+	struct ls1x_dma_lli *curr_lli;
+};
+
+struct ls1x_dma {
+	struct dma_device ddev;
+	unsigned int nr_chans;
+	struct ls1x_dma_chan chan[];
+};
+
+static irqreturn_t ls1x_dma_irq_handler(int irq, void *data);
+
+#define to_ls1x_dma_chan(dchan)		\
+	container_of(dchan, struct ls1x_dma_chan, vc.chan)
+
+#define to_ls1x_dma_desc(d)		\
+	container_of(d, struct ls1x_dma_desc, vd)
+
+static inline struct device *chan2dev(struct dma_chan *chan)
+{
+	return &chan->dev->device;
+}
+
+static inline int ls1x_dma_query(struct ls1x_dma_chan *chan,
+				 dma_addr_t *lli_phys)
+{
+	struct dma_chan *dchan = &chan->vc.chan;
+	int val, ret;
+
+	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
+	val |= LS1X_DMA_ASK_VALID;
+	val |= dchan->chan_id;
+	writel(val, chan->reg_base + LS1X_DMA_CTRL);
+	ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val,
+					!(val & LS1X_DMA_ASK_VALID), 0, 3000);
+	if (ret)
+		dev_err(chan2dev(dchan), "failed to query DMA\n");
+
+	return ret;
+}
+
+static inline int ls1x_dma_start(struct ls1x_dma_chan *chan,
+				 dma_addr_t *lli_phys)
+{
+	struct dma_chan *dchan = &chan->vc.chan;
+	struct device *dev = chan2dev(dchan);
+	int val, ret;
+
+	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
+	val |= LS1X_DMA_START;
+	val |= dchan->chan_id;
+	writel(val, chan->reg_base + LS1X_DMA_CTRL);
+	ret = readl_poll_timeout(chan->reg_base + LS1X_DMA_CTRL, val,
+				 !(val & LS1X_DMA_START), 0, 1000);
+	if (!ret)
+		dev_dbg(dev, "start DMA with lli_phys=%pad\n", lli_phys);
+	else
+		dev_err(dev, "failed to start DMA\n");
+
+	return ret;
+}
+
+static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan)
+{
+	int val = readl(chan->reg_base + LS1X_DMA_CTRL);
+
+	writel(val | LS1X_DMA_STOP, chan->reg_base + LS1X_DMA_CTRL);
+}
+
+static void ls1x_dma_free_chan_resources(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct device *dev = chan2dev(dchan);
+
+	dma_free_coherent(dev, sizeof(struct ls1x_dma_lli),
+			  chan->curr_lli, chan->curr_lli->phys);
+	dma_pool_destroy(chan->lli_pool);
+	chan->lli_pool = NULL;
+	devm_free_irq(dev, chan->irq, chan);
+	vchan_free_chan_resources(&chan->vc);
+}
+
+static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct device *dev = chan2dev(dchan);
+	dma_addr_t phys;
+	int ret;
+
+	ret = devm_request_irq(dev, chan->irq, ls1x_dma_irq_handler,
+			       IRQF_SHARED, dma_chan_name(dchan), chan);
+	if (ret) {
+		dev_err(dev, "failed to request IRQ %d\n", chan->irq);
+		return ret;
+	}
+
+	chan->lli_pool = dma_pool_create(dma_chan_name(dchan), dev,
+					 sizeof(struct ls1x_dma_lli),
+					 __alignof__(struct ls1x_dma_lli), 0);
+	if (!chan->lli_pool)
+		return -ENOMEM;
+
+	/* allocate memory for querying the current lli */
+	dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
+	chan->curr_lli = dma_alloc_coherent(dev, sizeof(struct ls1x_dma_lli),
+					    &phys, GFP_KERNEL);
+	if (!chan->curr_lli) {
+		dma_pool_destroy(chan->lli_pool);
+		return -ENOMEM;
+	}
+	chan->curr_lli->phys = phys;
+
+	return 0;
+}
+
+static void ls1x_dma_free_desc(struct virt_dma_desc *vd)
+{
+	struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(vd->tx.chan);
+	struct ls1x_dma_lli *lli, *_lli;
+
+	list_for_each_entry_safe(lli, _lli, &desc->lli_list, node) {
+		list_del(&lli->node);
+		dma_pool_free(chan->lli_pool, lli, lli->phys);
+	}
+
+	kfree(desc);
+}
+
+static struct ls1x_dma_desc *ls1x_dma_alloc_desc(void)
+{
+	struct ls1x_dma_desc *desc;
+
+	desc = kzalloc_obj(*desc, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	INIT_LIST_HEAD(&desc->lli_list);
+
+	return desc;
+}
+
+static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc,
+			     struct scatterlist *sgl, unsigned int sg_len,
+			     enum dma_transfer_direction dir, bool is_cyclic)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL;
+	struct device *dev = chan2dev(dchan);
+	struct list_head *pos = NULL;
+	struct scatterlist *sg;
+	unsigned int dev_addr, cmd, i;
+
+	switch (dir) {
+	case DMA_MEM_TO_DEV:
+		dev_addr = chan->dst_addr;
+		chan->bus_width = chan->dst_addr_width;
+		cmd = LS1X_DMA_RAM2DEV | LS1X_DMA_INT;
+		break;
+	case DMA_DEV_TO_MEM:
+		dev_addr = chan->src_addr;
+		chan->bus_width = chan->src_addr_width;
+		cmd = LS1X_DMA_INT;
+		break;
+	default:
+		dev_err(dev, "unsupported DMA direction: %s\n",
+			dmaengine_get_direction_text(dir));
+		return -EINVAL;
+	}
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		dma_addr_t buf_addr = sg_dma_address(sg);
+		size_t buf_len = sg_dma_len(sg);
+		dma_addr_t phys;
+
+		if (!is_dma_copy_aligned(dchan->device, buf_addr, 0, buf_len)) {
+			dev_err(dev, "buffer is not aligned\n");
+			return -EINVAL;
+		}
+
+		/* allocate HW descriptors */
+		lli = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, &phys);
+		if (!lli) {
+			dev_err(dev, "failed to alloc lli %u\n", i);
+			return -ENOMEM;
+		}
+
+		/* setup HW descriptors */
+		lli->phys = phys;
+		lli->hw[LS1X_DMADESC_SADDR] = buf_addr;
+		lli->hw[LS1X_DMADESC_DADDR] = dev_addr;
+		lli->hw[LS1X_DMADESC_LENGTH] = buf_len / chan->bus_width;
+		lli->hw[LS1X_DMADESC_STRIDE] = 0;
+		lli->hw[LS1X_DMADESC_CYCLES] = 1;
+		lli->hw[LS1X_DMADESC_CMD] = cmd;
+
+		if (prev)
+			prev->hw[LS1X_DMADESC_NEXT] =
+			    lli->phys | LS1X_DMA_NEXT_VALID;
+		prev = lli;
+
+		if (!first)
+			first = lli;
+
+		list_add_tail(&lli->node, &desc->lli_list);
+	}
+
+	if (is_cyclic) {
+		lli->hw[LS1X_DMADESC_NEXT] = first->phys | LS1X_DMA_NEXT_VALID;
+		chan->is_cyclic = is_cyclic;
+	}
+
+	list_for_each(pos, &desc->lli_list) {
+		lli = list_entry(pos, struct ls1x_dma_lli, node);
+		print_hex_dump_debug("LLI: ", DUMP_PREFIX_OFFSET, 16, 4,
+				     lli, sizeof(*lli), false);
+	}
+
+	return 0;
+}
+
+static struct dma_async_tx_descriptor *
+ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+		       unsigned int sg_len, enum dma_transfer_direction dir,
+		       unsigned long flags, void *context)
+{
+	struct ls1x_dma_desc *desc;
+
+	dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n",
+		sg_len, flags, dmaengine_get_direction_text(dir));
+
+	desc = ls1x_dma_alloc_desc();
+	if (!desc)
+		return NULL;
+
+	if (ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, false)) {
+		ls1x_dma_free_desc(&desc->vd);
+		return NULL;
+	}
+
+	return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *
+ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr,
+			 size_t buf_len, size_t period_len,
+			 enum dma_transfer_direction dir, unsigned long flags)
+{
+	struct ls1x_dma_desc *desc;
+	struct scatterlist *sgl;
+	unsigned int sg_len;
+	unsigned int i;
+	int ret;
+
+	dev_dbg(chan2dev(dchan),
+		"buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n",
+		buf_len, period_len, flags, dmaengine_get_direction_text(dir));
+
+	desc = ls1x_dma_alloc_desc();
+	if (!desc)
+		return NULL;
+
+	/* allocate the scatterlist */
+	sg_len = buf_len / period_len;
+	sgl = kmalloc_objs(*sgl, sg_len, GFP_NOWAIT);
+	if (!sgl)
+		return NULL;
+
+	sg_init_table(sgl, sg_len);
+	for (i = 0; i < sg_len; ++i) {
+		sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(buf_addr)),
+			    period_len, offset_in_page(buf_addr));
+		sg_dma_address(&sgl[i]) = buf_addr;
+		sg_dma_len(&sgl[i]) = period_len;
+		buf_addr += period_len;
+	}
+
+	ret = ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, true);
+	kfree(sgl);
+	if (ret) {
+		ls1x_dma_free_desc(&desc->vd);
+		return NULL;
+	}
+
+	return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags);
+}
+
+static int ls1x_dma_slave_config(struct dma_chan *dchan,
+				 struct dma_slave_config *config)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+
+	chan->src_addr = config->src_addr;
+	chan->src_addr_width = config->src_addr_width;
+	chan->dst_addr = config->dst_addr;
+	chan->dst_addr_width = config->dst_addr_width;
+
+	return 0;
+}
+
+static int ls1x_dma_pause(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	int ret;
+
+	guard(spinlock_irqsave)(&chan->vc.lock);
+	/* save the current lli */
+	ret = ls1x_dma_query(chan, &chan->curr_lli->phys);
+	if (!ret)
+		ls1x_dma_stop(chan);
+
+	return ret;
+}
+
+static int ls1x_dma_resume(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+
+	guard(spinlock_irqsave)(&chan->vc.lock);
+
+	return ls1x_dma_start(chan, &chan->curr_lli->phys);
+}
+
+static int ls1x_dma_terminate_all(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct virt_dma_desc *vd;
+	LIST_HEAD(head);
+
+	ls1x_dma_stop(chan);
+
+	scoped_guard(spinlock_irqsave, &chan->vc.lock) {
+		vd = vchan_next_desc(&chan->vc);
+		if (vd)
+			vchan_terminate_vdesc(vd);
+
+		vchan_get_all_descriptors(&chan->vc, &head);
+	}
+
+	vchan_dma_desc_free_list(&chan->vc, &head);
+
+	return 0;
+}
+
+static void ls1x_dma_synchronize(struct dma_chan *dchan)
+{
+	vchan_synchronize(to_virt_chan(dchan));
+}
+
+static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan,
+					  dma_cookie_t cookie,
+					  struct dma_tx_state *state)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct virt_dma_desc *vd;
+	enum dma_status status;
+	size_t bytes = 0;
+
+	status = dma_cookie_status(dchan, cookie, state);
+	if (status == DMA_COMPLETE)
+		return status;
+
+	scoped_guard(spinlock_irqsave, &chan->vc.lock) {
+		vd = vchan_find_desc(&chan->vc, cookie);
+		if (vd) {
+			struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
+			struct ls1x_dma_lli *lli;
+			dma_addr_t next_phys;
+
+			/* get the current lli */
+			if (ls1x_dma_query(chan, &chan->curr_lli->phys))
+				return status;
+
+			/* locate the current lli */
+			next_phys = chan->curr_lli->hw[LS1X_DMADESC_NEXT];
+			list_for_each_entry(lli, &desc->lli_list, node)
+				if (lli->hw[LS1X_DMADESC_NEXT] == next_phys)
+					break;
+
+			dev_dbg(chan2dev(dchan), "current lli_phys=%pad",
+				&lli->phys);
+
+			/* count the residues */
+			list_for_each_entry_from(lli, &desc->lli_list, node)
+				bytes += lli->hw[LS1X_DMADESC_LENGTH] *
+					 chan->bus_width;
+		}
+	}
+
+	dma_set_residue(state, bytes);
+
+	return status;
+}
+
+static void ls1x_dma_issue_pending(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+
+	guard(spinlock_irqsave)(&chan->vc.lock);
+
+	if (vchan_issue_pending(&chan->vc)) {
+		struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
+
+		if (vd) {
+			struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
+			struct ls1x_dma_lli *lli;
+
+			lli = list_first_entry(&desc->lli_list,
+					       struct ls1x_dma_lli, node);
+			ls1x_dma_start(chan, &lli->phys);
+		}
+	}
+}
+
+static irqreturn_t ls1x_dma_irq_handler(int irq, void *data)
+{
+	struct ls1x_dma_chan *chan = data;
+	struct dma_chan *dchan = &chan->vc.chan;
+	struct device *dev = chan2dev(dchan);
+	struct virt_dma_desc *vd;
+
+	scoped_guard(spinlock, &chan->vc.lock) {
+		vd = vchan_next_desc(&chan->vc);
+		if (!vd) {
+			dev_warn(dev,
+				 "IRQ %d with no active desc on channel %d\n",
+				 irq, dchan->chan_id);
+			return IRQ_NONE;
+		}
+
+		if (chan->is_cyclic) {
+			vchan_cyclic_callback(vd);
+		} else {
+			list_del(&vd->node);
+			vchan_cookie_complete(vd);
+		}
+	}
+
+	dev_dbg(dev, "DMA IRQ %d on channel %d\n", irq, dchan->chan_id);
+
+	return IRQ_HANDLED;
+}
+
+static int ls1x_dma_chan_probe(struct platform_device *pdev,
+			       struct ls1x_dma *dma)
+{
+	void __iomem *reg_base;
+	int id;
+
+	reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(reg_base))
+		return PTR_ERR(reg_base);
+
+	for (id = 0; id < dma->nr_chans; id++) {
+		struct ls1x_dma_chan *chan = &dma->chan[id];
+		char pdev_irqname[16];
+
+		snprintf(pdev_irqname, sizeof(pdev_irqname), "ch%d", id);
+		chan->irq = platform_get_irq_byname(pdev, pdev_irqname);
+		if (chan->irq < 0)
+			return dev_err_probe(&pdev->dev, chan->irq,
+					     "failed to get IRQ for ch%d\n",
+					     id);
+
+		chan->reg_base = reg_base;
+		chan->vc.desc_free = ls1x_dma_free_desc;
+		vchan_init(&chan->vc, &dma->ddev);
+	}
+
+	return 0;
+}
+
+static void ls1x_dma_chan_remove(struct ls1x_dma *dma)
+{
+	int id;
+
+	for (id = 0; id < dma->nr_chans; id++) {
+		struct ls1x_dma_chan *chan = &dma->chan[id];
+
+		if (chan->vc.chan.device == &dma->ddev) {
+			list_del(&chan->vc.chan.device_node);
+			tasklet_kill(&chan->vc.task);
+		}
+	}
+}
+
+static int ls1x_dma_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dma_device *ddev;
+	struct ls1x_dma *dma;
+	int ret;
+
+	ret = platform_irq_count(pdev);
+	if (ret <= 0 || ret > LS1X_DMA_MAX_CHANNELS)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid number of IRQ channels: %d\n",
+				     ret);
+
+	dma = devm_kzalloc(dev, struct_size(dma, chan, ret), GFP_KERNEL);
+	if (!dma)
+		return -ENOMEM;
+	dma->nr_chans = ret;
+
+	/* initialize DMA device */
+	ddev = &dma->ddev;
+	ddev->dev = dev;
+	ddev->copy_align = DMAENGINE_ALIGN_4_BYTES;
+	ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
+				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
+				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	ddev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
+				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
+				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+	ddev->device_alloc_chan_resources = ls1x_dma_alloc_chan_resources;
+	ddev->device_free_chan_resources = ls1x_dma_free_chan_resources;
+	ddev->device_prep_slave_sg = ls1x_dma_prep_slave_sg;
+	ddev->device_prep_dma_cyclic = ls1x_dma_prep_dma_cyclic;
+	ddev->device_config = ls1x_dma_slave_config;
+	ddev->device_pause = ls1x_dma_pause;
+	ddev->device_resume = ls1x_dma_resume;
+	ddev->device_terminate_all = ls1x_dma_terminate_all;
+	ddev->device_synchronize = ls1x_dma_synchronize;
+	ddev->device_tx_status = ls1x_dma_tx_status;
+	ddev->device_issue_pending = ls1x_dma_issue_pending;
+	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
+	INIT_LIST_HEAD(&ddev->channels);
+
+	/* initialize DMA channels */
+	ret = ls1x_dma_chan_probe(pdev, dma);
+	if (ret)
+		goto err;
+
+	ret = dmaenginem_async_device_register(ddev);
+	if (ret) {
+		dev_err(dev, "failed to register DMA device\n");
+		goto err;
+	}
+
+	ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id,
+					 ddev);
+	if (ret) {
+		dev_err(dev, "failed to register DMA controller\n");
+		goto err;
+	}
+
+	platform_set_drvdata(pdev, dma);
+	dev_info(dev, "Loongson1 DMA driver registered\n");
+
+	return 0;
+
+err:
+	ls1x_dma_chan_remove(dma);
+
+	return ret;
+}
+
+static void ls1x_dma_remove(struct platform_device *pdev)
+{
+	struct ls1x_dma *dma = platform_get_drvdata(pdev);
+
+	of_dma_controller_free(pdev->dev.of_node);
+	ls1x_dma_chan_remove(dma);
+}
+
+static const struct of_device_id ls1x_dma_match[] = {
+	{ .compatible = "loongson,ls1b-apbdma" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ls1x_dma_match);
+
+static struct platform_driver ls1x_dma_driver = {
+	.probe = ls1x_dma_probe,
+	.remove = ls1x_dma_remove,
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = ls1x_dma_match,
+	},
+};
+
+module_platform_driver(ls1x_dma_driver);
+
+MODULE_AUTHOR("Keguang Zhang <keguang.zhang@gmail.com>");
+MODULE_DESCRIPTION("Loongson-1 APB DMA Controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/loongson/loongson2-apb-cmc-dma.c b/drivers/dma/loongson/loongson2-apb-cmc-dma.c
new file mode 100644
index 000000000..1c9a542ed
--- /dev/null
+++ b/drivers/dma/loongson/loongson2-apb-cmc-dma.c
@@ -0,0 +1,730 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Loongson-2 Chain Multi-Channel DMA Controller driver
+ *
+ * Copyright (C) 2024-2026 Loongson Technology Corporation Limited
+ */
+
+#include <linux/acpi.h>
+#include <linux/acpi_dma.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+#define LOONGSON2_CMCDMA_ISR		0x0	/* DMA Interrupt Status Register */
+#define LOONGSON2_CMCDMA_IFCR		0x4	/* DMA Interrupt Flag Clear Register */
+#define LOONGSON2_CMCDMA_CCR		0x8	/* DMA Channel Configuration Register */
+#define LOONGSON2_CMCDMA_CNDTR		0xc	/* DMA Channel Transmit Count Register */
+#define LOONGSON2_CMCDMA_CPAR		0x10	/* DMA Channel Peripheral Address Register */
+#define LOONGSON2_CMCDMA_CMAR		0x14	/* DMA Channel Memory Address Register */
+
+/* Bitfields of DMA interrupt status register */
+#define LOONGSON2_CMCDMA_TCI		BIT(1) /* Transfer Complete Interrupt */
+#define LOONGSON2_CMCDMA_HTI		BIT(2) /* Half Transfer Interrupt */
+#define LOONGSON2_CMCDMA_TEI		BIT(3) /* Transfer Error Interrupt */
+
+#define LOONGSON2_CMCDMA_MASKI		\
+	(LOONGSON2_CMCDMA_TCI | LOONGSON2_CMCDMA_HTI | LOONGSON2_CMCDMA_TEI)
+
+/* Bitfields of DMA channel x Configuration Register */
+#define LOONGSON2_CMCDMA_CCR_EN		BIT(0) /* Stream Enable */
+#define LOONGSON2_CMCDMA_CCR_TCIE	BIT(1) /* Transfer Complete Interrupt Enable */
+#define LOONGSON2_CMCDMA_CCR_HTIE	BIT(2) /* Half Transfer Complete Interrupt Enable */
+#define LOONGSON2_CMCDMA_CCR_TEIE	BIT(3) /* Transfer Error Interrupt Enable */
+#define LOONGSON2_CMCDMA_CCR_DIR	BIT(4) /* Data Transfer Direction */
+#define LOONGSON2_CMCDMA_CCR_CIRC	BIT(5) /* Circular mode */
+#define LOONGSON2_CMCDMA_CCR_PINC	BIT(6) /* Peripheral increment mode */
+#define LOONGSON2_CMCDMA_CCR_MINC	BIT(7) /* Memory increment mode */
+#define LOONGSON2_CMCDMA_CCR_PSIZE_MASK	GENMASK(9, 8)
+#define LOONGSON2_CMCDMA_CCR_MSIZE_MASK	GENMASK(11, 10)
+#define LOONGSON2_CMCDMA_CCR_PL_MASK	GENMASK(13, 12)
+#define LOONGSON2_CMCDMA_CCR_M2M	BIT(14)
+
+#define LOONGSON2_CMCDMA_CCR_CFG_MASK	\
+	(LOONGSON2_CMCDMA_CCR_PINC | LOONGSON2_CMCDMA_CCR_MINC | LOONGSON2_CMCDMA_CCR_PL_MASK)
+
+#define LOONGSON2_CMCDMA_CCR_IRQ_MASK	\
+	(LOONGSON2_CMCDMA_CCR_TCIE | LOONGSON2_CMCDMA_CCR_HTIE | LOONGSON2_CMCDMA_CCR_TEIE)
+
+#define LOONGSON2_CMCDMA_STREAM_MASK	\
+	(LOONGSON2_CMCDMA_CCR_CFG_MASK | LOONGSON2_CMCDMA_CCR_IRQ_MASK)
+
+#define LOONGSON2_CMCDMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+					 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+					 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
+
+#define LOONSON2_CMCDMA_MAX_DATA_ITEMS	SZ_64K
+
+struct loongson2_cmc_dma_chan_reg {
+	u32 ccr;
+	u32 cndtr;
+	u32 cpar;
+	u32 cmar;
+};
+
+struct loongson2_cmc_dma_sg_req {
+	u32 len;
+	struct loongson2_cmc_dma_chan_reg chan_reg;
+};
+
+struct loongson2_cmc_dma_desc {
+	struct virt_dma_desc vdesc;
+	bool cyclic;
+	u32 num_sgs;
+	struct loongson2_cmc_dma_sg_req sg_req[] __counted_by(num_sgs);
+};
+
+struct loongson2_cmc_dma_chan {
+	struct virt_dma_chan vchan;
+	struct dma_slave_config	dma_sconfig;
+	struct loongson2_cmc_dma_desc *desc;
+	u32 id;
+	u32 irq;
+	u32 next_sg;
+	struct loongson2_cmc_dma_chan_reg chan_reg;
+};
+
+struct loongson2_cmc_dma_dev {
+	struct dma_device ddev;
+	struct clk *dma_clk;
+	void __iomem *base;
+	u32 nr_channels;
+	u32 chan_reg_offset;
+	struct loongson2_cmc_dma_chan chan[] __counted_by(nr_channels);
+};
+
+struct loongson2_cmc_dma_config {
+	u32 max_channels;
+	u32 chan_reg_offset;
+};
+
+static const struct loongson2_cmc_dma_config ls2k0300_cmc_dma_config = {
+	.max_channels = 8,
+	.chan_reg_offset = 0x14,
+};
+
+static const struct loongson2_cmc_dma_config ls2k3000_cmc_dma_config = {
+	.max_channels = 4,
+	.chan_reg_offset = 0x18,
+};
+
+static struct loongson2_cmc_dma_dev *lmdma_get_dev(struct loongson2_cmc_dma_chan *lchan)
+{
+	return container_of(lchan->vchan.chan.device, struct loongson2_cmc_dma_dev, ddev);
+}
+
+static struct loongson2_cmc_dma_chan *to_lmdma_chan(struct dma_chan *chan)
+{
+	return container_of(chan, struct loongson2_cmc_dma_chan, vchan.chan);
+}
+
+static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc)
+{
+	return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc);
+}
+
+static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan)
+{
+	return &lchan->vchan.chan.dev->device;
+}
+
+static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id)
+{
+	return readl(lddev->base + (reg + lddev->chan_reg_offset * id));
+}
+
+static void loongson2_cmc_dma_write(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id, u32 val)
+{
+	writel(val, lddev->base + (reg + lddev->chan_reg_offset * id));
+}
+
+static int loongson2_cmc_dma_get_width(enum dma_slave_buswidth width)
+{
+	switch (width) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+		return ffs(width) - 1;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int loongson2_cmc_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *config)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	memcpy(&lchan->dma_sconfig, config, sizeof(*config));
+
+	return 0;
+}
+
+static void loongson2_cmc_dma_irq_clear(struct loongson2_cmc_dma_chan *lchan, u32 flags)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	u32 ifcr;
+
+	ifcr = flags << (4 * lchan->id);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_IFCR, 0, ifcr);
+}
+
+static void loongson2_cmc_dma_stop(struct loongson2_cmc_dma_chan *lchan)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	u32 ccr;
+
+	ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
+	ccr &= ~(LOONGSON2_CMCDMA_CCR_IRQ_MASK | LOONGSON2_CMCDMA_CCR_EN);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, ccr);
+
+	loongson2_cmc_dma_irq_clear(lchan, LOONGSON2_CMCDMA_MASKI);
+}
+
+static int loongson2_cmc_dma_terminate_all(struct dma_chan *chan)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	LIST_HEAD(head);
+
+	scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
+		if (lchan->desc) {
+			vchan_terminate_vdesc(&lchan->desc->vdesc);
+			loongson2_cmc_dma_stop(lchan);
+			lchan->desc = NULL;
+		}
+		vchan_get_all_descriptors(&lchan->vchan, &head);
+	}
+
+	vchan_dma_desc_free_list(&lchan->vchan, &head);
+
+	return 0;
+}
+
+static void loongson2_cmc_dma_synchronize(struct dma_chan *chan)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	vchan_synchronize(&lchan->vchan);
+}
+
+static void loongson2_cmc_dma_start_transfer(struct loongson2_cmc_dma_chan *lchan)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	struct loongson2_cmc_dma_sg_req *sg_req;
+	struct loongson2_cmc_dma_chan_reg *reg;
+	struct virt_dma_desc *vdesc;
+
+	loongson2_cmc_dma_stop(lchan);
+
+	if (!lchan->desc) {
+		vdesc = vchan_next_desc(&lchan->vchan);
+		if (!vdesc)
+			return;
+
+		list_del(&vdesc->node);
+		lchan->desc = to_lmdma_desc(vdesc);
+		lchan->next_sg = 0;
+	}
+
+	if (lchan->next_sg == lchan->desc->num_sgs)
+		lchan->next_sg = 0;
+
+	sg_req = &lchan->desc->sg_req[lchan->next_sg];
+	reg = &sg_req->chan_reg;
+
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id, reg->cndtr);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CPAR, lchan->id, reg->cpar);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, lchan->id, reg->cmar);
+
+	lchan->next_sg++;
+
+	/* Start DMA */
+	reg->ccr |= LOONGSON2_CMCDMA_CCR_EN;
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr);
+}
+
+static void loongson2_cmc_dma_configure_next_sg(struct loongson2_cmc_dma_chan *lchan)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	struct loongson2_cmc_dma_sg_req *sg_req;
+	u32 ccr, id = lchan->id;
+
+	if (lchan->next_sg == lchan->desc->num_sgs)
+		lchan->next_sg = 0;
+
+	/* Stop to update mem addr */
+	ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, id);
+	ccr &= ~LOONGSON2_CMCDMA_CCR_EN;
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr);
+
+	sg_req = &lchan->desc->sg_req[lchan->next_sg];
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, id, sg_req->chan_reg.cmar);
+
+	/* Start transition */
+	ccr |= LOONGSON2_CMCDMA_CCR_EN;
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr);
+}
+
+static void loongson2_cmc_dma_handle_chan_done(struct loongson2_cmc_dma_chan *lchan)
+{
+	if (!lchan->desc)
+		return;
+
+	if (lchan->desc->cyclic) {
+		vchan_cyclic_callback(&lchan->desc->vdesc);
+		/* LOONGSON2_CMCDMA_CCR_CIRC mode don't need update register */
+		if (lchan->desc->num_sgs == 1)
+			return;
+		loongson2_cmc_dma_configure_next_sg(lchan);
+		lchan->next_sg++;
+	} else {
+		if (lchan->next_sg == lchan->desc->num_sgs) {
+			vchan_cookie_complete(&lchan->desc->vdesc);
+			lchan->desc = NULL;
+		}
+		loongson2_cmc_dma_start_transfer(lchan);
+	}
+}
+
+static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid)
+{
+	struct loongson2_cmc_dma_chan *lchan = devid;
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	struct device *dev = chan2dev(lchan);
+	u32 ists, status, ccr;
+
+	scoped_guard(spinlock, &lchan->vchan.lock) {
+		ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
+		ists = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_ISR, 0);
+		status = (ists >> (4 * lchan->id)) & LOONGSON2_CMCDMA_MASKI;
+
+		loongson2_cmc_dma_irq_clear(lchan, status);
+
+		if (status & LOONGSON2_CMCDMA_TCI) {
+			loongson2_cmc_dma_handle_chan_done(lchan);
+			status &= ~LOONGSON2_CMCDMA_TCI;
+		}
+
+		if (status & LOONGSON2_CMCDMA_HTI)
+			status &= ~LOONGSON2_CMCDMA_HTI;
+
+		if (status & LOONGSON2_CMCDMA_TEI) {
+			dev_err(dev, "DMA Transform Error.\n");
+			if (!(ccr & LOONGSON2_CMCDMA_CCR_EN))
+				dev_err(dev, "Channel disabled by HW.\n");
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) {
+		dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan);
+		loongson2_cmc_dma_start_transfer(lchan);
+	}
+}
+
+static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan,
+					    enum dma_transfer_direction direction,
+					    enum dma_slave_buswidth *buswidth, u32 buf_len)
+{
+	struct dma_slave_config	sconfig = lchan->dma_sconfig;
+	struct device *dev = chan2dev(lchan);
+	int dev_width;
+	u32 ccr;
+
+	switch (direction) {
+	case DMA_MEM_TO_DEV:
+		dev_width = loongson2_cmc_dma_get_width(sconfig.dst_addr_width);
+		if (dev_width < 0) {
+			dev_err(dev, "DMA_MEM_TO_DEV bus width not supported\n");
+			return dev_width;
+		}
+		lchan->chan_reg.cpar = sconfig.dst_addr;
+		ccr = LOONGSON2_CMCDMA_CCR_DIR;
+		*buswidth = sconfig.dst_addr_width;
+		break;
+	case DMA_DEV_TO_MEM:
+		dev_width = loongson2_cmc_dma_get_width(sconfig.src_addr_width);
+		if (dev_width < 0) {
+			dev_err(dev, "DMA_DEV_TO_MEM bus width not supported\n");
+			return dev_width;
+		}
+		lchan->chan_reg.cpar = sconfig.src_addr;
+		ccr = LOONGSON2_CMCDMA_CCR_MINC;
+		*buswidth = sconfig.src_addr_width;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ccr |= FIELD_PREP(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, dev_width) |
+	       FIELD_PREP(LOONGSON2_CMCDMA_CCR_MSIZE_MASK, dev_width);
+
+	/* Set DMA control register */
+	lchan->chan_reg.ccr &= ~(LOONGSON2_CMCDMA_CCR_PSIZE_MASK | LOONGSON2_CMCDMA_CCR_MSIZE_MASK);
+	lchan->chan_reg.ccr |= ccr;
+
+	return 0;
+}
+
+static struct dma_async_tx_descriptor *
+loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 sg_len,
+				enum dma_transfer_direction direction,
+				unsigned long flags, void *context)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct loongson2_cmc_dma_desc *desc;
+	enum dma_slave_buswidth buswidth;
+	struct scatterlist *sg;
+	u32 num_items, i;
+	int ret;
+
+	desc = kzalloc_flex(*desc, sg_req, sg_len, GFP_NOWAIT);
+	if (!desc)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, sg_dma_len(sg));
+		if (ret)
+			return ERR_PTR(ret);
+
+		num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth);
+		if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
+			dev_err(chan2dev(lchan), "Number of items not supported\n");
+			kfree(desc);
+			return ERR_PTR(-EINVAL);
+		}
+
+		desc->sg_req[i].len = sg_dma_len(sg);
+		desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr;
+		desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar;
+		desc->sg_req[i].chan_reg.cmar = sg_dma_address(sg);
+		desc->sg_req[i].chan_reg.cndtr = num_items;
+	}
+
+	desc->num_sgs = sg_len;
+	desc->cyclic = false;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+static struct dma_async_tx_descriptor *
+loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+				  size_t period_len, enum dma_transfer_direction direction,
+				  unsigned long flags)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct loongson2_cmc_dma_desc *desc;
+	enum dma_slave_buswidth buswidth;
+	u32 num_periods, num_items, i;
+	int ret;
+
+	if (unlikely(buf_len % period_len))
+		return ERR_PTR(-EINVAL);
+
+	ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, period_len);
+	if (ret)
+		return ERR_PTR(ret);
+
+	num_items = DIV_ROUND_UP(period_len, buswidth);
+	if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
+		dev_err(chan2dev(lchan), "Number of items not supported\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Enable Circular mode */
+	if (buf_len == period_len)
+		lchan->chan_reg.ccr |= LOONGSON2_CMCDMA_CCR_CIRC;
+
+	num_periods = DIV_ROUND_UP(buf_len, period_len);
+	desc = kzalloc_flex(*desc, sg_req, num_periods, GFP_NOWAIT);
+	if (!desc)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < num_periods; i++) {
+		desc->sg_req[i].len = period_len;
+		desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr;
+		desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar;
+		desc->sg_req[i].chan_reg.cmar = buf_addr;
+		desc->sg_req[i].chan_reg.cndtr = num_items;
+		buf_addr += period_len;
+	}
+
+	desc->num_sgs = num_periods;
+	desc->cyclic = true;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+static size_t loongson2_cmc_dma_desc_residue(struct loongson2_cmc_dma_chan *lchan,
+					     struct loongson2_cmc_dma_desc *desc, u32 next_sg)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	u32 residue, width, ndtr, ccr, i;
+
+	ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
+	width = FIELD_GET(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, ccr);
+
+	ndtr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id);
+	residue = ndtr << width;
+
+	if (lchan->desc->cyclic && next_sg == 0)
+		return residue;
+
+	for (i = next_sg; i < desc->num_sgs; i++)
+		residue += desc->sg_req[i].len;
+
+	return residue;
+}
+
+static enum dma_status loongson2_cmc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
+						   struct dma_tx_state *state)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct virt_dma_desc *vdesc;
+	enum dma_status status;
+
+	status = dma_cookie_status(chan, cookie, state);
+	if (status == DMA_COMPLETE || !state)
+		return status;
+
+	scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
+		vdesc = vchan_find_desc(&lchan->vchan, cookie);
+		if (lchan->desc && cookie == lchan->desc->vdesc.tx.cookie)
+			state->residue = loongson2_cmc_dma_desc_residue(lchan, lchan->desc,
+									lchan->next_sg);
+		else if (vdesc)
+			state->residue = loongson2_cmc_dma_desc_residue(lchan,
+									to_lmdma_desc(vdesc), 0);
+	}
+
+	return status;
+}
+
+static void loongson2_cmc_dma_free_chan_resources(struct dma_chan *chan)
+{
+	vchan_free_chan_resources(to_virt_chan(chan));
+}
+
+static void loongson2_cmc_dma_desc_free(struct virt_dma_desc *vdesc)
+{
+	kfree(to_lmdma_desc(vdesc));
+}
+
+static bool loongson2_cmc_dma_acpi_filter(struct dma_chan *chan, void *param)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct acpi_dma_spec *dma_spec = param;
+
+	memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg));
+	lchan->chan_reg.ccr = dma_spec->chan_id & LOONGSON2_CMCDMA_STREAM_MASK;
+
+	return true;
+}
+
+static int loongson2_cmc_dma_acpi_controller_register(struct loongson2_cmc_dma_dev *lddev)
+{
+	struct device *dev = lddev->ddev.dev;
+	struct acpi_dma_filter_info *info;
+
+	if (!is_acpi_node(dev_fwnode(dev)))
+		return 0;
+
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	dma_cap_zero(info->dma_cap);
+	info->dma_cap = lddev->ddev.cap_mask;
+	info->filter_fn = loongson2_cmc_dma_acpi_filter;
+
+	return devm_acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info);
+}
+
+static struct dma_chan *loongson2_cmc_dma_of_xlate(struct of_phandle_args *dma_spec,
+						   struct of_dma *ofdma)
+{
+	struct loongson2_cmc_dma_dev *lddev = ofdma->of_dma_data;
+	struct device *dev = lddev->ddev.dev;
+	struct loongson2_cmc_dma_chan *lchan;
+	struct dma_chan *chan;
+
+	if (dma_spec->args_count < 2)
+		return ERR_PTR(-EINVAL);
+
+	if (dma_spec->args[0] >= lddev->nr_channels) {
+		dev_err(dev, "Invalid channel id.\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	lchan = &lddev->chan[dma_spec->args[0]];
+	chan = dma_get_slave_channel(&lchan->vchan.chan);
+	if (!chan) {
+		dev_err(dev, "No more channels available.\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg));
+	lchan->chan_reg.ccr = dma_spec->args[1] & LOONGSON2_CMCDMA_STREAM_MASK;
+
+	return chan;
+}
+
+static int loongson2_cmc_dma_of_controller_register(struct loongson2_cmc_dma_dev *lddev)
+{
+	struct device *dev = lddev->ddev.dev;
+
+	if (!is_of_node(dev_fwnode(dev)))
+		return 0;
+
+	return of_dma_controller_register(dev->of_node, loongson2_cmc_dma_of_xlate, lddev);
+}
+
+static int loongson2_cmc_dma_probe(struct platform_device *pdev)
+{
+	const struct loongson2_cmc_dma_config *config;
+	struct loongson2_cmc_dma_chan *lchan;
+	struct loongson2_cmc_dma_dev *lddev;
+	struct device *dev = &pdev->dev;
+	struct dma_device *ddev;
+	u32 nr_chans, i;
+	int ret;
+
+	config = (const struct loongson2_cmc_dma_config *)device_get_match_data(dev);
+	if (!config)
+		return -EINVAL;
+
+	ret = device_property_read_u32(dev, "dma-channels", &nr_chans);
+	if (ret || nr_chans > config->max_channels) {
+		dev_err(dev, "missing or invalid dma-channels property\n");
+		nr_chans = config->max_channels;
+	}
+
+	lddev = devm_kzalloc(dev, struct_size(lddev, chan, nr_chans), GFP_KERNEL);
+	if (!lddev)
+		return -ENOMEM;
+
+	lddev->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(lddev->base))
+		return PTR_ERR(lddev->base);
+
+	platform_set_drvdata(pdev, lddev);
+	lddev->nr_channels = nr_chans;
+	lddev->chan_reg_offset = config->chan_reg_offset;
+
+	lddev->dma_clk = devm_clk_get_optional_enabled(dev, NULL);
+	if (IS_ERR(lddev->dma_clk))
+		return dev_err_probe(dev, PTR_ERR(lddev->dma_clk), "Failed to get dma clock\n");
+
+	ddev = &lddev->ddev;
+	ddev->dev = dev;
+
+	dma_cap_zero(ddev->cap_mask);
+	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
+	dma_cap_set(DMA_PRIVATE, ddev->cap_mask);
+	dma_cap_set(DMA_CYCLIC, ddev->cap_mask);
+
+	ddev->device_free_chan_resources = loongson2_cmc_dma_free_chan_resources;
+	ddev->device_config = loongson2_cmc_dma_slave_config;
+	ddev->device_prep_slave_sg = loongson2_cmc_dma_prep_slave_sg;
+	ddev->device_prep_dma_cyclic = loongson2_cmc_dma_prep_dma_cyclic;
+	ddev->device_issue_pending = loongson2_cmc_dma_issue_pending;
+	ddev->device_synchronize = loongson2_cmc_dma_synchronize;
+	ddev->device_tx_status = loongson2_cmc_dma_tx_status;
+	ddev->device_terminate_all = loongson2_cmc_dma_terminate_all;
+
+	ddev->max_sg_burst = LOONSON2_CMCDMA_MAX_DATA_ITEMS;
+	ddev->src_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS;
+	ddev->dst_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS;
+	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	INIT_LIST_HEAD(&ddev->channels);
+
+	for (i = 0; i < nr_chans; i++) {
+		lchan = &lddev->chan[i];
+
+		lchan->id = i;
+		lchan->vchan.desc_free = loongson2_cmc_dma_desc_free;
+		vchan_init(&lchan->vchan, ddev);
+	}
+
+	ret = dmaenginem_async_device_register(ddev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n");
+
+	for (i = 0; i < nr_chans; i++) {
+		lchan = &lddev->chan[i];
+
+		lchan->irq = platform_get_irq(pdev, i);
+		if (lchan->irq < 0)
+			return lchan->irq;
+
+		ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED,
+				       dev_name(chan2dev(lchan)), lchan);
+		if (ret)
+			return ret;
+	}
+
+	ret = loongson2_cmc_dma_acpi_controller_register(lddev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register dma controller with ACPI.\n");
+
+	ret = loongson2_cmc_dma_of_controller_register(lddev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register dma controller with FDT.\n");
+
+	dev_info(dev, "Loongson-2 Multi-Channel DMA Controller registered successfully.\n");
+
+	return 0;
+}
+
+static void loongson2_cmc_dma_remove(struct platform_device *pdev)
+{
+	of_dma_controller_free(pdev->dev.of_node);
+}
+
+static const struct of_device_id loongson2_cmc_dma_of_match[] = {
+	{ .compatible = "loongson,ls2k0300-dma", .data = &ls2k0300_cmc_dma_config },
+	{ .compatible = "loongson,ls2k3000-dma", .data = &ls2k3000_cmc_dma_config },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, loongson2_cmc_dma_of_match);
+
+static const struct acpi_device_id loongson2_cmc_dma_acpi_match[] = {
+	{ "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(acpi, loongson2_cmc_dma_acpi_match);
+
+static struct platform_driver loongson2_cmc_dma_driver = {
+	.driver = {
+		.name = "loongson2-apb-cmc-dma",
+		.of_match_table = loongson2_cmc_dma_of_match,
+		.acpi_match_table = loongson2_cmc_dma_acpi_match,
+	},
+	.probe = loongson2_cmc_dma_probe,
+	.remove = loongson2_cmc_dma_remove,
+};
+module_platform_driver(loongson2_cmc_dma_driver);
+
+MODULE_DESCRIPTION("Loongson-2 Chain Multi-Channel DMA Controller driver");
+MODULE_AUTHOR("Loongson Technology Corporation Limited");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/loongson/loongson2-apb-dma.c b/drivers/dma/loongson/loongson2-apb-dma.c
new file mode 100644
index 000000000..aceb069e7
--- /dev/null
+++ b/drivers/dma/loongson/loongson2-apb-dma.c
@@ -0,0 +1,686 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for the Loongson-2 APB DMA Controller
+ *
+ * Copyright (C) 2017-2023 Loongson Corporation
+ */
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+/* Global Configuration Register */
+#define LDMA_ORDER_ERG		0x0
+
+/* Bitfield definitions */
+
+/* Bitfields in Global Configuration Register */
+#define LDMA_64BIT_EN		BIT(0) /* 1: 64 bit support */
+#define LDMA_UNCOHERENT_EN	BIT(1) /* 0: cache, 1: uncache */
+#define LDMA_ASK_VALID		BIT(2)
+#define LDMA_START		BIT(3) /* DMA start operation */
+#define LDMA_STOP		BIT(4) /* DMA stop operation */
+#define LDMA_CONFIG_MASK	GENMASK_ULL(4, 0) /* DMA controller config bits mask */
+
+/* Bitfields in ndesc_addr field of HW descriptor */
+#define LDMA_DESC_EN		BIT(0) /*1: The next descriptor is valid */
+#define LDMA_DESC_ADDR_LOW	GENMASK(31, 1)
+
+/* Bitfields in cmd field of HW descriptor */
+#define LDMA_INT		BIT(1) /* Enable DMA interrupts */
+#define LDMA_DATA_DIRECTION	BIT(12) /* 1: write to device, 0: read from device */
+
+#define LDMA_SLAVE_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
+
+#define LDMA_MAX_TRANS_LEN	U32_MAX
+
+/*--  descriptors  -----------------------------------------------------*/
+
+/*
+ * struct ls2x_dma_hw_desc - DMA HW descriptor
+ * @ndesc_addr: the next descriptor low address.
+ * @mem_addr: memory low address.
+ * @apb_addr: device buffer address.
+ * @len: length of a piece of carried content, in words.
+ * @step_len: length between two moved memory data blocks.
+ * @step_times: number of blocks to be carried in a single DMA operation.
+ * @cmd: descriptor command or state.
+ * @stats: DMA status.
+ * @high_ndesc_addr: the next descriptor high address.
+ * @high_mem_addr: memory high address.
+ * @reserved: reserved
+ */
+struct ls2x_dma_hw_desc {
+	u32 ndesc_addr;
+	u32 mem_addr;
+	u32 apb_addr;
+	u32 len;
+	u32 step_len;
+	u32 step_times;
+	u32 cmd;
+	u32 stats;
+	u32 high_ndesc_addr;
+	u32 high_mem_addr;
+	u32 reserved[2];
+} __packed;
+
+/*
+ * struct ls2x_dma_sg - ls2x dma scatter gather entry
+ * @hw: the pointer to DMA HW descriptor.
+ * @llp: physical address of the DMA HW descriptor.
+ * @phys: destination or source address(mem).
+ * @len: number of Bytes to read.
+ */
+struct ls2x_dma_sg {
+	struct ls2x_dma_hw_desc	*hw;
+	dma_addr_t		llp;
+	dma_addr_t		phys;
+	u32			len;
+};
+
+/*
+ * struct ls2x_dma_desc - software descriptor
+ * @vdesc: pointer to the virtual dma descriptor.
+ * @cyclic: flag to dma cyclic
+ * @burst_size: burst size of transaction, in words.
+ * @desc_num: number of sg entries.
+ * @direction: transfer direction, to or from device.
+ * @status: dma controller status.
+ * @sg: array of sgs.
+ */
+struct ls2x_dma_desc {
+	struct virt_dma_desc		vdesc;
+	bool				cyclic;
+	size_t				burst_size;
+	u32				desc_num;
+	enum dma_transfer_direction	direction;
+	enum dma_status			status;
+	struct ls2x_dma_sg		sg[] __counted_by(desc_num);
+};
+
+/*--  Channels  --------------------------------------------------------*/
+
+/*
+ * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel
+ * @vchan: virtual dma channel entry.
+ * @desc: pointer to the ls2x sw dma descriptor.
+ * @pool: hw desc table
+ * @irq: irq line
+ * @sconfig: configuration for slave transfers, passed via .device_config
+ */
+struct ls2x_dma_chan {
+	struct virt_dma_chan	vchan;
+	struct ls2x_dma_desc	*desc;
+	void			*pool;
+	int			irq;
+	struct dma_slave_config	sconfig;
+};
+
+/*--  Controller  ------------------------------------------------------*/
+
+/*
+ * struct ls2x_dma_priv - LS2X APB DMAC specific information
+ * @ddev: dmaengine dma_device object members
+ * @dma_clk: DMAC clock source
+ * @regs: memory mapped register base
+ * @lchan: channel to store ls2x_dma_chan structures
+ */
+struct ls2x_dma_priv {
+	struct dma_device	ddev;
+	struct clk		*dma_clk;
+	void __iomem		*regs;
+	struct ls2x_dma_chan	lchan;
+};
+
+/*--  Helper functions  ------------------------------------------------*/
+
+static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc)
+{
+	return container_of(vdesc, struct ls2x_dma_desc, vdesc);
+}
+
+static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan)
+{
+	return container_of(chan, struct ls2x_dma_chan, vchan.chan);
+}
+
+static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev)
+{
+	return container_of(ddev, struct ls2x_dma_priv, ddev);
+}
+
+static struct device *chan2dev(struct dma_chan *chan)
+{
+	return &chan->dev->device;
+}
+
+static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan);
+	struct ls2x_dma_desc *desc = to_ldma_desc(vdesc);
+	int i;
+
+	for (i = 0; i < desc->desc_num; i++) {
+		if (desc->sg[i].hw)
+			dma_pool_free(lchan->pool, desc->sg[i].hw,
+				      desc->sg[i].llp);
+	}
+
+	kfree(desc);
+}
+
+static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd)
+{
+	struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device);
+	u64 val;
+
+	val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK;
+	val |= LDMA_64BIT_EN | cmd;
+	lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG);
+}
+
+static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan)
+{
+	struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device);
+	struct ls2x_dma_sg *ldma_sg;
+	struct virt_dma_desc *vdesc;
+	u64 val;
+
+	/* Get the next descriptor */
+	vdesc = vchan_next_desc(&lchan->vchan);
+	if (!vdesc) {
+		lchan->desc = NULL;
+		return;
+	}
+
+	list_del(&vdesc->node);
+	lchan->desc = to_ldma_desc(vdesc);
+	ldma_sg = &lchan->desc->sg[0];
+
+	/* Start DMA */
+	lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG);
+	val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START;
+	lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG);
+}
+
+static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan)
+{
+	u32 maxburst, buswidth;
+
+	/* Reject definitely invalid configurations */
+	if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) &&
+	    (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS))
+		return 0;
+
+	if (lchan->sconfig.direction == DMA_MEM_TO_DEV) {
+		maxburst = lchan->sconfig.dst_maxburst;
+		buswidth = lchan->sconfig.dst_addr_width;
+	} else {
+		maxburst = lchan->sconfig.src_maxburst;
+		buswidth = lchan->sconfig.src_addr_width;
+	}
+
+	/* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */
+	return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN;
+}
+
+static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index,
+			       struct ls2x_dma_desc *desc)
+{
+	struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index];
+	u32 num_segments, segment_size;
+
+	if (desc->direction == DMA_MEM_TO_DEV) {
+		ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION;
+		ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr;
+	} else {
+		ldma_sg->hw->cmd = LDMA_INT;
+		ldma_sg->hw->apb_addr = lchan->sconfig.src_addr;
+	}
+
+	ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys);
+	ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys);
+
+	/* Split into multiple equally sized segments if necessary */
+	num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size);
+	segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments);
+
+	/* Word count register takes input in words */
+	ldma_sg->hw->len = segment_size;
+	ldma_sg->hw->step_times = num_segments;
+	ldma_sg->hw->step_len = 0;
+
+	/* lets make a link list */
+	if (sg_index) {
+		desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN;
+		desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp);
+	}
+}
+
+/*--  DMA Engine API  --------------------------------------------------*/
+
+/*
+ * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel
+ * @chan: allocate descriptor resources for this channel
+ *
+ * return - the number of allocated descriptors
+ */
+static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	/* Create a pool of consistent memory blocks for hardware descriptors */
+	lchan->pool = dma_pool_create(dev_name(chan2dev(chan)),
+				      chan->device->dev, PAGE_SIZE,
+				      __alignof__(struct ls2x_dma_hw_desc), 0);
+	if (!lchan->pool) {
+		dev_err(chan2dev(chan), "No memory for descriptors\n");
+		return -ENOMEM;
+	}
+
+	return 1;
+}
+
+/*
+ * ls2x_dma_free_chan_resources - free all channel resources
+ * @chan: DMA channel
+ */
+static void ls2x_dma_free_chan_resources(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	vchan_free_chan_resources(to_virt_chan(chan));
+	dma_pool_destroy(lchan->pool);
+	lchan->pool = NULL;
+}
+
+/*
+ * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: tx descriptor status flags
+ * @context: transaction context (ignored)
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
+ */
+static struct dma_async_tx_descriptor *
+ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+		       u32 sg_len, enum dma_transfer_direction direction,
+		       unsigned long flags, void *context)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+	struct ls2x_dma_desc *desc;
+	struct scatterlist *sg;
+	size_t burst_size;
+	int i;
+
+	if (unlikely(!sg_len || !is_slave_direction(direction)))
+		return NULL;
+
+	burst_size = ls2x_dmac_detect_burst(lchan);
+	if (!burst_size)
+		return NULL;
+
+	desc = kzalloc_flex(*desc, sg, sg_len, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	desc->desc_num = sg_len;
+	desc->direction = direction;
+	desc->burst_size = burst_size;
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		struct ls2x_dma_sg *ldma_sg = &desc->sg[i];
+
+		/* Allocate DMA capable memory for hardware descriptor */
+		ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp);
+		if (!ldma_sg->hw) {
+			desc->desc_num = i;
+			ls2x_dma_desc_free(&desc->vdesc);
+			return NULL;
+		}
+
+		ldma_sg->phys = sg_dma_address(sg);
+		ldma_sg->len = sg_dma_len(sg);
+
+		ls2x_dma_fill_desc(lchan, i, desc);
+	}
+
+	/* Setting the last descriptor enable bit */
+	desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN;
+	desc->status = DMA_IN_PROGRESS;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+/*
+ * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer
+ * @chan: the DMA channel to prepare
+ * @buf_addr: physical DMA address where the buffer starts
+ * @buf_len: total number of bytes for the entire buffer
+ * @period_len: number of bytes for each period
+ * @direction: transfer direction, to or from device
+ * @flags: tx descriptor status flags
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
+ */
+static struct dma_async_tx_descriptor *
+ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+			 size_t period_len, enum dma_transfer_direction direction,
+			 unsigned long flags)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+	struct ls2x_dma_desc *desc;
+	size_t burst_size;
+	u32 num_periods;
+	int i;
+
+	if (unlikely(!buf_len || !period_len))
+		return NULL;
+
+	if (unlikely(!is_slave_direction(direction)))
+		return NULL;
+
+	burst_size = ls2x_dmac_detect_burst(lchan);
+	if (!burst_size)
+		return NULL;
+
+	num_periods = buf_len / period_len;
+	desc = kzalloc_flex(*desc, sg, num_periods, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	desc->desc_num = num_periods;
+	desc->direction = direction;
+	desc->burst_size = burst_size;
+
+	/* Build cyclic linked list */
+	for (i = 0; i < num_periods; i++) {
+		struct ls2x_dma_sg *ldma_sg = &desc->sg[i];
+
+		/* Allocate DMA capable memory for hardware descriptor */
+		ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp);
+		if (!ldma_sg->hw) {
+			desc->desc_num = i;
+			ls2x_dma_desc_free(&desc->vdesc);
+			return NULL;
+		}
+
+		ldma_sg->phys = buf_addr + period_len * i;
+		ldma_sg->len = period_len;
+
+		ls2x_dma_fill_desc(lchan, i, desc);
+	}
+
+	/* Lets make a cyclic list */
+	desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN;
+	desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp);
+	desc->cyclic = true;
+	desc->status = DMA_IN_PROGRESS;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+/*
+ * ls2x_slave_config - set slave configuration for channel
+ * @chan: dma channel
+ * @cfg: slave configuration
+ *
+ * Sets slave configuration for channel
+ */
+static int ls2x_dma_slave_config(struct dma_chan *chan,
+				 struct dma_slave_config *config)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	memcpy(&lchan->sconfig, config, sizeof(*config));
+	return 0;
+}
+
+/*
+ * ls2x_dma_issue_pending - push pending transactions to the hardware
+ * @chan: channel
+ *
+ * When this function is called, all pending transactions are pushed to the
+ * hardware and executed.
+ */
+static void ls2x_dma_issue_pending(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (vchan_issue_pending(&lchan->vchan) && !lchan->desc)
+		ls2x_dma_start_transfer(lchan);
+}
+
+/*
+ * ls2x_dma_terminate_all - terminate all transactions
+ * @chan: channel
+ *
+ * Stops all DMA transactions.
+ */
+static int ls2x_dma_terminate_all(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+	LIST_HEAD(head);
+
+	scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
+		/* Setting stop cmd */
+		ls2x_dma_write_cmd(lchan, LDMA_STOP);
+		if (lchan->desc) {
+			vchan_terminate_vdesc(&lchan->desc->vdesc);
+			lchan->desc = NULL;
+		}
+
+		vchan_get_all_descriptors(&lchan->vchan, &head);
+	}
+
+	vchan_dma_desc_free_list(&lchan->vchan, &head);
+	return 0;
+}
+
+/*
+ * ls2x_dma_synchronize - Synchronizes the termination of transfers to the
+ * current context.
+ * @chan: channel
+ */
+static void ls2x_dma_synchronize(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	vchan_synchronize(&lchan->vchan);
+}
+
+static int ls2x_dma_pause(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) {
+		ls2x_dma_write_cmd(lchan, LDMA_STOP);
+		lchan->desc->status = DMA_PAUSED;
+	}
+
+	return 0;
+}
+
+static int ls2x_dma_resume(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (lchan->desc && lchan->desc->status == DMA_PAUSED) {
+		lchan->desc->status = DMA_IN_PROGRESS;
+		ls2x_dma_write_cmd(lchan, LDMA_START);
+	}
+
+	return 0;
+}
+
+/*
+ * ls2x_dma_isr - LS2X DMA Interrupt handler
+ * @irq: IRQ number
+ * @dev_id: Pointer to ls2x_dma_chan
+ *
+ * Return: IRQ_HANDLED/IRQ_NONE
+ */
+static irqreturn_t ls2x_dma_isr(int irq, void *dev_id)
+{
+	struct ls2x_dma_chan *lchan = dev_id;
+	struct ls2x_dma_desc *desc;
+
+	scoped_guard(spinlock, &lchan->vchan.lock) {
+		desc = lchan->desc;
+		if (desc) {
+			if (desc->cyclic) {
+				vchan_cyclic_callback(&desc->vdesc);
+			} else {
+				desc->status = DMA_COMPLETE;
+				vchan_cookie_complete(&desc->vdesc);
+				ls2x_dma_start_transfer(lchan);
+			}
+
+			/* ls2x_dma_start_transfer() updates lchan->desc */
+			if (!lchan->desc)
+				ls2x_dma_write_cmd(lchan, LDMA_STOP);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int ls2x_dma_chan_init(struct platform_device *pdev,
+			      struct ls2x_dma_priv *priv)
+{
+	struct ls2x_dma_chan *lchan = &priv->lchan;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	lchan->irq = platform_get_irq(pdev, 0);
+	if (lchan->irq < 0)
+		return lchan->irq;
+
+	ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING,
+			       dev_name(&pdev->dev), lchan);
+	if (ret)
+		return ret;
+
+	/* Initialize channels related values */
+	INIT_LIST_HEAD(&priv->ddev.channels);
+	lchan->vchan.desc_free = ls2x_dma_desc_free;
+	vchan_init(&lchan->vchan, &priv->ddev);
+
+	return 0;
+}
+
+/*
+ * ls2x_dma_probe - Driver probe function
+ * @pdev: Pointer to the platform_device structure
+ *
+ * Return: '0' on success and failure value on error
+ */
+static int ls2x_dma_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ls2x_dma_priv *priv;
+	struct dma_device *ddev;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->regs))
+		return dev_err_probe(dev, PTR_ERR(priv->regs),
+				     "devm_platform_ioremap_resource failed.\n");
+
+	priv->dma_clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(priv->dma_clk))
+		return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "Couldn't start the clock.\n");
+
+	ret = ls2x_dma_chan_init(pdev, priv);
+	if (ret)
+		return ret;
+
+	ddev = &priv->ddev;
+	ddev->dev = dev;
+	dma_cap_zero(ddev->cap_mask);
+	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
+	dma_cap_set(DMA_CYCLIC, ddev->cap_mask);
+
+	ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources;
+	ddev->device_free_chan_resources = ls2x_dma_free_chan_resources;
+	ddev->device_tx_status = dma_cookie_status;
+	ddev->device_issue_pending = ls2x_dma_issue_pending;
+	ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg;
+	ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic;
+	ddev->device_config = ls2x_dma_slave_config;
+	ddev->device_terminate_all = ls2x_dma_terminate_all;
+	ddev->device_synchronize = ls2x_dma_synchronize;
+	ddev->device_pause = ls2x_dma_pause;
+	ddev->device_resume = ls2x_dma_resume;
+
+	ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS;
+	ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS;
+	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+
+	ret = dmaenginem_async_device_register(&priv->ddev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n");
+
+	ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to register dma controller.\n");
+
+	platform_set_drvdata(pdev, priv);
+
+	dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n");
+	return 0;
+}
+
+/*
+ * ls2x_dma_remove - Driver remove function
+ * @pdev: Pointer to the platform_device structure
+ */
+static void ls2x_dma_remove(struct platform_device *pdev)
+{
+	of_dma_controller_free(pdev->dev.of_node);
+}
+
+static const struct of_device_id ls2x_dma_of_match_table[] = {
+	{ .compatible = "loongson,ls2k1000-apbdma" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table);
+
+static struct platform_driver ls2x_dmac_driver = {
+	.probe		= ls2x_dma_probe,
+	.remove		= ls2x_dma_remove,
+	.driver = {
+		.name	= "ls2x-apbdma",
+		.of_match_table	= ls2x_dma_of_match_table,
+	},
+};
+module_platform_driver(ls2x_dmac_driver);
+
+MODULE_DESCRIPTION("Loongson-2 APB DMA Controller driver");
+MODULE_AUTHOR("Loongson Technology Corporation Limited");
+MODULE_LICENSE("GPL");