From 5febf432df1cfa5b25d99b54c32103fafbdd0eb9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 16 Aug 2026 15:32:33 +0200 Subject: kbuild: rust: keep Rust objects out of Clang LTO with inline helpers Under `CONFIG_LTO_CLANG` + `CONFIG_RUST_INLINE_HELPERS`, one may hit `objtool` errors such as: vmlinux.o: error: objtool: _R..._3Gsp4boot+0xd6a: can't find jump dest instruction at .text._R..._3Gsp4boot+0x1dfd The reason is that in such builds, the Clang invocation that compiles the combined Rust plus helpers bitcode emits LLVM bitcode (again) -- the final code generation happens in the linker's LTO step, which the `-mllvm` trap options passed to Clang do not reach. This, in turn, means that unreachable traps are missing, and the impossible paths do not merely fallthrough to the next symbol, but past the end of their own section, since LTO builds place each function in its own section. Thus filter `CC_FLAGS_LTO` out of the Clang invocation, so that it always emits machine code directly, with the traps in place. Assisted-by: LLM Cc: Gary Guo Cc: Boqun Feng Cc: Alice Ryhl Cc: Matthew Maurer Cc: Josh Poimboeuf Cc: Peter Zijlstra Cc: stable@vger.kernel.org Fixes: 3a2486cc1da5 ("kbuild: rust: provide an option to inline C helpers into Rust") Acked-by: Gary Guo Link: https://patch.msgid.link/20260816133233.197500-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- drivers/dma/amd/Kconfig | 42 ++ drivers/dma/amd/Makefile | 5 + drivers/dma/amd/ae4dma/Makefile | 10 + drivers/dma/amd/ae4dma/ae4dma-dev.c | 157 +++++ drivers/dma/amd/ae4dma/ae4dma-pci.c | 156 +++++ drivers/dma/amd/ae4dma/ae4dma.h | 102 +++ drivers/dma/amd/ptdma/Makefile | 10 + drivers/dma/amd/ptdma/ptdma-debugfs.c | 143 ++++ drivers/dma/amd/ptdma/ptdma-dev.c | 309 +++++++++ drivers/dma/amd/ptdma/ptdma-dmaengine.c | 659 ++++++++++++++++++ drivers/dma/amd/ptdma/ptdma-pci.c | 243 +++++++ drivers/dma/amd/ptdma/ptdma.h | 338 +++++++++ drivers/dma/amd/qdma/Makefile | 5 + drivers/dma/amd/qdma/qdma-comm-regs.c | 64 ++ drivers/dma/amd/qdma/qdma.c | 1142 +++++++++++++++++++++++++++++++ drivers/dma/amd/qdma/qdma.h | 266 +++++++ 16 files changed, 3651 insertions(+) create mode 100644 drivers/dma/amd/Kconfig create mode 100644 drivers/dma/amd/Makefile create mode 100644 drivers/dma/amd/ae4dma/Makefile create mode 100644 drivers/dma/amd/ae4dma/ae4dma-dev.c create mode 100644 drivers/dma/amd/ae4dma/ae4dma-pci.c create mode 100644 drivers/dma/amd/ae4dma/ae4dma.h create mode 100644 drivers/dma/amd/ptdma/Makefile create mode 100644 drivers/dma/amd/ptdma/ptdma-debugfs.c create mode 100644 drivers/dma/amd/ptdma/ptdma-dev.c create mode 100644 drivers/dma/amd/ptdma/ptdma-dmaengine.c create mode 100644 drivers/dma/amd/ptdma/ptdma-pci.c create mode 100644 drivers/dma/amd/ptdma/ptdma.h create mode 100644 drivers/dma/amd/qdma/Makefile create mode 100644 drivers/dma/amd/qdma/qdma-comm-regs.c create mode 100644 drivers/dma/amd/qdma/qdma.c create mode 100644 drivers/dma/amd/qdma/qdma.h (limited to 'drivers/dma/amd') diff --git a/drivers/dma/amd/Kconfig b/drivers/dma/amd/Kconfig new file mode 100644 index 000000000..00d874872 --- /dev/null +++ b/drivers/dma/amd/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only +# + +config AMD_AE4DMA + tristate "AMD AE4DMA Engine" + depends on (X86_64 || COMPILE_TEST) && PCI + depends on AMD_PTDMA + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD AE4DMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + +config AMD_PTDMA + tristate "AMD PassThru DMA Engine" + depends on X86_64 && PCI + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD PTDMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + +config AMD_QDMA + tristate "AMD Queue-based DMA" + depends on HAS_IOMEM + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select REGMAP_MMIO + help + Enable support for the AMD Queue-based DMA subsystem. The primary + mechanism to transfer data using the QDMA is for the QDMA engine to + operate on instructions (descriptors) provided by the host operating + system. Using the descriptors, the QDMA can move data in either the + Host to Card (H2C) direction or the Card to Host (C2H) direction. diff --git a/drivers/dma/amd/Makefile b/drivers/dma/amd/Makefile new file mode 100644 index 000000000..11278c063 --- /dev/null +++ b/drivers/dma/amd/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma/ +obj-$(CONFIG_AMD_PTDMA) += ptdma/ +obj-$(CONFIG_AMD_QDMA) += qdma/ diff --git a/drivers/dma/amd/ae4dma/Makefile b/drivers/dma/amd/ae4dma/Makefile new file mode 100644 index 000000000..e918f85a8 --- /dev/null +++ b/drivers/dma/amd/ae4dma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# AMD AE4DMA driver +# + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma.o + +ae4dma-objs := ae4dma-dev.o + +ae4dma-$(CONFIG_PCI) += ae4dma-pci.o diff --git a/drivers/dma/amd/ae4dma/ae4dma-dev.c b/drivers/dma/amd/ae4dma/ae4dma-dev.c new file mode 100644 index 000000000..8de3bef41 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-dev.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar + */ + +#include "ae4dma.h" + +static unsigned int max_hw_q = 1; +module_param(max_hw_q, uint, 0444); +MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)"); + +static void ae4_pending_work(struct work_struct *work) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct pt_cmd *cmd; + u32 cridx; + + for (;;) { + wait_event_interruptible(ae4cmd_q->q_w, + ((atomic64_read(&ae4cmd_q->done_cnt)) < + atomic64_read(&ae4cmd_q->intr_cnt))); + + atomic64_inc(&ae4cmd_q->done_cnt); + + mutex_lock(&ae4cmd_q->cmd_lock); + cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) { + cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + + ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx); + cmd->pt_cmd_callback(cmd->data, cmd->ret); + + ae4cmd_q->q_cmd_count--; + ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN; + + complete_all(&ae4cmd_q->cmp); + } + mutex_unlock(&ae4cmd_q->cmd_lock); + } +} + +static irqreturn_t ae4_core_irq_handler(int irq, void *data) +{ + struct ae4_cmd_queue *ae4cmd_q = data; + struct pt_cmd_queue *cmd_q; + struct pt_device *pt; + u32 status; + + cmd_q = &ae4cmd_q->cmd_q; + pt = cmd_q->pt; + + pt->total_interrupts++; + atomic64_inc(&ae4cmd_q->intr_cnt); + + status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF); + if (status & BIT(0)) { + status &= GENMASK(31, 1); + writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF); + } + + wake_up(&ae4cmd_q->q_w); + + return IRQ_HANDLED; +} + +void ae4_destroy_work(struct ae4_device *ae4) +{ + struct ae4_cmd_queue *ae4cmd_q; + int i; + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + if (!ae4cmd_q->pws) + break; + + cancel_delayed_work_sync(&ae4cmd_q->p_work); + destroy_workqueue(ae4cmd_q->pws); + } +} + +int ae4_core_init(struct ae4_device *ae4) +{ + struct pt_device *pt = &ae4->pt; + struct ae4_cmd_queue *ae4cmd_q; + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q; + int i, ret = 0; + + writel(max_hw_q, pt->io_regs); + + for (i = 0; i < max_hw_q; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + ae4cmd_q->id = ae4->cmd_q_count; + ae4->cmd_q_count++; + + cmd_q = &ae4cmd_q->cmd_q; + cmd_q->pt = pt; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0, + dev_name(pt->dev), ae4cmd_q); + if (ret) + return ret; + + cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc)); + + cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) + return -ENOMEM; + } + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + cmd_q = &ae4cmd_q->cmd_q; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + /* Update the device registers with queue information. */ + writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF); + writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF); + + INIT_LIST_HEAD(&ae4cmd_q->cmd); + init_waitqueue_head(&ae4cmd_q->q_w); + + ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id); + if (!ae4cmd_q->pws) { + ae4_destroy_work(ae4); + return -ENOMEM; + } + INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work); + queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work, usecs_to_jiffies(100)); + + init_completion(&ae4cmd_q->cmp); + } + + ret = pt_dmaengine_register(pt); + if (ret) + ae4_destroy_work(ae4); + else + ptdma_debugfs_setup(pt); + + return ret; +} diff --git a/drivers/dma/amd/ae4dma/ae4dma-pci.c b/drivers/dma/amd/ae4dma/ae4dma-pci.c new file mode 100644 index 000000000..2c63907db --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-pci.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar + */ + +#include "ae4dma.h" + +static int ae4_get_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + int i, v, ret; + + pdev = to_pci_dev(dev); + + for (v = 0; v < ARRAY_SIZE(ae4_msix->msix_entry); v++) + ae4_msix->msix_entry[v].entry = v; + + ret = pci_alloc_irq_vectors(pdev, v, v, PCI_IRQ_MSIX); + if (ret != v) { + if (ret > 0) + pci_free_irq_vectors(pdev); + + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(dev, "could not enable MSI (%d)\n", ret); + return ret; + } + + ret = pci_irq_vector(pdev, 0); + if (ret < 0) { + pci_free_irq_vectors(pdev); + return ret; + } + + for (i = 0; i < MAX_AE4_HW_QUEUES; i++) + ae4->ae4_irq[i] = ret; + + } else { + ae4_msix->msix_count = ret; + for (i = 0; i < ae4_msix->msix_count; i++) + ae4->ae4_irq[i] = pci_irq_vector(pdev, i); + } + + return ret; +} + +static void ae4_free_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + + pdev = to_pci_dev(dev); + + if (ae4_msix && (ae4_msix->msix_count || ae4->ae4_irq[MAX_AE4_HW_QUEUES - 1])) + pci_free_irq_vectors(pdev); +} + +static void ae4_deinit(struct ae4_device *ae4) +{ + ae4_free_irqs(ae4); +} + +static int ae4_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct ae4_device *ae4; + struct pt_device *pt; + int bar_mask; + int ret = 0; + + ae4 = devm_kzalloc(dev, sizeof(*ae4), GFP_KERNEL); + if (!ae4) + return -ENOMEM; + + ae4->ae4_msix = devm_kzalloc(dev, sizeof(struct ae4_msix), GFP_KERNEL); + if (!ae4->ae4_msix) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) + goto ae4_error; + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ae4dma"); + if (ret) + goto ae4_error; + + pt = &ae4->pt; + pt->dev = dev; + pt->ver = AE4_DMA_VERSION; + + pt->io_regs = pcim_iomap_table(pdev)[0]; + if (!pt->io_regs) { + ret = -ENOMEM; + goto ae4_error; + } + + ret = ae4_get_irqs(ae4); + if (ret < 0) + goto ae4_error; + + pci_set_master(pdev); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + + dev_set_drvdata(dev, ae4); + + ret = ae4_core_init(ae4); + if (ret) + goto ae4_error; + + return 0; + +ae4_error: + ae4_deinit(ae4); + + return ret; +} + +static void ae4_pci_remove(struct pci_dev *pdev) +{ + struct ae4_device *ae4 = dev_get_drvdata(&pdev->dev); + + ae4_destroy_work(ae4); + ae4_deinit(ae4); +} + +static const struct pci_device_id ae4_pci_table[] = { + { PCI_VDEVICE(AMD, 0x149B), }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ae4_pci_table); + +static struct pci_driver ae4_pci_driver = { + .name = "ae4dma", + .id_table = ae4_pci_table, + .probe = ae4_pci_probe, + .remove = ae4_pci_remove, +}; + +module_pci_driver(ae4_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD AE4DMA driver"); diff --git a/drivers/dma/amd/ae4dma/ae4dma.h b/drivers/dma/amd/ae4dma/ae4dma.h new file mode 100644 index 000000000..57f604872 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar + */ +#ifndef __AE4DMA_H__ +#define __AE4DMA_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../ptdma/ptdma.h" +#include "../../virt-dma.h" + +#define MAX_AE4_HW_QUEUES 16 + +#define AE4_DESC_COMPLETED 0x03 + +#define AE4_MAX_IDX_OFF 0x08 +#define AE4_RD_IDX_OFF 0x0c +#define AE4_WR_IDX_OFF 0x10 +#define AE4_INTR_STS_OFF 0x14 +#define AE4_Q_BASE_L_OFF 0x18 +#define AE4_Q_BASE_H_OFF 0x1c +#define AE4_Q_SZ 0x20 + +#define AE4_DMA_VERSION 4 +#define CMD_AE4_DESC_DW0_VAL 2 + +#define AE4_TIME_OUT 5000 + +struct ae4_msix { + int msix_count; + struct msix_entry msix_entry[MAX_AE4_HW_QUEUES]; +}; + +struct ae4_cmd_queue { + struct ae4_device *ae4; + struct pt_cmd_queue cmd_q; + struct list_head cmd; + /* protect command operations */ + struct mutex cmd_lock; + struct delayed_work p_work; + struct workqueue_struct *pws; + struct completion cmp; + wait_queue_head_t q_w; + atomic64_t intr_cnt; + atomic64_t done_cnt; + u64 q_cmd_count; + u32 dridx; + u32 tail_wi; + u32 id; +}; + +union dwou { + u32 dw0; + struct dword0 { + u8 byte0; + u8 byte1; + u16 timestamp; + } dws; +}; + +struct dword1 { + u8 status; + u8 err_code; + u16 desc_id; +}; + +struct ae4dma_desc { + union dwou dwouv; + struct dword1 dw1; + u32 length; + u32 rsvd; + u32 src_hi; + u32 src_lo; + u32 dst_hi; + u32 dst_lo; +}; + +struct ae4_device { + struct pt_device pt; + struct ae4_msix *ae4_msix; + struct ae4_cmd_queue ae4cmd_q[MAX_AE4_HW_QUEUES]; + unsigned int ae4_irq[MAX_AE4_HW_QUEUES]; + unsigned int cmd_q_count; +}; + +int ae4_core_init(struct ae4_device *ae4); +void ae4_destroy_work(struct ae4_device *ae4); +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx); +#endif diff --git a/drivers/dma/amd/ptdma/Makefile b/drivers/dma/amd/ptdma/Makefile new file mode 100644 index 000000000..ce5410268 --- /dev/null +++ b/drivers/dma/amd/ptdma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# AMD Passthru DMA driver +# + +obj-$(CONFIG_AMD_PTDMA) += ptdma.o + +ptdma-objs := ptdma-dev.o ptdma-dmaengine.o ptdma-debugfs.o + +ptdma-$(CONFIG_PCI) += ptdma-pci.o diff --git a/drivers/dma/amd/ptdma/ptdma-debugfs.c b/drivers/dma/amd/ptdma/ptdma-debugfs.c new file mode 100644 index 000000000..c7c90bbf6 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-debugfs.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthrough DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Gary R Hook + */ + +#include +#include + +#include "ptdma.h" +#include "../ae4dma/ae4dma.h" + +/* DebugFS helpers */ +#define RI_VERSION_NUM 0x0000003F + +#define RI_NUM_VQM 0x00078000 +#define RI_NVQM_SHIFT 15 + +static int pt_debugfs_info_show(struct seq_file *s, void *p) +{ + struct pt_device *pt = s->private; + struct ae4_device *ae4; + unsigned int regval; + + seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + seq_printf(s, " # Queues: %d\n", ae4->cmd_q_count); + seq_printf(s, " # Cmds per queue: %d\n", CMD_Q_LEN); + } else { + seq_printf(s, " # Queues: %d\n", 1); + seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + } + + regval = ioread32(pt->io_regs + CMD_PT_VERSION); + + seq_printf(s, " Version: %d\n", regval & RI_VERSION_NUM); + seq_puts(s, " Engines:"); + seq_puts(s, "\n"); + seq_printf(s, " Queues: %d\n", (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); + + return 0; +} + +/* + * Return a formatted buffer containing the current + * statistics of queue for PTDMA + */ +static int pt_debugfs_stats_show(struct seq_file *s, void *p) +{ + struct pt_device *pt = s->private; + + seq_printf(s, "Total Interrupts Handled: %ld\n", pt->total_interrupts); + + return 0; +} + +static int pt_debugfs_queue_show(struct seq_file *s, void *p) +{ + struct pt_cmd_queue *cmd_q = s->private; + struct pt_device *pt; + unsigned int regval; + + if (!cmd_q) + return 0; + + seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); + + pt = cmd_q->pt; + if (pt->ver == AE4_DMA_VERSION) { + regval = readl(cmd_q->reg_control + 0x4); + seq_printf(s, " Enabled Interrupts:: status 0x%x\n", regval); + } else { + regval = ioread32(cmd_q->reg_control + 0x000C); + + seq_puts(s, " Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + seq_puts(s, " EMPTY"); + if (regval & INT_QUEUE_STOPPED) + seq_puts(s, " STOPPED"); + if (regval & INT_ERROR) + seq_puts(s, " ERROR"); + if (regval & INT_COMPLETION) + seq_puts(s, " COMPLETION"); + seq_puts(s, "\n"); + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_info); +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_queue); +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); + +void ptdma_debugfs_setup(struct pt_device *pt) +{ + struct dentry *debugfs_q_instance; + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + char name[30]; + int i; + + if (!debugfs_initialized()) + return; + + debugfs_create_file("info", 0400, pt->dma_dev.dbg_dev_root, pt, + &pt_debugfs_info_fops); + + debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, + &pt_debugfs_stats_fops); + + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + cmd_q = &ae4cmd_q->cmd_q; + + memset(name, 0, sizeof(name)); + snprintf(name, 29, "q%d", ae4cmd_q->id); + + debugfs_q_instance = + debugfs_create_dir(name, pt->dma_dev.dbg_dev_root); + + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } + } else { + debugfs_q_instance = + debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); + cmd_q = &pt->cmd_q; + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } +} +EXPORT_SYMBOL_GPL(ptdma_debugfs_setup); diff --git a/drivers/dma/amd/ptdma/ptdma-dev.c b/drivers/dma/amd/ptdma/ptdma-dev.c new file mode 100644 index 000000000..a2bf13ff1 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-dev.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ptdma.h" + +/* Human-readable error strings */ +static char *pt_error_codes[] = { + "", + "ERR 01: ILLEGAL_ENGINE", + "ERR 03: ILLEGAL_FUNCTION_TYPE", + "ERR 04: ILLEGAL_FUNCTION_MODE", + "ERR 06: ILLEGAL_FUNCTION_SIZE", + "ERR 08: ILLEGAL_FUNCTION_RSVD", + "ERR 09: ILLEGAL_BUFFER_LENGTH", + "ERR 10: VLSB_FAULT", + "ERR 11: ILLEGAL_MEM_ADDR", + "ERR 12: ILLEGAL_MEM_SEL", + "ERR 13: ILLEGAL_CONTEXT_ID", + "ERR 15: 0xF Reserved", + "ERR 18: CMD_TIMEOUT", + "ERR 19: IDMA0_AXI_SLVERR", + "ERR 20: IDMA0_AXI_DECERR", + "ERR 21: 0x15 Reserved", + "ERR 22: IDMA1_AXI_SLAVE_FAULT", + "ERR 23: IDMA1_AIXI_DECERR", + "ERR 24: 0x18 Reserved", + "ERR 27: 0x1B Reserved", + "ERR 38: ODMA0_AXI_SLVERR", + "ERR 39: ODMA0_AXI_DECERR", + "ERR 40: 0x28 Reserved", + "ERR 41: ODMA1_AXI_SLVERR", + "ERR 42: ODMA1_AXI_DECERR", + "ERR 43: LSB_PARITY_ERR", +}; + +static void pt_log_error(struct pt_device *d, int e) +{ + dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); +} + +void pt_start_queue(struct pt_cmd_queue *cmd_q) +{ + /* Turn on the run bit */ + iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); +} + +void pt_stop_queue(struct pt_cmd_queue *cmd_q) +{ + /* Turn off the run bit */ + iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); +} + +static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); + u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; + u32 tail; + unsigned long flags; + + if (soc) { + desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); + desc->dw0 &= ~DWORD0_SOC; + } + spin_lock_irqsave(&cmd_q->q_lock, flags); + + /* Copy 32-byte command descriptor to hw queue. */ + memcpy(q_desc, desc, 32); + cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; + + /* The data used by this command must be flushed to memory */ + wmb(); + + /* Write the new tail address back to the queue register */ + tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + iowrite32(tail, cmd_q->reg_control + 0x0004); + + /* Turn the queue back on using our cached control register */ + pt_start_queue(cmd_q); + spin_unlock_irqrestore(&cmd_q->q_lock, flags); + + return 0; +} + +int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ptdma_desc desc; + struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q); + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dw0 = CMD_DESC_DW0_VAL; + desc.length = pt_engine->src_len; + desc.src_lo = lower_32_bits(pt_engine->src_dma); + desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); + desc.dst_lo = lower_32_bits(pt_engine->dst_dma); + desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); + + if (cmd_q->int_en) + pt_core_enable_queue_interrupts(pt); + else + pt_core_disable_queue_interrupts(pt); + + return pt_core_execute_cmd(&desc, cmd_q); +} + +static void pt_do_cmd_complete(unsigned long data) +{ + struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; + struct pt_cmd *cmd = tdata->cmd; + struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; + u32 tail; + + if (cmd_q->cmd_error) { + /* + * Log the error and flush the queue by + * moving the head pointer + */ + tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + pt_log_error(cmd_q->pt, cmd_q->cmd_error); + iowrite32(tail, cmd_q->reg_control + 0x0008); + } + + cmd->pt_cmd_callback(cmd->data, cmd->ret); +} + +void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + u32 status; + + status = ioread32(cmd_q->reg_control + 0x0010); + if (status) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); + cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); + + /* On error, only save the first error value */ + if ((status & INT_ERROR) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + /* Acknowledge the completion */ + iowrite32(status, cmd_q->reg_control + 0x0010); + pt_do_cmd_complete((ulong)&pt->tdata); + } +} + +static irqreturn_t pt_core_irq_handler(int irq, void *data) +{ + struct pt_device *pt = data; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + + pt_core_disable_queue_interrupts(pt); + pt->total_interrupts++; + pt_check_status_trans(pt, cmd_q); + pt_core_enable_queue_interrupts(pt); + return IRQ_HANDLED; +} + +int pt_core_init(struct pt_device *pt) +{ + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + u32 dma_addr_lo, dma_addr_hi; + struct device *dev = pt->dev; + struct dma_pool *dma_pool; + int ret; + + /* Allocate a dma pool for the queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); + + dma_pool = dma_pool_create(dma_pool_name, dev, + PT_DMAPOOL_MAX_SIZE, + PT_DMAPOOL_ALIGN, 0); + if (!dma_pool) + return -ENOMEM; + + /* ptdma core initialisation */ + iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); + iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); + iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); + iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); + iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); + + cmd_q->pt = pt; + cmd_q->dma_pool = dma_pool; + spin_lock_init(&cmd_q->q_lock); + + /* Page alignment satisfies our needs for N <= 128 */ + cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); + cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, + &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) { + dev_err(dev, "unable to allocate command queue\n"); + ret = -ENOMEM; + goto e_destroy_pool; + } + + cmd_q->qidx = 0; + + /* Preset some register values */ + cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; + + /* Turn off the queues and disable interrupts until ready */ + pt_core_disable_queue_interrupts(pt); + + cmd_q->qcontrol = 0; /* Start with nothing */ + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + ioread32(cmd_q->reg_control + 0x0104); + ioread32(cmd_q->reg_control + 0x0100); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + + /* Request an irq */ + ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_free_dma; + } + + /* Update the device registers with queue information. */ + cmd_q->qcontrol &= ~CMD_Q_SIZE; + cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); + iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); + iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); + + dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); + cmd_q->qcontrol |= (dma_addr_hi << 16); + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + pt_core_enable_queue_interrupts(pt); + + /* Register the DMA engine support */ + ret = pt_dmaengine_register(pt); + if (ret) + goto e_free_irq; + + /* Set up debugfs entries */ + ptdma_debugfs_setup(pt); + + return 0; + +e_free_irq: + free_irq(pt->pt_irq, pt); + +e_free_dma: + dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); + +e_destroy_pool: + dma_pool_destroy(pt->cmd_q.dma_pool); + + return ret; +} + +void pt_core_destroy(struct pt_device *pt) +{ + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + struct pt_cmd *cmd; + + /* Unregister the DMA engine */ + pt_dmaengine_unregister(pt); + + /* Disable and clear interrupts */ + pt_core_disable_queue_interrupts(pt); + + /* Turn off the run bit */ + pt_stop_queue(cmd_q); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + ioread32(cmd_q->reg_control + 0x0104); + ioread32(cmd_q->reg_control + 0x0100); + + free_irq(pt->pt_irq, pt); + + dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, + cmd_q->qbase_dma); + + /* Flush the cmd queue */ + while (!list_empty(&pt->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + cmd->pt_cmd_callback(cmd->data, -ENODEV); + } +} diff --git a/drivers/dma/amd/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c new file mode 100644 index 000000000..628c49ce5 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -0,0 +1,659 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthrough DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Gary R Hook + */ + +#include +#include "ptdma.h" +#include "../ae4dma/ae4dma.h" +#include "../../dmaengine.h" + +static char *ae4_error_codes[] = { + "", + "ERR 01: INVALID HEADER DW0", + "ERR 02: INVALID STATUS", + "ERR 03: INVALID LENGTH - 4 BYTE ALIGNMENT", + "ERR 04: INVALID SRC ADDR - 4 BYTE ALIGNMENT", + "ERR 05: INVALID DST ADDR - 4 BYTE ALIGNMENT", + "ERR 06: INVALID ALIGNMENT", + "ERR 07: INVALID DESCRIPTOR", +}; + +static void ae4_log_error(struct pt_device *d, int e) +{ + /* ERR 01 - 07 represents Invalid AE4 errors */ + if (e <= 7) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", ae4_error_codes[e], e); + /* ERR 08 - 15 represents Invalid Descriptor errors */ + else if (e > 7 && e <= 15) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "INVALID DESCRIPTOR", e); + /* ERR 16 - 31 represents Firmware errors */ + else if (e > 15 && e <= 31) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FIRMWARE ERROR", e); + /* ERR 32 - 63 represents Fatal errors */ + else if (e > 31 && e <= 63) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FATAL ERROR", e); + /* ERR 64 - 255 represents PTE errors */ + else if (e > 63 && e <= 255) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "PTE ERROR", e); + else + dev_info(d->dev, "Unknown AE4DMA error"); +} + +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx) +{ + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct ae4dma_desc desc; + u8 status; + + memcpy(&desc, &cmd_q->qbase[idx], sizeof(struct ae4dma_desc)); + status = desc.dw1.status; + if (status && status != AE4_DESC_COMPLETED) { + cmd_q->cmd_error = desc.dw1.err_code; + if (cmd_q->cmd_error) + ae4_log_error(cmd_q->pt, cmd_q->cmd_error); + } +} +EXPORT_SYMBOL_GPL(ae4_check_status_error); + +static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) +{ + return container_of(dma_chan, struct pt_dma_chan, vc.chan); +} + +static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct pt_dma_desc, vd); +} + +static void pt_free_chan_resources(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + + vchan_free_chan_resources(&chan->vc); +} + +static void pt_synchronize(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + + vchan_synchronize(&chan->vc); +} + +static void pt_do_cleanup(struct virt_dma_desc *vd) +{ + struct pt_dma_desc *desc = to_pt_desc(vd); + struct pt_device *pt = desc->pt; + + kmem_cache_free(pt->dma_desc_cache, desc); +} + +static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma_chan *chan) +{ + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + cmd_q = &ae4cmd_q->cmd_q; + } else { + cmd_q = &pt->cmd_q; + } + + return cmd_q; +} + +static int ae4_core_execute_cmd(struct ae4dma_desc *desc, struct ae4_cmd_queue *ae4cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dwouv.dw0); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + + if (soc) { + desc->dwouv.dw0 |= FIELD_PREP(DWORD0_IOC, desc->dwouv.dw0); + desc->dwouv.dw0 &= ~DWORD0_SOC; + } + + mutex_lock(&ae4cmd_q->cmd_lock); + memcpy(&cmd_q->qbase[ae4cmd_q->tail_wi], desc, sizeof(struct ae4dma_desc)); + ae4cmd_q->q_cmd_count++; + ae4cmd_q->tail_wi = (ae4cmd_q->tail_wi + 1) % CMD_Q_LEN; + writel(ae4cmd_q->tail_wi, cmd_q->reg_control + AE4_WR_IDX_OFF); + mutex_unlock(&ae4cmd_q->cmd_lock); + + wake_up(&ae4cmd_q->q_w); + + return 0; +} + +static int pt_core_perform_passthru_ae4(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + struct ae4dma_desc desc; + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dwouv.dws.byte0 = CMD_AE4_DESC_DW0_VAL; + + desc.dw1.status = 0; + desc.dw1.err_code = 0; + desc.dw1.desc_id = 0; + + desc.length = pt_engine->src_len; + + desc.src_lo = upper_32_bits(pt_engine->src_dma); + desc.src_hi = lower_32_bits(pt_engine->src_dma); + desc.dst_lo = upper_32_bits(pt_engine->dst_dma); + desc.dst_hi = lower_32_bits(pt_engine->dst_dma); + + return ae4_core_execute_cmd(&desc, ae4cmd_q); +} + +static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) +{ + struct pt_passthru_engine *pt_engine; + struct pt_device *pt; + struct pt_cmd *pt_cmd; + struct pt_cmd_queue *cmd_q; + + desc->issued_to_hw = 1; + + pt_cmd = &desc->pt_cmd; + pt = pt_cmd->pt; + + cmd_q = pt_get_cmd_queue(pt, chan); + + pt_engine = &pt_cmd->passthru; + + pt->tdata.cmd = pt_cmd; + + /* Execute the command */ + if (pt->ver == AE4_DMA_VERSION) + pt_cmd->ret = pt_core_perform_passthru_ae4(cmd_q, pt_engine); + else + pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); + + return 0; +} + +static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) +{ + /* Get the next DMA descriptor on the active list */ + struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); + + return vd ? to_pt_desc(vd) : NULL; +} + +static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, + struct pt_dma_desc *desc) +{ + struct dma_async_tx_descriptor *tx_desc; + struct virt_dma_desc *vd; + struct pt_device *pt; + unsigned long flags; + + pt = chan->pt; + /* Loop over descriptors until one is found with commands */ + do { + if (desc) { + if (!desc->issued_to_hw) { + /* No errors, keep going */ + if (desc->status != DMA_ERROR) + return desc; + } + + tx_desc = &desc->vd.tx; + vd = &desc->vd; + } else { + tx_desc = NULL; + } + + spin_lock_irqsave(&chan->vc.lock, flags); + + if (pt->ver != AE4_DMA_VERSION && desc) { + if (desc->status != DMA_COMPLETE) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + list_del(&desc->vd.node); + } else { + /* Don't handle it twice */ + tx_desc = NULL; + } + } + + desc = pt_next_dma_desc(chan); + + spin_unlock_irqrestore(&chan->vc.lock, flags); + + if (pt->ver != AE4_DMA_VERSION && tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + dma_run_dependencies(tx_desc); + vchan_vdesc_fini(vd); + } + } while (desc); + + return NULL; +} + +static inline bool ae4_core_queue_full(struct pt_cmd_queue *cmd_q) +{ + u32 front_wi = readl(cmd_q->reg_control + AE4_WR_IDX_OFF); + u32 rear_ri = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + + if (((MAX_CMD_QLEN + front_wi - rear_ri) % MAX_CMD_QLEN) >= (MAX_CMD_QLEN - 1)) + return true; + + return false; +} + +static void pt_cmd_callback(void *data, int err) +{ + struct pt_dma_desc *desc = data; + struct ae4_cmd_queue *ae4cmd_q; + struct dma_chan *dma_chan; + struct pt_dma_chan *chan; + struct ae4_device *ae4; + struct pt_device *pt; + int ret; + + if (err == -EINPROGRESS) + return; + + dma_chan = desc->vd.tx.chan; + chan = to_pt_chan(dma_chan); + pt = chan->pt; + + if (err) + desc->status = DMA_ERROR; + + while (true) { + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + + if (ae4cmd_q->q_cmd_count >= (CMD_Q_LEN - 1) || + ae4_core_queue_full(&ae4cmd_q->cmd_q)) { + wake_up(&ae4cmd_q->q_w); + + if (wait_for_completion_timeout(&ae4cmd_q->cmp, + msecs_to_jiffies(AE4_TIME_OUT)) + == 0) { + dev_err(pt->dev, "TIMEOUT %d:\n", ae4cmd_q->id); + break; + } + + reinit_completion(&ae4cmd_q->cmp); + continue; + } + } + + /* Check for DMA descriptor completion */ + desc = pt_handle_active_desc(chan, desc); + + /* Don't submit cmd if no descriptor or DMA is paused */ + if (!desc) + break; + + ret = pt_dma_start_desc(desc, chan); + if (!ret) + break; + + desc->status = DMA_ERROR; + } +} + +static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, + unsigned long flags) +{ + struct pt_dma_desc *desc; + + desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT); + if (!desc) + return NULL; + + vchan_tx_prep(&chan->vc, &desc->vd, flags); + + desc->pt = chan->pt; + desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); + desc->issued_to_hw = 0; + desc->status = DMA_IN_PROGRESS; + + return desc; +} + +static void pt_cmd_callback_work(void *data, int err) +{ + struct dma_async_tx_descriptor *tx_desc; + struct pt_dma_desc *desc = data; + struct dma_chan *dma_chan; + struct virt_dma_desc *vd; + struct pt_dma_chan *chan; + unsigned long flags; + + if (!desc) + return; + + dma_chan = desc->vd.tx.chan; + chan = to_pt_chan(dma_chan); + + if (err == -EINPROGRESS) + return; + + tx_desc = &desc->vd.tx; + vd = &desc->vd; + + if (err) + desc->status = DMA_ERROR; + + spin_lock_irqsave(&chan->vc.lock, flags); + if (desc->status != DMA_COMPLETE) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + } else { + tx_desc = NULL; + } + spin_unlock_irqrestore(&chan->vc.lock, flags); + + if (tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + dma_run_dependencies(tx_desc); + list_del(&desc->vd.node); + vchan_vdesc_fini(vd); + } +} + +static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, + dma_addr_t dst, + dma_addr_t src, + unsigned int len, + unsigned long flags) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_passthru_engine *pt_engine; + struct pt_device *pt = chan->pt; + struct ae4_cmd_queue *ae4cmd_q; + struct pt_dma_desc *desc; + struct ae4_device *ae4; + struct pt_cmd *pt_cmd; + + desc = pt_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + pt_cmd = &desc->pt_cmd; + pt_cmd->pt = pt; + pt_engine = &pt_cmd->passthru; + pt_cmd->engine = PT_ENGINE_PASSTHRU; + pt_engine->src_dma = src; + pt_engine->dst_dma = dst; + pt_engine->src_len = len; + pt_cmd->pt_cmd_callback = pt_cmd_callback; + pt_cmd->data = desc; + + desc->len = len; + + if (pt->ver == AE4_DMA_VERSION) { + pt_cmd->pt_cmd_callback = pt_cmd_callback_work; + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + mutex_lock(&ae4cmd_q->cmd_lock); + list_add_tail(&pt_cmd->entry, &ae4cmd_q->cmd); + mutex_unlock(&ae4cmd_q->cmd_lock); + } + + return desc; +} + +static struct dma_async_tx_descriptor * +pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct pt_dma_desc *desc; + + desc = pt_create_desc(dma_chan, dst, src, len, flags); + if (!desc) + return NULL; + + return &desc->vd.tx; +} + +static struct dma_async_tx_descriptor * +pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc; + + desc = pt_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + return &desc->vd.tx; +} + +static void pt_issue_pending(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc; + struct pt_device *pt; + unsigned long flags; + bool engine_is_idle = true; + + pt = chan->pt; + + spin_lock_irqsave(&chan->vc.lock, flags); + + desc = pt_next_dma_desc(chan); + if (desc && pt->ver != AE4_DMA_VERSION) + engine_is_idle = false; + + vchan_issue_pending(&chan->vc); + + desc = pt_next_dma_desc(chan); + + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* If there was nothing active, start processing */ + if (engine_is_idle && desc) + pt_cmd_callback(desc, 0); +} + +static void pt_check_status_trans_ae4(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + int i; + + for (i = 0; i < CMD_Q_LEN; i++) + ae4_check_status_error(ae4cmd_q, i); +} + +static enum dma_status +pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct pt_dma_chan *chan = to_pt_chan(c); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + + cmd_q = pt_get_cmd_queue(pt, chan); + + if (pt->ver == AE4_DMA_VERSION) + pt_check_status_trans_ae4(pt, cmd_q); + else + pt_check_status_trans(pt, cmd_q); + + return dma_cookie_status(c, cookie, txstate); +} + +static int pt_pause(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_stop_queue(cmd_q); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + return 0; +} + +static int pt_resume(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc = NULL; + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_start_queue(cmd_q); + desc = pt_next_dma_desc(chan); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* If there was something active, re-start */ + if (desc) + pt_cmd_callback(desc, 0); + + return 0; +} + +static int pt_terminate_all(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + unsigned long flags; + LIST_HEAD(head); + + cmd_q = pt_get_cmd_queue(pt, chan); + if (pt->ver == AE4_DMA_VERSION) + pt_stop_queue(cmd_q); + else + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + + spin_lock_irqsave(&chan->vc.lock, flags); + vchan_get_all_descriptors(&chan->vc, &head); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + vchan_dma_desc_free_list(&chan->vc, &head); + vchan_free_chan_resources(&chan->vc); + + return 0; +} + +int pt_dmaengine_register(struct pt_device *pt) +{ + struct dma_device *dma_dev = &pt->dma_dev; + struct ae4_cmd_queue *ae4cmd_q = NULL; + struct ae4_device *ae4 = NULL; + struct pt_dma_chan *chan; + char *desc_cache_name; + int ret, i; + + if (pt->ver == AE4_DMA_VERSION) + ae4 = container_of(pt, struct ae4_device, pt); + + if (ae4) + pt->pt_dma_chan = devm_kcalloc(pt->dev, ae4->cmd_q_count, + sizeof(*pt->pt_dma_chan), GFP_KERNEL); + else + pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), + GFP_KERNEL); + + if (!pt->pt_dma_chan) + return -ENOMEM; + + desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, + "%s-dmaengine-desc-cache", + dev_name(pt->dev)); + if (!desc_cache_name) + return -ENOMEM; + + pt->dma_desc_cache = kmem_cache_create(desc_cache_name, + sizeof(struct pt_dma_desc), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!pt->dma_desc_cache) + return -ENOMEM; + + dma_dev->dev = pt->dev; + dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; + dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; + dma_dev->directions = DMA_MEM_TO_MEM; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); + + /* + * PTDMA is intended to be used with the AMD NTB devices, hence + * marking it as DMA_PRIVATE. + */ + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + + INIT_LIST_HEAD(&dma_dev->channels); + + /* Set base and prep routines */ + dma_dev->device_free_chan_resources = pt_free_chan_resources; + dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; + dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; + dma_dev->device_issue_pending = pt_issue_pending; + dma_dev->device_tx_status = pt_tx_status; + dma_dev->device_pause = pt_pause; + dma_dev->device_resume = pt_resume; + dma_dev->device_terminate_all = pt_terminate_all; + dma_dev->device_synchronize = pt_synchronize; + + if (ae4) { + for (i = 0; i < ae4->cmd_q_count; i++) { + chan = pt->pt_dma_chan + i; + ae4cmd_q = &ae4->ae4cmd_q[i]; + chan->id = ae4cmd_q->id; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } + } else { + chan = pt->pt_dma_chan; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } + + ret = dma_async_device_register(dma_dev); + if (ret) + goto err_reg; + + return 0; + +err_reg: + kmem_cache_destroy(pt->dma_desc_cache); + + return ret; +} +EXPORT_SYMBOL_GPL(pt_dmaengine_register); + +void pt_dmaengine_unregister(struct pt_device *pt) +{ + struct dma_device *dma_dev = &pt->dma_dev; + + dma_async_device_unregister(dma_dev); + + kmem_cache_destroy(pt->dma_desc_cache); +} diff --git a/drivers/dma/amd/ptdma/ptdma-pci.c b/drivers/dma/amd/ptdma/ptdma-pci.c new file mode 100644 index 000000000..22739ff0c --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-pci.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ptdma.h" + +struct pt_msix { + int msix_count; + struct msix_entry msix_entry; +}; + +/* + * pt_alloc_struct - allocate and initialize the pt_device struct + * + * @dev: device struct of the PTDMA + */ +static struct pt_device *pt_alloc_struct(struct device *dev) +{ + struct pt_device *pt; + + pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL); + + if (!pt) + return NULL; + pt->dev = dev; + + INIT_LIST_HEAD(&pt->cmd); + + return pt; +} + +static int pt_get_msix_irqs(struct pt_device *pt) +{ + struct pt_msix *pt_msix = pt->pt_msix; + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pt_msix->msix_entry.entry = 0; + + ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1); + if (ret < 0) + return ret; + + pt_msix->msix_count = ret; + + pt->pt_irq = pt_msix->msix_entry.vector; + + return 0; +} + +static int pt_get_msi_irq(struct pt_device *pt) +{ + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + pt->pt_irq = pdev->irq; + + return 0; +} + +static int pt_get_irqs(struct pt_device *pt) +{ + struct device *dev = pt->dev; + int ret; + + ret = pt_get_msix_irqs(pt); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pt_get_msi_irq(pt); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_err(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void pt_free_irqs(struct pt_device *pt) +{ + struct pt_msix *pt_msix = pt->pt_msix; + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (pt_msix->msix_count) + pci_disable_msix(pdev); + else if (pt->pt_irq) + pci_disable_msi(pdev); + + pt->pt_irq = 0; +} + +static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct pt_device *pt; + struct pt_msix *pt_msix; + struct device *dev = &pdev->dev; + void __iomem * const *iomap_table; + int bar_mask; + int ret = -ENOMEM; + + pt = pt_alloc_struct(dev); + if (!pt) + goto e_err; + + pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL); + if (!pt_msix) + goto e_err; + + pt->pt_msix = pt_msix; + pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data; + if (!pt->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "pcim_enable_device failed (%d)\n", ret); + goto e_err; + } + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ptdma"); + if (ret) { + dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); + goto e_err; + } + + iomap_table = pcim_iomap_table(pdev); + if (!iomap_table) { + dev_err(dev, "pcim_iomap_table failed\n"); + ret = -ENOMEM; + goto e_err; + } + + pt->io_regs = iomap_table[pt->dev_vdata->bar]; + if (!pt->io_regs) { + dev_err(dev, "ioremap failed\n"); + ret = -ENOMEM; + goto e_err; + } + + ret = pt_get_irqs(pt); + if (ret) + goto e_err; + + pci_set_master(pdev); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", + ret); + goto e_err; + } + } + + dev_set_drvdata(dev, pt); + + if (pt->dev_vdata) + ret = pt_core_init(pt); + + if (ret) + goto e_err; + + return 0; + +e_err: + dev_err(dev, "initialization failed ret = %d\n", ret); + + return ret; +} + +static void pt_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct pt_device *pt = dev_get_drvdata(dev); + + if (!pt) + return; + + if (pt->dev_vdata) + pt_core_destroy(pt); + + pt_free_irqs(pt); +} + +static const struct pt_dev_vdata dev_vdata[] = { + { + .bar = 2, + }, +}; + +static const struct pci_device_id pt_pci_table[] = { + { PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, pt_pci_table); + +static struct pci_driver pt_pci_driver = { + .name = "ptdma", + .id_table = pt_pci_table, + .probe = pt_pci_probe, + .remove = pt_pci_remove, +}; + +module_pci_driver(pt_pci_driver); + +MODULE_AUTHOR("Sanjay R Mehta "); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD PassThru DMA driver"); diff --git a/drivers/dma/amd/ptdma/ptdma.h b/drivers/dma/amd/ptdma/ptdma.h new file mode 100644 index 000000000..ef3f55632 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma.h @@ -0,0 +1,338 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#ifndef __PT_DEV_H__ +#define __PT_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../virt-dma.h" + +#define MAX_PT_NAME_LEN 16 +#define MAX_DMAPOOL_NAME_LEN 32 + +#define MAX_HW_QUEUES 1 +#define MAX_CMD_QLEN 100 + +#define PT_ENGINE_PASSTHRU 5 + +/* Register Mappings */ +#define IRQ_MASK_REG 0x040 +#define IRQ_STATUS_REG 0x200 + +#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) + +#define CMD_QUEUE_PRIO_OFFSET 0x00 +#define CMD_REQID_CONFIG_OFFSET 0x04 +#define CMD_TIMEOUT_OFFSET 0x08 +#define CMD_PT_VERSION 0x10 + +#define CMD_Q_CONTROL_BASE 0x0000 +#define CMD_Q_TAIL_LO_BASE 0x0004 +#define CMD_Q_HEAD_LO_BASE 0x0008 +#define CMD_Q_INT_ENABLE_BASE 0x000C +#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 + +#define CMD_Q_STATUS_BASE 0x0100 +#define CMD_Q_INT_STATUS_BASE 0x0104 +#define CMD_Q_DMA_STATUS_BASE 0x0108 +#define CMD_Q_DMA_READ_STATUS_BASE 0x010C +#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110 +#define CMD_Q_ABORT_BASE 0x0114 +#define CMD_Q_AX_CACHE_BASE 0x0118 + +#define CMD_CONFIG_OFFSET 0x1120 +#define CMD_CLK_GATE_CTL_OFFSET 0x6004 + +#define CMD_DESC_DW0_VAL 0x500012 + +/* Address offset for virtual queue registers */ +#define CMD_Q_STATUS_INCR 0x1000 + +/* Bit masks */ +#define CMD_CONFIG_REQID 0 +#define CMD_TIMEOUT_DISABLE 0 +#define CMD_CLK_DYN_GATING_DIS 0 +#define CMD_CLK_SW_GATE_MODE 0 +#define CMD_CLK_GATE_CTL 0 +#define CMD_QUEUE_PRIO GENMASK(2, 1) +#define CMD_CONFIG_VHB_EN BIT(0) +#define CMD_CLK_DYN_GATING_EN BIT(0) +#define CMD_CLK_HW_GATE_MODE BIT(0) +#define CMD_CLK_GATE_ON_DELAY BIT(12) +#define CMD_CLK_GATE_OFF_DELAY BIT(12) + +#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \ + CMD_CLK_HW_GATE_MODE | \ + CMD_CLK_GATE_ON_DELAY | \ + CMD_CLK_DYN_GATING_EN | \ + CMD_CLK_GATE_OFF_DELAY) + +#define CMD_Q_LEN 32 +#define CMD_Q_RUN BIT(0) +#define CMD_Q_HALT BIT(1) +#define CMD_Q_MEM_LOCATION BIT(2) +#define CMD_Q_SIZE_MASK GENMASK(4, 0) +#define CMD_Q_SIZE GENMASK(7, 3) +#define CMD_Q_SHIFT GENMASK(1, 0) +#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \ + CMD_Q_SIZE_MASK) +#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) +#define Q_DESC_SIZE sizeof(struct ptdma_desc) +#define Q_SIZE(n) (CMD_Q_LEN * (n)) + +#define INT_COMPLETION BIT(0) +#define INT_ERROR BIT(1) +#define INT_QUEUE_STOPPED BIT(2) +#define INT_EMPTY_QUEUE BIT(3) +#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) + +/****** Local Storage Block ******/ +#define LSB_START 0 +#define LSB_END 127 +#define LSB_COUNT (LSB_END - LSB_START + 1) + +#define PT_DMAPOOL_MAX_SIZE 64 +#define PT_DMAPOOL_ALIGN BIT(5) + +#define PT_PASSTHRU_BLOCKSIZE 512 + +struct pt_device; + +struct pt_tasklet_data { + struct completion completion; + struct pt_cmd *cmd; +}; + +/* + * struct pt_passthru_engine - pass-through operation + * without performing DMA mapping + * @mask: mask to be applied to data + * @mask_len: length in bytes of mask + * @src_dma: data to be used for this operation + * @dst_dma: data produced by this operation + * @src_len: length in bytes of data used for this operation + * + * Variables required to be set when calling pt_enqueue_cmd(): + * - bit_mod, byte_swap, src, dst, src_len + * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP + */ +struct pt_passthru_engine { + dma_addr_t mask; + u32 mask_len; /* In bytes */ + + dma_addr_t src_dma, dst_dma; + u64 src_len; /* In bytes */ +}; + +/* + * struct pt_cmd - PTDMA operation request + * @entry: list element + * @work: work element used for callbacks + * @pt: PT device to be run on + * @ret: operation return code + * @flags: cmd processing flags + * @engine: PTDMA operation to perform (passthru) + * @engine_error: PT engine return code + * @passthru: engine specific structures, refer to specific engine struct below + * @callback: operation completion callback function + * @data: parameter value to be supplied to the callback function + * + * Variables required to be set when calling pt_enqueue_cmd(): + * - engine, callback + * - See the operation structures below for what is required for each + * operation. + */ +struct pt_cmd { + struct list_head entry; + struct work_struct work; + struct pt_device *pt; + int ret; + u32 engine; + u32 engine_error; + struct pt_passthru_engine passthru; + /* Completion callback support */ + void (*pt_cmd_callback)(void *data, int err); + void *data; +}; + +struct pt_dma_desc { + struct virt_dma_desc vd; + struct pt_device *pt; + enum dma_status status; + size_t len; + bool issued_to_hw; + struct pt_cmd pt_cmd; +}; + +struct pt_dma_chan { + struct virt_dma_chan vc; + struct pt_device *pt; + u32 id; +}; + +struct pt_cmd_queue { + struct pt_device *pt; + + /* Queue dma pool */ + struct dma_pool *dma_pool; + + /* Queue base address (not necessarily aligned)*/ + struct ptdma_desc *qbase; + + /* Aligned queue start address (per requirement) */ + spinlock_t q_lock ____cacheline_aligned; + unsigned int qidx; + + unsigned int qsize; + dma_addr_t qbase_dma; + dma_addr_t qdma_tail; + + unsigned int active; + unsigned int suspended; + + /* Interrupt flag */ + bool int_en; + + /* Register addresses for queue */ + void __iomem *reg_control; + u32 qcontrol; /* Cached control register */ + + /* Status values from job */ + u32 int_status; + u32 q_status; + u32 q_int_status; + u32 cmd_error; + /* Queue Statistics */ + unsigned long total_pt_ops; +} ____cacheline_aligned; + +struct pt_device { + struct list_head entry; + + unsigned int ord; + char name[MAX_PT_NAME_LEN]; + + struct device *dev; + + /* Bus specific device information */ + struct pt_msix *pt_msix; + + struct pt_dev_vdata *dev_vdata; + + unsigned int pt_irq; + + /* I/O area used for device communication */ + void __iomem *io_regs; + + spinlock_t cmd_lock ____cacheline_aligned; + unsigned int cmd_count; + struct list_head cmd; + + /* + * The command queue. This represent the queue available on the + * PTDMA that are available for processing cmds + */ + struct pt_cmd_queue cmd_q; + + /* Support for the DMA Engine capabilities */ + struct dma_device dma_dev; + struct pt_dma_chan *pt_dma_chan; + struct kmem_cache *dma_desc_cache; + + wait_queue_head_t lsb_queue; + + /* Device Statistics */ + unsigned long total_interrupts; + + struct pt_tasklet_data tdata; + int ver; +}; + +/* + * descriptor for PTDMA commands + * 8 32-bit words: + * word 0: function; engine; control bits + * word 1: length of source data + * word 2: low 32 bits of source pointer + * word 3: upper 16 bits of source pointer; source memory type + * word 4: low 32 bits of destination pointer + * word 5: upper 16 bits of destination pointer; destination memory type + * word 6: reserved 32 bits + * word 7: reserved 32 bits + */ + +#define DWORD0_SOC BIT(0) +#define DWORD0_IOC BIT(1) + +struct dword3 { + unsigned int src_hi:16; + unsigned int src_mem:2; + unsigned int lsb_cxt_id:8; + unsigned int rsvd1:5; + unsigned int fixed:1; +}; + +struct dword5 { + unsigned int dst_hi:16; + unsigned int dst_mem:2; + unsigned int rsvd1:13; + unsigned int fixed:1; +}; + +struct ptdma_desc { + u32 dw0; + u32 length; + u32 src_lo; + struct dword3 dw3; + u32 dst_lo; + struct dword5 dw5; + __le32 rsvd1; + __le32 rsvd2; +}; + +/* Structure to hold PT device data */ +struct pt_dev_vdata { + const unsigned int bar; +}; + +int pt_dmaengine_register(struct pt_device *pt); +void pt_dmaengine_unregister(struct pt_device *pt); + +void ptdma_debugfs_setup(struct pt_device *pt); +int pt_core_init(struct pt_device *pt); +void pt_core_destroy(struct pt_device *pt); + +int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine); + +void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q); +void pt_start_queue(struct pt_cmd_queue *cmd_q); +void pt_stop_queue(struct pt_cmd_queue *cmd_q); + +static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) +{ + iowrite32(0, pt->cmd_q.reg_control + 0x000C); +} + +static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) +{ + iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); +} +#endif diff --git a/drivers/dma/amd/qdma/Makefile b/drivers/dma/amd/qdma/Makefile new file mode 100644 index 000000000..011268fef --- /dev/null +++ b/drivers/dma/amd/qdma/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AMD_QDMA) += amd-qdma.o + +amd-qdma-$(CONFIG_AMD_QDMA) := qdma.o qdma-comm-regs.o diff --git a/drivers/dma/amd/qdma/qdma-comm-regs.c b/drivers/dma/amd/qdma/qdma-comm-regs.c new file mode 100644 index 000000000..9162f9d36 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma-comm-regs.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef __QDMA_REGS_DEF_H +#define __QDMA_REGS_DEF_H + +#include "qdma.h" + +const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX] = { + [QDMA_REGO_CTXT_DATA] = QDMA_REGO(0x804, 8), + [QDMA_REGO_CTXT_CMD] = QDMA_REGO(0x844, 1), + [QDMA_REGO_CTXT_MASK] = QDMA_REGO(0x824, 8), + [QDMA_REGO_MM_H2C_CTRL] = QDMA_REGO(0x1004, 1), + [QDMA_REGO_MM_C2H_CTRL] = QDMA_REGO(0x1204, 1), + [QDMA_REGO_QUEUE_COUNT] = QDMA_REGO(0x120, 1), + [QDMA_REGO_RING_SIZE] = QDMA_REGO(0x204, 1), + [QDMA_REGO_H2C_PIDX] = QDMA_REGO(0x18004, 1), + [QDMA_REGO_C2H_PIDX] = QDMA_REGO(0x18008, 1), + [QDMA_REGO_INTR_CIDX] = QDMA_REGO(0x18000, 1), + [QDMA_REGO_FUNC_ID] = QDMA_REGO(0x12c, 1), + [QDMA_REGO_ERR_INT] = QDMA_REGO(0xb04, 1), + [QDMA_REGO_ERR_STAT] = QDMA_REGO(0x248, 1), +}; + +const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX] = { + /* QDMA_REGO_CTXT_DATA fields */ + [QDMA_REGF_IRQ_ENABLE] = QDMA_REGF(53, 53), + [QDMA_REGF_WBK_ENABLE] = QDMA_REGF(52, 52), + [QDMA_REGF_WBI_CHECK] = QDMA_REGF(34, 34), + [QDMA_REGF_IRQ_ARM] = QDMA_REGF(16, 16), + [QDMA_REGF_IRQ_VEC] = QDMA_REGF(138, 128), + [QDMA_REGF_IRQ_AGG] = QDMA_REGF(139, 139), + [QDMA_REGF_WBI_INTVL_ENABLE] = QDMA_REGF(35, 35), + [QDMA_REGF_MRKR_DISABLE] = QDMA_REGF(62, 62), + [QDMA_REGF_QUEUE_ENABLE] = QDMA_REGF(32, 32), + [QDMA_REGF_QUEUE_MODE] = QDMA_REGF(63, 63), + [QDMA_REGF_DESC_BASE] = QDMA_REGF(127, 64), + [QDMA_REGF_DESC_SIZE] = QDMA_REGF(49, 48), + [QDMA_REGF_RING_ID] = QDMA_REGF(47, 44), + [QDMA_REGF_QUEUE_BASE] = QDMA_REGF(11, 0), + [QDMA_REGF_QUEUE_MAX] = QDMA_REGF(44, 32), + [QDMA_REGF_FUNCTION_ID] = QDMA_REGF(24, 17), + [QDMA_REGF_INTR_AGG_BASE] = QDMA_REGF(66, 15), + [QDMA_REGF_INTR_VECTOR] = QDMA_REGF(11, 1), + [QDMA_REGF_INTR_SIZE] = QDMA_REGF(69, 67), + [QDMA_REGF_INTR_VALID] = QDMA_REGF(0, 0), + [QDMA_REGF_INTR_COLOR] = QDMA_REGF(14, 14), + [QDMA_REGF_INTR_FUNCTION_ID] = QDMA_REGF(125, 114), + /* QDMA_REGO_CTXT_CMD fields */ + [QDMA_REGF_CMD_INDX] = QDMA_REGF(19, 7), + [QDMA_REGF_CMD_CMD] = QDMA_REGF(6, 5), + [QDMA_REGF_CMD_TYPE] = QDMA_REGF(4, 1), + [QDMA_REGF_CMD_BUSY] = QDMA_REGF(0, 0), + /* QDMA_REGO_QUEUE_COUNT fields */ + [QDMA_REGF_QUEUE_COUNT] = QDMA_REGF(11, 0), + /* QDMA_REGO_ERR_INT fields */ + [QDMA_REGF_ERR_INT_FUNC] = QDMA_REGF(11, 0), + [QDMA_REGF_ERR_INT_VEC] = QDMA_REGF(22, 12), + [QDMA_REGF_ERR_INT_ARM] = QDMA_REGF(24, 24), +}; + +#endif /* __QDMA_REGS_DEF_H */ diff --git a/drivers/dma/amd/qdma/qdma.c b/drivers/dma/amd/qdma/qdma.c new file mode 100644 index 000000000..3e61e8c43 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma.c @@ -0,0 +1,1142 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DMA driver for AMD Queue-based DMA Subsystem + * + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qdma.h" + +#define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H") +#define QDMA_REG_OFF(d, r) ((d)->roffs[r].off) + +/* MMIO regmap config for all QDMA registers */ +static const struct regmap_config qdma_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan) +{ + return container_of(chan, struct qdma_queue, vchan.chan); +} + +static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct qdma_mm_vdesc, vdesc); +} + +static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev) +{ + u32 idx; + + idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; + qdev->qintr_ring_idx %= qdev->qintr_ring_num; + + return idx; +} + +static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data, + enum qdma_reg_fields field) +{ + const struct qdma_reg_field *f = &qdev->rfields[field]; + u16 low_pos, hi_pos, low_bit, hi_bit; + u64 value = 0, mask; + + low_pos = f->lsb / BITS_PER_TYPE(*data); + hi_pos = f->msb / BITS_PER_TYPE(*data); + + if (low_pos == hi_pos) { + low_bit = f->lsb % BITS_PER_TYPE(*data); + hi_bit = f->msb % BITS_PER_TYPE(*data); + mask = GENMASK(hi_bit, low_bit); + value = (data[low_pos] & mask) >> low_bit; + } else if (hi_pos == low_pos + 1) { + low_bit = f->lsb % BITS_PER_TYPE(*data); + hi_bit = low_bit + (f->msb - f->lsb); + value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) | + data[low_pos]; + mask = GENMASK_ULL(hi_bit, low_bit); + value = (value & mask) >> low_bit; + } else { + hi_bit = f->msb % BITS_PER_TYPE(*data); + mask = GENMASK(hi_bit, 0); + value = data[hi_pos] & mask; + low_bit = f->msb - f->lsb - hi_bit; + value <<= low_bit; + low_bit -= 32; + value |= (u64)data[hi_pos - 1] << low_bit; + mask = GENMASK(31, 32 - low_bit); + value |= (data[hi_pos - 2] & mask) >> low_bit; + } + + return value; +} + +static void qdma_set_field(const struct qdma_device *qdev, u32 *data, + enum qdma_reg_fields field, u64 value) +{ + const struct qdma_reg_field *f = &qdev->rfields[field]; + u16 low_pos, hi_pos, low_bit; + + low_pos = f->lsb / BITS_PER_TYPE(*data); + hi_pos = f->msb / BITS_PER_TYPE(*data); + low_bit = f->lsb % BITS_PER_TYPE(*data); + + data[low_pos++] |= value << low_bit; + if (low_pos <= hi_pos) + data[low_pos++] |= (u32)(value >> (32 - low_bit)); + if (low_pos <= hi_pos) + data[low_pos] |= (u32)(value >> (64 - low_bit)); +} + +static inline int qdma_reg_write(const struct qdma_device *qdev, + const u32 *data, enum qdma_regs reg) +{ + const struct qdma_reg *r = &qdev->roffs[reg]; + int ret; + + if (r->count > 1) + ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); + else + ret = regmap_write(qdev->regmap, r->off, *data); + + return ret; +} + +static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data, + enum qdma_regs reg) +{ + const struct qdma_reg *r = &qdev->roffs[reg]; + int ret; + + if (r->count > 1) + ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); + else + ret = regmap_read(qdev->regmap, r->off, data); + + return ret; +} + +static int qdma_context_cmd_execute(const struct qdma_device *qdev, + enum qdma_ctxt_type type, + enum qdma_ctxt_cmd cmd, u16 index) +{ + u32 value = 0; + int ret; + + qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index); + qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd); + qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type); + + ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(qdev->regmap, + QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD), + value, + !qdma_get_field(qdev, &value, + QDMA_REGF_CMD_BUSY), + QDMA_POLL_INTRVL_US, + QDMA_POLL_TIMEOUT_US); + if (ret) { + qdma_err(qdev, "Context command execution timed out"); + return ret; + } + + return 0; +} + +static int qdma_context_write_data(const struct qdma_device *qdev, + const u32 *data) +{ + u32 mask[QDMA_CTXT_REGMAP_LEN]; + int ret; + + memset(mask, ~0, sizeof(mask)); + + ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK); + if (ret) + return ret; + + ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA); + if (ret) + return ret; + + return 0; +} + +static void qdma_prep_sw_desc_context(const struct qdma_device *qdev, + const struct qdma_ctxt_sw_desc *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); + qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); + + qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B); + qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1); +} + +static void qdma_prep_intr_context(const struct qdma_device *qdev, + const struct qdma_ctxt_intr *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); + qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); + qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); + qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); + qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); + qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); +} + +static void qdma_prep_fmap_context(const struct qdma_device *qdev, + const struct qdma_ctxt_fmap *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); +} + +/* + * Program the indirect context register space + * + * Once the queue is enabled, context is dynamically updated by hardware. Any + * modification of the context through this API when the queue is enabled can + * result in unexpected behavior. Reading the context when the queue is enabled + * is not recommended as it can result in reduced performance. + */ +static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type, + enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt) +{ + int ret; + + mutex_lock(&qdev->ctxt_lock); + if (cmd == QDMA_CTXT_WRITE) { + ret = qdma_context_write_data(qdev, ctxt); + if (ret) + goto failed; + } + + ret = qdma_context_cmd_execute(qdev, type, cmd, index); + if (ret) + goto failed; + + if (cmd == QDMA_CTXT_READ) { + ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA); + if (ret) + goto failed; + } + +failed: + mutex_unlock(&qdev->ctxt_lock); + + return ret; +} + +static int qdma_check_queue_status(struct qdma_device *qdev, + enum dma_transfer_direction dir, u16 qid) +{ + u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0}; + enum qdma_ctxt_type type; + int ret; + + if (dir == DMA_MEM_TO_DEV) + type = QDMA_CTXT_DESC_SW_H2C; + else + type = QDMA_CTXT_DESC_SW_C2H; + ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data); + if (ret) + return ret; + + status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE); + if (status) { + qdma_err(qdev, "queue %d already in use", qid); + return -EBUSY; + } + + return 0; +} + +static int qdma_clear_queue_context(const struct qdma_queue *queue) +{ + static const enum qdma_ctxt_type h2c_types[] = { + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_PFTCH, + }; + static const enum qdma_ctxt_type c2h_types[] = { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_PFTCH, + }; + struct qdma_device *qdev = queue->qdev; + const enum qdma_ctxt_type *type; + int ret, num, i; + + if (queue->dir == DMA_MEM_TO_DEV) { + type = h2c_types; + num = ARRAY_SIZE(h2c_types); + } else { + type = c2h_types; + num = ARRAY_SIZE(c2h_types); + } + for (i = 0; i < num; i++) { + ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR, + queue->qid, NULL); + if (ret) { + qdma_err(qdev, "Failed to clear ctxt %d", type[i]); + return ret; + } + } + + return 0; +} + +static int qdma_setup_fmap_context(struct qdma_device *qdev) +{ + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + struct qdma_ctxt_fmap fmap; + int ret; + + ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR, + qdev->fid, NULL); + if (ret) { + qdma_err(qdev, "Failed clearing context"); + return ret; + } + + fmap.qbase = 0; + fmap.qmax = qdev->chan_num * 2; + qdma_prep_fmap_context(qdev, &fmap, ctxt); + ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE, + qdev->fid, ctxt); + if (ret) + qdma_err(qdev, "Failed setup fmap, ret %d", ret); + + return ret; +} + +static int qdma_setup_queue_context(struct qdma_device *qdev, + const struct qdma_ctxt_sw_desc *sw_desc, + enum dma_transfer_direction dir, u16 qid) +{ + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + enum qdma_ctxt_type type; + int ret; + + if (dir == DMA_MEM_TO_DEV) + type = QDMA_CTXT_DESC_SW_H2C; + else + type = QDMA_CTXT_DESC_SW_C2H; + + qdma_prep_sw_desc_context(qdev, sw_desc, ctxt); + /* Setup SW descriptor context */ + ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt); + if (ret) + qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid); + + return ret; +} + +/* + * Enable or disable memory-mapped DMA engines + * 1: enable, 0: disable + */ +static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl) +{ + int ret; + + ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL); + ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL); + + return ret; +} + +static int qdma_get_hw_info(struct qdma_device *qdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); + u32 value = 0; + int ret; + + ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT); + if (ret) + return ret; + + value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1; + if (pdata->max_mm_channels * 2 > value) { + qdma_err(qdev, "not enough hw queues %d", value); + return -EINVAL; + } + qdev->chan_num = pdata->max_mm_channels; + + ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); + if (ret) + return ret; + + qdma_info(qdev, "max channel %d, function id %d", + qdev->chan_num, qdev->fid); + + return 0; +} + +static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx) +{ + struct qdma_device *qdev = queue->qdev; + + return regmap_write(qdev->regmap, queue->pidx_reg, + pidx | QDMA_QUEUE_ARM_BIT); +} + +static inline int qdma_update_cidx(const struct qdma_queue *queue, + u16 ridx, u16 cidx) +{ + struct qdma_device *qdev = queue->qdev; + + return regmap_write(qdev->regmap, queue->cidx_reg, + ((u32)ridx << 16) | cidx); +} + +/** + * qdma_free_vdesc - Free descriptor + * @vdesc: Virtual DMA descriptor + */ +static void qdma_free_vdesc(struct virt_dma_desc *vdesc) +{ + struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc); + + kfree(vd); +} + +static int qdma_alloc_queues(struct qdma_device *qdev, + enum dma_transfer_direction dir) +{ + struct qdma_queue *q, **queues; + u32 i, pidx_base; + int ret; + + if (dir == DMA_MEM_TO_DEV) { + queues = &qdev->h2c_queues; + pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX); + } else { + queues = &qdev->c2h_queues; + pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX); + } + + *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), + GFP_KERNEL); + if (!*queues) + return -ENOMEM; + + for (i = 0; i < qdev->chan_num; i++) { + ret = qdma_check_queue_status(qdev, dir, i); + if (ret) + return ret; + + q = &(*queues)[i]; + q->ring_size = QDMA_DEFAULT_RING_SIZE; + q->idx_mask = q->ring_size - 2; + q->qdev = qdev; + q->dir = dir; + q->qid = i; + q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; + q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + + i * QDMA_DMAP_REG_STRIDE; + q->vchan.desc_free = qdma_free_vdesc; + vchan_init(&q->vchan, &qdev->dma_dev); + } + + return 0; +} + +static int qdma_device_verify(struct qdma_device *qdev) +{ + u32 value; + int ret; + + ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); + if (ret) + return ret; + + value = FIELD_GET(QDMA_IDENTIFIER_MASK, value); + if (value != QDMA_IDENTIFIER) { + qdma_err(qdev, "Invalid identifier"); + return -ENODEV; + } + qdev->rfields = qdma_regfs_default; + qdev->roffs = qdma_regos_default; + + return 0; +} + +static int qdma_device_setup(struct qdma_device *qdev) +{ + u32 ring_sz = QDMA_DEFAULT_RING_SIZE; + int ret = 0; + + ret = qdma_setup_fmap_context(qdev); + if (ret) { + qdma_err(qdev, "Failed setup fmap context"); + return ret; + } + + /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */ + ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE); + if (ret) { + qdma_err(qdev, "Failed to setup ring %d of size %ld", + QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE); + return ret; + } + + /* Enable memory-mapped DMA engine in both directions */ + ret = qdma_sgdma_control(qdev, 1); + if (ret) { + qdma_err(qdev, "Failed to SGDMA with error %d", ret); + return ret; + } + + ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV); + if (ret) { + qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); + return ret; + } + + ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM); + if (ret) { + qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); + return ret; + } + + return 0; +} + +/** + * qdma_free_queue_resources() - Free queue resources + * @chan: DMA channel + */ +static void qdma_free_queue_resources(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_device *qdev = queue->qdev; + struct qdma_platdata *pdata; + + qdma_clear_queue_context(queue); + vchan_free_chan_resources(&queue->vchan); + pdata = dev_get_platdata(&qdev->pdev->dev); + dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE, + queue->desc_base, queue->dma_desc_base); +} + +/** + * qdma_alloc_queue_resources() - Allocate queue resources + * @chan: DMA channel + */ +static int qdma_alloc_queue_resources(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_device *qdev = queue->qdev; + struct qdma_ctxt_sw_desc desc; + struct qdma_platdata *pdata; + size_t size; + int ret; + + ret = qdma_clear_queue_context(queue); + if (ret) + return ret; + + pdata = dev_get_platdata(&qdev->pdev->dev); + size = queue->ring_size * QDMA_MM_DESC_SIZE; + queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size, + &queue->dma_desc_base, + GFP_KERNEL); + if (!queue->desc_base) { + qdma_err(qdev, "Failed to allocate descriptor ring"); + return -ENOMEM; + } + + /* Setup SW descriptor queue context for DMA memory map */ + desc.vec = qdma_get_intr_ring_idx(qdev); + desc.desc_base = queue->dma_desc_base; + ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); + if (ret) { + qdma_err(qdev, "Failed to setup SW desc ctxt for %s", + chan->name); + dma_free_coherent(pdata->dma_dev, size, queue->desc_base, + queue->dma_desc_base); + return ret; + } + + queue->pidx = 0; + queue->cidx = 0; + + return 0; +} + +static bool qdma_filter_fn(struct dma_chan *chan, void *param) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_queue_info *info = param; + + return info->dir == queue->dir; +} + +static int qdma_xfer_start(struct qdma_queue *queue) +{ + struct qdma_device *qdev = queue->qdev; + int ret; + + if (!vchan_next_desc(&queue->vchan)) + return 0; + + qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d", + queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); + + ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); + if (ret) { + qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d", + queue->pidx, CHAN_STR(queue), queue->qid); + } + + return ret; +} + +static void qdma_issue_pending(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + unsigned long flags; + + spin_lock_irqsave(&queue->vchan.lock, flags); + if (vchan_issue_pending(&queue->vchan)) { + if (queue->submitted_vdesc) { + queue->issued_vdesc = queue->submitted_vdesc; + queue->submitted_vdesc = NULL; + } + qdma_xfer_start(queue); + } + + spin_unlock_irqrestore(&queue->vchan.lock, flags); +} + +static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q) +{ + struct qdma_mm_desc *desc; + + if (((q->pidx + 1) & q->idx_mask) == q->cidx) + return NULL; + + desc = q->desc_base + q->pidx; + q->pidx = (q->pidx + 1) & q->idx_mask; + + return desc; +} + +static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc) +{ + struct qdma_mm_desc *desc; + struct scatterlist *sg; + u64 addr, *src, *dst; + u32 rest, len; + int ret = 0; + u32 i; + + if (!vdesc->sg_len) + return 0; + + if (q->dir == DMA_MEM_TO_DEV) { + dst = &vdesc->dev_addr; + src = &addr; + } else { + dst = &addr; + src = &vdesc->dev_addr; + } + + for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { + addr = sg_dma_address(sg) + vdesc->sg_off; + rest = sg_dma_len(sg) - vdesc->sg_off; + while (rest) { + len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN); + desc = qdma_get_desc(q); + if (!desc) { + ret = -EBUSY; + goto out; + } + + desc->src_addr = cpu_to_le64(*src); + desc->dst_addr = cpu_to_le64(*dst); + desc->len = cpu_to_le32(len); + + vdesc->dev_addr += len; + vdesc->sg_off += len; + vdesc->pending_descs++; + addr += len; + rest -= len; + } + vdesc->sg_off = 0; + } +out: + vdesc->sg_len -= i; + vdesc->pidx = q->pidx; + return ret; +} + +static void qdma_fill_pending_vdesc(struct qdma_queue *q) +{ + struct virt_dma_chan *vc = &q->vchan; + struct qdma_mm_vdesc *vdesc = NULL; + struct virt_dma_desc *vd; + int ret; + + if (!list_empty(&vc->desc_issued)) { + vd = &q->issued_vdesc->vdesc; + list_for_each_entry_from(vd, &vc->desc_issued, node) { + vdesc = to_qdma_vdesc(vd); + ret = qdma_hw_enqueue(q, vdesc); + if (ret) { + q->issued_vdesc = vdesc; + return; + } + } + q->issued_vdesc = vdesc; + } + + if (list_empty(&vc->desc_submitted)) + return; + + if (q->submitted_vdesc) + vd = &q->submitted_vdesc->vdesc; + else + vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); + + list_for_each_entry_from(vd, &vc->desc_submitted, node) { + vdesc = to_qdma_vdesc(vd); + ret = qdma_hw_enqueue(q, vdesc); + if (ret) + break; + } + q->submitted_vdesc = vdesc; +} + +static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct virt_dma_chan *vc = to_virt_chan(tx->chan); + struct qdma_queue *q = to_qdma_queue(&vc->chan); + struct virt_dma_desc *vd; + unsigned long flags; + dma_cookie_t cookie; + + vd = container_of(tx, struct virt_dma_desc, tx); + spin_lock_irqsave(&vc->lock, flags); + cookie = dma_cookie_assign(tx); + + list_move_tail(&vd->node, &vc->desc_submitted); + qdma_fill_pending_vdesc(q); + spin_unlock_irqrestore(&vc->lock, flags); + + return cookie; +} + +static struct dma_async_tx_descriptor * +qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct qdma_queue *q = to_qdma_queue(chan); + struct dma_async_tx_descriptor *tx; + struct qdma_mm_vdesc *vdesc; + + vdesc = kzalloc_obj(*vdesc, GFP_NOWAIT); + if (!vdesc) + return NULL; + vdesc->sgl = sgl; + vdesc->sg_len = sg_len; + if (dir == DMA_MEM_TO_DEV) + vdesc->dev_addr = q->cfg.dst_addr; + else + vdesc->dev_addr = q->cfg.src_addr; + + tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); + tx->tx_submit = qdma_tx_submit; + + return tx; +} + +static int qdma_device_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct qdma_queue *q = to_qdma_queue(chan); + + memcpy(&q->cfg, cfg, sizeof(*cfg)); + + return 0; +} + +static int qdma_arm_err_intr(const struct qdma_device *qdev) +{ + u32 value = 0; + + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1); + + return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT); +} + +static irqreturn_t qdma_error_isr(int irq, void *data) +{ + struct qdma_device *qdev = data; + u32 err_stat = 0; + int ret; + + ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT); + if (ret) { + qdma_err(qdev, "read error state failed, ret %d", ret); + goto out; + } + + qdma_err(qdev, "global error %d", err_stat); + ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT); + if (ret) + qdma_err(qdev, "clear error state failed, ret %d", ret); + +out: + qdma_arm_err_intr(qdev); + return IRQ_HANDLED; +} + +static irqreturn_t qdma_queue_isr(int irq, void *data) +{ + struct qdma_intr_ring *intr = data; + struct qdma_queue *q = NULL; + struct qdma_device *qdev; + u32 index, comp_desc; + u64 intr_ent; + u8 color; + int ret; + u16 qid; + + qdev = intr->qdev; + index = intr->cidx; + while (1) { + struct virt_dma_desc *vd; + struct qdma_mm_vdesc *vdesc; + unsigned long flags; + u32 cidx; + + intr_ent = le64_to_cpu(intr->base[index]); + color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent); + if (color != intr->color) + break; + + qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent); + if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent)) + q = qdev->c2h_queues; + else + q = qdev->h2c_queues; + q += qid; + + cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent); + + spin_lock_irqsave(&q->vchan.lock, flags); + comp_desc = (cidx - q->cidx) & q->idx_mask; + + vd = vchan_next_desc(&q->vchan); + if (!vd) + goto skip; + + vdesc = to_qdma_vdesc(vd); + while (comp_desc > vdesc->pending_descs) { + list_del(&vd->node); + vchan_cookie_complete(vd); + comp_desc -= vdesc->pending_descs; + vd = vchan_next_desc(&q->vchan); + vdesc = to_qdma_vdesc(vd); + } + vdesc->pending_descs -= comp_desc; + if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + q->cidx = cidx; + + qdma_fill_pending_vdesc(q); + qdma_xfer_start(q); + +skip: + spin_unlock_irqrestore(&q->vchan.lock, flags); + + /* + * Wrap the index value and flip the expected color value if + * interrupt aggregation PIDX has wrapped around. + */ + index++; + index &= QDMA_INTR_RING_IDX_MASK; + if (!index) + intr->color = !intr->color; + } + + /* + * Update the software interrupt aggregation ring CIDX if a valid entry + * was found. + */ + if (q) { + qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); + + /* + * Record the last read index of status descriptor from the + * interrupt aggregation ring. + */ + intr->cidx = index; + + ret = qdma_update_cidx(q, intr->ridx, index); + if (ret) { + qdma_err(qdev, "Failed to update IRQ CIDX"); + return IRQ_NONE; + } + } + + return IRQ_HANDLED; +} + +static int qdma_init_error_irq(struct qdma_device *qdev) +{ + struct device *dev = &qdev->pdev->dev; + int ret; + u32 vec; + + vec = qdev->queue_irq_start - 1; + + ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr, + IRQF_ONESHOT, "amd-qdma-error", qdev); + if (ret) { + qdma_err(qdev, "Failed to request error IRQ vector: %d", vec); + return ret; + } + + ret = qdma_arm_err_intr(qdev); + if (ret) + qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret); + + return ret; +} + +static int qdmam_alloc_qintr_rings(struct qdma_device *qdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); + struct device *dev = &qdev->pdev->dev; + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + struct qdma_intr_ring *ring; + struct qdma_ctxt_intr intr_ctxt; + u32 vector; + int ret, i; + + qdev->qintr_ring_num = qdev->queue_irq_num; + qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, + sizeof(*qdev->qintr_rings), + GFP_KERNEL); + if (!qdev->qintr_rings) + return -ENOMEM; + + vector = qdev->queue_irq_start; + for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { + ring = &qdev->qintr_rings[i]; + ring->qdev = qdev; + ring->msix_id = qdev->err_irq_idx + i + 1; + ring->ridx = i; + ring->color = 1; + ring->base = dmam_alloc_coherent(pdata->dma_dev, + QDMA_INTR_RING_SIZE, + &ring->dev_base, GFP_KERNEL); + if (!ring->base) { + qdma_err(qdev, "Failed to alloc intr ring %d", i); + return -ENOMEM; + } + intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); + intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; + intr_ctxt.vec = ring->msix_id; + intr_ctxt.valid = true; + intr_ctxt.color = true; + ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, + QDMA_CTXT_CLEAR, ring->ridx, NULL); + if (ret) { + qdma_err(qdev, "Failed clear intr ctx, ret %d", ret); + return ret; + } + + qdma_prep_intr_context(qdev, &intr_ctxt, ctxt); + ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, + QDMA_CTXT_WRITE, ring->ridx, ctxt); + if (ret) { + qdma_err(qdev, "Failed setup intr ctx, ret %d", ret); + return ret; + } + + ret = devm_request_threaded_irq(dev, vector, NULL, + qdma_queue_isr, IRQF_ONESHOT, + "amd-qdma-queue", ring); + if (ret) { + qdma_err(qdev, "Failed to request irq %d", vector); + return ret; + } + } + + return 0; +} + +static int qdma_intr_init(struct qdma_device *qdev) +{ + int ret; + + ret = qdma_init_error_irq(qdev); + if (ret) { + qdma_err(qdev, "Failed to init error IRQs, ret %d", ret); + return ret; + } + + ret = qdmam_alloc_qintr_rings(qdev); + if (ret) { + qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret); + return ret; + } + + return 0; +} + +static void amd_qdma_remove(struct platform_device *pdev) +{ + struct qdma_device *qdev = platform_get_drvdata(pdev); + + qdma_sgdma_control(qdev, 0); + dma_async_device_unregister(&qdev->dma_dev); + + mutex_destroy(&qdev->ctxt_lock); +} + +static int amd_qdma_probe(struct platform_device *pdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); + struct qdma_device *qdev; + struct resource *res; + void __iomem *regs; + int ret; + + qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); + if (!qdev) + return -ENOMEM; + + platform_set_drvdata(pdev, qdev); + qdev->pdev = pdev; + mutex_init(&qdev->ctxt_lock); + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res) { + qdma_err(qdev, "Failed to get IRQ resource"); + ret = -ENODEV; + goto failed; + } + qdev->err_irq_idx = pdata->irq_index; + qdev->queue_irq_start = res->start + 1; + qdev->queue_irq_num = resource_size(res) - 1; + + regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(regs)) { + ret = PTR_ERR(regs); + qdma_err(qdev, "Failed to map IO resource, err %d", ret); + goto failed; + } + + qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, + &qdma_regmap_config); + if (IS_ERR(qdev->regmap)) { + ret = PTR_ERR(qdev->regmap); + qdma_err(qdev, "Regmap init failed, err %d", ret); + goto failed; + } + + ret = qdma_device_verify(qdev); + if (ret) + goto failed; + + ret = qdma_get_hw_info(qdev); + if (ret) + goto failed; + + INIT_LIST_HEAD(&qdev->dma_dev.channels); + + ret = qdma_device_setup(qdev); + if (ret) + goto failed; + + ret = qdma_intr_init(qdev); + if (ret) { + qdma_err(qdev, "Failed to initialize IRQs %d", ret); + goto failed_disable_engine; + } + + dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); + dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); + + qdev->dma_dev.dev = &pdev->dev; + qdev->dma_dev.filter.map = pdata->device_map; + qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; + qdev->dma_dev.filter.fn = qdma_filter_fn; + qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; + qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; + qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; + qdev->dma_dev.device_config = qdma_device_config; + qdev->dma_dev.device_issue_pending = qdma_issue_pending; + qdev->dma_dev.device_tx_status = dma_cookie_status; + qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + + ret = dma_async_device_register(&qdev->dma_dev); + if (ret) { + qdma_err(qdev, "Failed to register AMD QDMA: %d", ret); + goto failed_disable_engine; + } + + return 0; + +failed_disable_engine: + qdma_sgdma_control(qdev, 0); +failed: + mutex_destroy(&qdev->ctxt_lock); + qdma_err(qdev, "Failed to probe AMD QDMA driver"); + return ret; +} + +static struct platform_driver amd_qdma_driver = { + .driver = { + .name = "amd-qdma", + }, + .probe = amd_qdma_probe, + .remove = amd_qdma_remove, +}; + +module_platform_driver(amd_qdma_driver); + +MODULE_DESCRIPTION("AMD QDMA driver"); +MODULE_AUTHOR("XRT Team "); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/amd/qdma/qdma.h b/drivers/dma/amd/qdma/qdma.h new file mode 100644 index 000000000..94089f1f0 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * DMA header for AMD Queue-based DMA Subsystem + * + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef __QDMA_H +#define __QDMA_H + +#include +#include +#include +#include +#include + +#include "../../virt-dma.h" + +#define DISABLE 0 +#define ENABLE 1 + +#define QDMA_MIN_IRQ 3 +#define QDMA_INTR_NAME_MAX_LEN 30 +#define QDMA_INTR_PREFIX "amd-qdma" + +#define QDMA_IDENTIFIER 0x1FD3 +#define QDMA_DEFAULT_RING_SIZE (BIT(10) + 1) +#define QDMA_DEFAULT_RING_ID 0 +#define QDMA_POLL_INTRVL_US 10 /* 10us */ +#define QDMA_POLL_TIMEOUT_US (500 * 1000) /* 500ms */ +#define QDMA_DMAP_REG_STRIDE 16 +#define QDMA_CTXT_REGMAP_LEN 8 /* 8 regs */ +#define QDMA_MM_DESC_SIZE 32 /* Bytes */ +#define QDMA_MM_DESC_LEN_BITS 28 +#define QDMA_MM_DESC_MAX_LEN (BIT(QDMA_MM_DESC_LEN_BITS) - 1) +#define QDMA_MIN_DMA_ALLOC_SIZE 4096 +#define QDMA_INTR_RING_SIZE BIT(13) +#define QDMA_INTR_RING_IDX_MASK GENMASK(9, 0) +#define QDMA_INTR_RING_BASE(_addr) ((_addr) >> 12) + +#define QDMA_IDENTIFIER_REGOFF 0x0 +#define QDMA_IDENTIFIER_MASK GENMASK(31, 16) +#define QDMA_QUEUE_ARM_BIT BIT(16) + +#define qdma_err(qdev, fmt, args...) \ + dev_err(&(qdev)->pdev->dev, fmt, ##args) + +#define qdma_dbg(qdev, fmt, args...) \ + dev_dbg(&(qdev)->pdev->dev, fmt, ##args) + +#define qdma_info(qdev, fmt, args...) \ + dev_info(&(qdev)->pdev->dev, fmt, ##args) + +enum qdma_reg_fields { + QDMA_REGF_IRQ_ENABLE, + QDMA_REGF_WBK_ENABLE, + QDMA_REGF_WBI_CHECK, + QDMA_REGF_IRQ_ARM, + QDMA_REGF_IRQ_VEC, + QDMA_REGF_IRQ_AGG, + QDMA_REGF_WBI_INTVL_ENABLE, + QDMA_REGF_MRKR_DISABLE, + QDMA_REGF_QUEUE_ENABLE, + QDMA_REGF_QUEUE_MODE, + QDMA_REGF_DESC_BASE, + QDMA_REGF_DESC_SIZE, + QDMA_REGF_RING_ID, + QDMA_REGF_CMD_INDX, + QDMA_REGF_CMD_CMD, + QDMA_REGF_CMD_TYPE, + QDMA_REGF_CMD_BUSY, + QDMA_REGF_QUEUE_COUNT, + QDMA_REGF_QUEUE_MAX, + QDMA_REGF_QUEUE_BASE, + QDMA_REGF_FUNCTION_ID, + QDMA_REGF_INTR_AGG_BASE, + QDMA_REGF_INTR_VECTOR, + QDMA_REGF_INTR_SIZE, + QDMA_REGF_INTR_VALID, + QDMA_REGF_INTR_COLOR, + QDMA_REGF_INTR_FUNCTION_ID, + QDMA_REGF_ERR_INT_FUNC, + QDMA_REGF_ERR_INT_VEC, + QDMA_REGF_ERR_INT_ARM, + QDMA_REGF_MAX +}; + +enum qdma_regs { + QDMA_REGO_CTXT_DATA, + QDMA_REGO_CTXT_CMD, + QDMA_REGO_CTXT_MASK, + QDMA_REGO_MM_H2C_CTRL, + QDMA_REGO_MM_C2H_CTRL, + QDMA_REGO_QUEUE_COUNT, + QDMA_REGO_RING_SIZE, + QDMA_REGO_H2C_PIDX, + QDMA_REGO_C2H_PIDX, + QDMA_REGO_INTR_CIDX, + QDMA_REGO_FUNC_ID, + QDMA_REGO_ERR_INT, + QDMA_REGO_ERR_STAT, + QDMA_REGO_MAX +}; + +struct qdma_reg_field { + u16 lsb; /* Least significant bit of field */ + u16 msb; /* Most significant bit of field */ +}; + +struct qdma_reg { + u32 off; + u32 count; +}; + +#define QDMA_REGF(_msb, _lsb) { \ + .lsb = (_lsb), \ + .msb = (_msb), \ +} + +#define QDMA_REGO(_off, _count) { \ + .off = (_off), \ + .count = (_count), \ +} + +enum qdma_desc_size { + QDMA_DESC_SIZE_8B, + QDMA_DESC_SIZE_16B, + QDMA_DESC_SIZE_32B, + QDMA_DESC_SIZE_64B, +}; + +enum qdma_queue_op_mode { + QDMA_QUEUE_OP_STREAM, + QDMA_QUEUE_OP_MM, +}; + +enum qdma_ctxt_type { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_WRB, + QDMA_CTXT_PFTCH, + QDMA_CTXT_INTR_COAL, + QDMA_CTXT_RSVD, + QDMA_CTXT_HOST_PROFILE, + QDMA_CTXT_TIMER, + QDMA_CTXT_FMAP, + QDMA_CTXT_FNC_STS, +}; + +enum qdma_ctxt_cmd { + QDMA_CTXT_CLEAR, + QDMA_CTXT_WRITE, + QDMA_CTXT_READ, + QDMA_CTXT_INVALIDATE, + QDMA_CTXT_MAX +}; + +struct qdma_ctxt_sw_desc { + u64 desc_base; + u16 vec; +}; + +struct qdma_ctxt_intr { + u64 agg_base; + u16 vec; + u32 size; + bool valid; + bool color; +}; + +struct qdma_ctxt_fmap { + u16 qbase; + u16 qmax; +}; + +struct qdma_device; + +struct qdma_mm_desc { + __le64 src_addr; + __le32 len; + __le32 reserved1; + __le64 dst_addr; + __le64 reserved2; +} __packed; + +struct qdma_mm_vdesc { + struct virt_dma_desc vdesc; + struct qdma_queue *queue; + struct scatterlist *sgl; + u64 sg_off; + u32 sg_len; + u64 dev_addr; + u32 pidx; + u32 pending_descs; + struct dma_slave_config cfg; +}; + +#define QDMA_VDESC_QUEUED(vdesc) (!(vdesc)->sg_len) + +struct qdma_queue { + struct qdma_device *qdev; + struct virt_dma_chan vchan; + enum dma_transfer_direction dir; + struct dma_slave_config cfg; + struct qdma_mm_desc *desc_base; + struct qdma_mm_vdesc *submitted_vdesc; + struct qdma_mm_vdesc *issued_vdesc; + dma_addr_t dma_desc_base; + u32 pidx_reg; + u32 cidx_reg; + u32 ring_size; + u32 idx_mask; + u16 qid; + u32 pidx; + u32 cidx; +}; + +struct qdma_intr_ring { + struct qdma_device *qdev; + __le64 *base; + dma_addr_t dev_base; + char msix_name[QDMA_INTR_NAME_MAX_LEN]; + u32 msix_vector; + u16 msix_id; + u32 ring_size; + u16 ridx; + u16 cidx; + u8 color; +}; + +#define QDMA_INTR_MASK_PIDX GENMASK_ULL(15, 0) +#define QDMA_INTR_MASK_CIDX GENMASK_ULL(31, 16) +#define QDMA_INTR_MASK_DESC_COLOR GENMASK_ULL(32, 32) +#define QDMA_INTR_MASK_STATE GENMASK_ULL(34, 33) +#define QDMA_INTR_MASK_ERROR GENMASK_ULL(36, 35) +#define QDMA_INTR_MASK_TYPE GENMASK_ULL(38, 38) +#define QDMA_INTR_MASK_QID GENMASK_ULL(62, 39) +#define QDMA_INTR_MASK_COLOR GENMASK_ULL(63, 63) + +struct qdma_device { + struct platform_device *pdev; + struct dma_device dma_dev; + struct regmap *regmap; + struct mutex ctxt_lock; /* protect ctxt registers */ + const struct qdma_reg_field *rfields; + const struct qdma_reg *roffs; + struct qdma_queue *h2c_queues; + struct qdma_queue *c2h_queues; + struct qdma_intr_ring *qintr_rings; + u32 qintr_ring_num; + u32 qintr_ring_idx; + u32 chan_num; + u32 queue_irq_start; + u32 queue_irq_num; + u32 err_irq_idx; + u32 fid; +}; + +extern const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX]; +extern const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX]; + +#endif /* __QDMA_H */ -- cgit v1.3.1