diff options
Diffstat (limited to 'drivers/accel/amdxdna')
49 files changed, 13706 insertions, 0 deletions
diff --git a/drivers/accel/amdxdna/Kconfig b/drivers/accel/amdxdna/Kconfig new file mode 100644 index 000000000..f39d7a872 --- /dev/null +++ b/drivers/accel/amdxdna/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config DRM_ACCEL_AMDXDNA + tristate "AMD AI Engine" + depends on AMD_IOMMU + depends on DRM_ACCEL + depends on PCI && HAS_IOMEM + depends on X86_64 + select DRM_SCHED + select DRM_GEM_SHMEM_HELPER + select FW_LOADER + select HMM_MIRROR + help + Choose this option to enable support for NPU integrated into AMD + client CPUs like AMD Ryzen AI 300 Series. AMD NPU can be used to + accelerate machine learning applications. + + If "M" is selected, the driver module will be amdxdna. diff --git a/drivers/accel/amdxdna/Makefile b/drivers/accel/amdxdna/Makefile new file mode 100644 index 000000000..05cce0a38 --- /dev/null +++ b/drivers/accel/amdxdna/Makefile @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0-only + +amdxdna-y := \ + aie.o \ + aie_psp.o \ + aie_smu.o \ + aie2_ctx.o \ + aie2_error.o \ + aie2_message.o \ + aie2_pci.o \ + aie2_pm.o \ + aie2_solver.o \ + aie4_ctx.o \ + aie4_message.o \ + aie4_pci.o \ + amdxdna_cbuf.o \ + amdxdna_ctx.o \ + amdxdna_gem.o \ + amdxdna_iommu.o \ + amdxdna_mailbox.o \ + amdxdna_mailbox_helper.o \ + amdxdna_pci_drv.o \ + amdxdna_pm.o \ + amdxdna_sysfs.o \ + amdxdna_ubuf.o \ + npu1_regs.o \ + npu3_regs.o \ + npu4_regs.o \ + npu5_regs.o \ + npu6_regs.o + +amdxdna-$(CONFIG_PCI_IOV) += aie4_sriov.o +amdxdna-$(CONFIG_DEBUG_FS) += amdxdna_debugfs.o +obj-$(CONFIG_DRM_ACCEL_AMDXDNA) = amdxdna.o diff --git a/drivers/accel/amdxdna/TODO b/drivers/accel/amdxdna/TODO new file mode 100644 index 000000000..0e4bbebea --- /dev/null +++ b/drivers/accel/amdxdna/TODO @@ -0,0 +1 @@ +- Add debugfs support diff --git a/drivers/accel/amdxdna/aie.c b/drivers/accel/amdxdna/aie.c new file mode 100644 index 000000000..dd6f36f22 --- /dev/null +++ b/drivers/accel/amdxdna/aie.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <linux/errno.h> + +#include "aie.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +void aie_dump_mgmt_chann_debug(struct aie_device *aie) +{ + struct amdxdna_dev *xdna = aie->xdna; + + XDNA_DBG(xdna, "i2x tail 0x%x", aie->mgmt_i2x.mb_tail_ptr_reg); + XDNA_DBG(xdna, "i2x head 0x%x", aie->mgmt_i2x.mb_head_ptr_reg); + XDNA_DBG(xdna, "i2x ringbuf 0x%x", aie->mgmt_i2x.rb_start_addr); + XDNA_DBG(xdna, "i2x rsize 0x%x", aie->mgmt_i2x.rb_size); + XDNA_DBG(xdna, "x2i tail 0x%x", aie->mgmt_x2i.mb_tail_ptr_reg); + XDNA_DBG(xdna, "x2i head 0x%x", aie->mgmt_x2i.mb_head_ptr_reg); + XDNA_DBG(xdna, "x2i ringbuf 0x%x", aie->mgmt_x2i.rb_start_addr); + XDNA_DBG(xdna, "x2i rsize 0x%x", aie->mgmt_x2i.rb_size); + XDNA_DBG(xdna, "x2i chann index 0x%x", aie->mgmt_chan_idx); + XDNA_DBG(xdna, "mailbox protocol major 0x%x", aie->mgmt_prot_major); + XDNA_DBG(xdna, "mailbox protocol minor 0x%x", aie->mgmt_prot_minor); +} + +void aie_destroy_chann(struct aie_device *aie, struct mailbox_channel **chann) +{ + struct amdxdna_dev *xdna = aie->xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!*chann) + return; + + xdna_mailbox_stop_channel(*chann); + xdna_mailbox_free_channel(*chann); + *chann = NULL; +} + +int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg) +{ + struct amdxdna_dev *xdna = aie->xdna; + struct xdna_notify *hdl = msg->handle; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!aie->mgmt_chann) + return -ENODEV; + + ret = xdna_send_msg_wait(xdna, aie->mgmt_chann, msg); + if (ret == -ETIME) + aie_destroy_chann(aie, &aie->mgmt_chann); + + if (!ret && *hdl->status) { + XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", + msg->opcode, *hdl->data); + ret = -EINVAL; + } + + return ret; +} + +int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor) +{ + const struct amdxdna_fw_feature_tbl *feature; + bool found = false; + + for (feature = aie->xdna->dev_info->fw_feature_tbl; + feature->major; feature++) { + if (feature->major != fw_major) + continue; + if (fw_minor < feature->min_minor) + continue; + if (feature->max_minor > 0 && fw_minor > feature->max_minor) + continue; + + aie->feature_mask |= feature->features; + + /* firmware version matches one of the driver support entry */ + found = true; + } + + return found ? 0 : -EOPNOTSUPP; +} + +static void amdxdna_update_vbnv(struct amdxdna_dev *xdna, + const struct amdxdna_rev_vbnv *tbl, + u32 rev) +{ + int i; + + for (i = 0; tbl[i].vbnv; i++) { + if (tbl[i].revision == rev) { + xdna->vbnv = tbl[i].vbnv; + break; + } + } +} + +void amdxdna_vbnv_init(struct amdxdna_dev *xdna) +{ + const struct amdxdna_dev_info *info = xdna->dev_info; + u32 rev; + + xdna->vbnv = info->default_vbnv; + + if (!info->ops->get_dev_revision || !info->rev_vbnv_tbl) + return; + + if (info->ops->get_dev_revision(xdna, &rev)) + return; + + amdxdna_update_vbnv(xdna, info->rev_vbnv_tbl, rev); +} + +int amdxdna_get_metadata(struct aie_device *aie, + struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + int ret = 0; + u32 buf_sz; + + buf_sz = min(args->buffer_size, sizeof(aie->metadata)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &aie->metadata, buf_sz)) + ret = -EFAULT; + + return ret; +} + +void *amdxdna_alloc_msg_buffer(struct amdxdna_dev *xdna, u32 *size, + dma_addr_t *dma_addr) +{ + void *vaddr; + int order; + + *size = max_t(u32, *size, SZ_8K); + order = get_order(*size); + if (order > MAX_PAGE_ORDER) + return ERR_PTR(-EINVAL); + *size = PAGE_SIZE << order; + + if (amdxdna_iova_on(xdna)) + return amdxdna_iommu_alloc(xdna, *size, dma_addr); + + vaddr = dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr, + DMA_FROM_DEVICE, GFP_KERNEL); + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +void amdxdna_free_msg_buffer(struct amdxdna_dev *xdna, size_t size, + void *cpu_addr, dma_addr_t dma_addr) +{ + if (amdxdna_iova_on(xdna)) { + amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr); + return; + } + + dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE); +} diff --git a/drivers/accel/amdxdna/aie.h b/drivers/accel/amdxdna/aie.h new file mode 100644 index 000000000..0483d582b --- /dev/null +++ b/drivers/accel/amdxdna/aie.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ +#ifndef _AIE_H_ +#define _AIE_H_ + +#include <drm/amdxdna_accel.h> +#include "amdxdna_pci_drv.h" +#include "amdxdna_mailbox.h" + +#define AIE_INTERVAL 20000 /* us */ +#define AIE_TIMEOUT 1000000 /* us */ + +struct psp_device; +struct smu_device; + +struct aie_device { + struct amdxdna_dev *xdna; + struct mailbox_channel *mgmt_chann; + struct xdna_mailbox_chann_res mgmt_x2i; + struct xdna_mailbox_chann_res mgmt_i2x; + u32 mgmt_chan_idx; + u32 mgmt_prot_major; + u32 mgmt_prot_minor; + unsigned long feature_mask; + + struct psp_device *psp_hdl; + struct smu_device *smu_hdl; + + struct amdxdna_drm_query_aie_metadata metadata; +}; + +#define DECLARE_AIE_MSG(name, op) \ + DECLARE_XDNA_MSG_COMMON(name, op, -1) +#define AIE_FEATURE_ON(aie, feature) test_bit(feature, &(aie)->feature_mask) + +#define PSP_REG_BAR(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].bar_idx) +#define PSP_REG_OFF(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].offset) + +#define SMU_REG_BAR(ndev, idx) ((ndev)->priv->smu_regs_off[(idx)].bar_idx) +#define SMU_REG_OFF(ndev, idx) ((ndev)->priv->smu_regs_off[(idx)].offset) + +#define DEFINE_BAR_OFFSET(reg_name, bar, reg_addr) \ + [reg_name] = {bar##_BAR_INDEX, (reg_addr) - bar##_BAR_BASE} + +enum smu_reg_idx { + SMU_CMD_REG = 0, + SMU_ARG_REG, + SMU_INTR_REG, + SMU_RESP_REG, + SMU_OUT_REG, + SMU_MAX_REGS /* Keep this at the end */ +}; + +enum psp_reg_idx { + PSP_CMD_REG = 0, + PSP_ARG0_REG, + PSP_ARG1_REG, + PSP_ARG2_REG, + PSP_NUM_IN_REGS, /* number of input registers */ + PSP_INTR_REG = PSP_NUM_IN_REGS, + PSP_STATUS_REG, + PSP_RESP_REG, + PSP_PWAITMODE_REG, + PSP_MAX_REGS /* Keep this at the end */ +}; + +struct aie_bar_off_pair { + int bar_idx; + u32 offset; +}; + +struct smu_config { + void __iomem *smu_regs[SMU_MAX_REGS]; +}; + +struct psp_config { + const void *fw_buf; + u32 fw_size; + const void *certfw_buf; + u32 certfw_size; + void __iomem *psp_regs[PSP_MAX_REGS]; + u32 arg2_mask; + u32 notify_val; +}; + +/* Device revision to VBNV string mapping table entry */ +struct amdxdna_rev_vbnv { + u32 revision; + const char *vbnv; +}; + +/* aie.c */ +void aie_dump_mgmt_chann_debug(struct aie_device *aie); +void aie_destroy_chann(struct aie_device *aie, struct mailbox_channel **chann); +int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg); +int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor); +void amdxdna_vbnv_init(struct amdxdna_dev *xdna); +int amdxdna_get_metadata(struct aie_device *aie, struct amdxdna_client *client, + struct amdxdna_drm_get_info *args); +void *amdxdna_alloc_msg_buffer(struct amdxdna_dev *xdna, u32 *size, + dma_addr_t *dma_addr); +void amdxdna_free_msg_buffer(struct amdxdna_dev *xdna, size_t size, + void *cpu_addr, dma_addr_t dma_addr); + +/* aie_psp.c */ +struct psp_device *aiem_psp_create(struct drm_device *ddev, struct psp_config *conf); +int aie_psp_start(struct psp_device *psp); +void aie_psp_stop(struct psp_device *psp); +int aie_psp_waitmode_poll(struct psp_device *psp); + +/* aie_smu.c */ +struct smu_device *aiem_smu_create(struct drm_device *ddev, struct smu_config *conf); +int aie_smu_init(struct smu_device *smu); +void aie_smu_fini(struct smu_device *smu); +int aie_smu_set_clocks(struct smu_device *smu, u32 *npuclk, u32 *hclk); +int aie_smu_set_dpm(struct smu_device *smu, u32 dpm_level); + +#endif /* _AIE_H_ */ diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c new file mode 100644 index 000000000..baf9a8b90 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -0,0 +1,1246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_print.h> +#include <drm/drm_syncobj.h> +#include <linux/hmm.h> +#include <linux/types.h> +#include <linux/xarray.h> +#include <trace/events/amdxdna.h> + +#include "aie2_msg_priv.h" +#include "aie2_pci.h" +#include "aie2_solver.h" +#include "amdxdna_ctx.h" +#include "amdxdna_gem.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" + +static bool force_cmdlist = true; +module_param(force_cmdlist, bool, 0600); +MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)"); + +uint tdr_timeout_ms = 2000; +module_param(tdr_timeout_ms, int, 0400); +MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)"); + +struct aie2_ctx_health { + struct amdxdna_ctx_health header; + u32 txn_op_idx; + u32 ctx_pc; + u32 fatal_error_type; + u32 fatal_error_exception_type; + u32 fatal_error_exception_pc; + u32 fatal_error_app_module; +}; + +static inline void aie2_tdr_signal(struct amdxdna_dev *xdna) +{ + WRITE_ONCE(xdna->dev_handle->last_signal_ts, jiffies); +} + +static bool aie2_tdr_detect(struct amdxdna_dev *xdna) +{ + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + unsigned long last = READ_ONCE(ndev->last_signal_ts); + + if (!tdr_timeout_ms) + return false; + + if (!time_after(jiffies, last + msecs_to_jiffies(tdr_timeout_ms))) + return false; + + XDNA_ERR(xdna, "TDR timeout detected"); + return true; +} + +static void aie2_cmd_release(struct kref *ref) +{ + struct amdxdna_drv_cmd *drv_cmd = container_of(ref, struct amdxdna_drv_cmd, refcnt); + + kfree(drv_cmd); +} + +static void aie2_cmd_put(struct amdxdna_drv_cmd *drv_cmd) +{ + kref_put(&drv_cmd->refcnt, aie2_cmd_release); +} + +static void aie2_job_release(struct kref *ref) +{ + struct amdxdna_sched_job *job; + + job = container_of(ref, struct amdxdna_sched_job, refcnt); + + amdxdna_sched_job_cleanup(job); + atomic64_inc(&job->hwctx->job_free_cnt); + wake_up(&job->hwctx->priv->job_free_wq); + if (job->out_fence) + dma_fence_put(job->out_fence); + if (job->drv_cmd) + aie2_cmd_put(job->drv_cmd); + kfree(job->aie2_job_health); + kfree(job); +} + +static void aie2_job_put(struct amdxdna_sched_job *job) +{ + kref_put(&job->refcnt, aie2_job_release); +} + +/* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ +static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, + struct drm_sched_job *bad_job) +{ + drm_sched_stop(&hwctx->priv->sched, bad_job); + aie2_destroy_context(xdna->dev_handle, hwctx); + drm_sched_start(&hwctx->priv->sched, 0); +} + +static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_gem_obj *heap = hwctx->priv->heap; + unsigned long heap_id; + int ret; + + ret = aie2_create_context(xdna->dev_handle, hwctx); + if (ret) { + XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); + goto out; + } + + ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, + amdxdna_obj_dma_addr(heap), + heap->mem.size); + if (ret) { + XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); + goto out; + } + + xa_for_each_range(&hwctx->client->dev_heap_xa, heap_id, heap, 1, + hwctx->last_attached_heap) { + ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, + amdxdna_obj_dma_addr(heap), + heap->mem.size); + if (ret) { + XDNA_ERR(xdna, "Add heap %ld failed ret %d", heap_id, ret); + goto out; + } + } + + ret = aie2_config_cu(hwctx, NULL); + if (ret) { + XDNA_ERR(xdna, "Config cu failed, ret %d", ret); + goto out; + } + +out: + XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); + return ret; +} + +static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq) +{ + struct dma_fence *fence, *out_fence = NULL; + int ret; + + fence = drm_syncobj_fence_get(hwctx->priv->syncobj); + if (!fence) + return NULL; + + ret = dma_fence_chain_find_seqno(&fence, seq); + if (ret) + goto out; + + out_fence = dma_fence_get(dma_fence_chain_contained(fence)); + +out: + dma_fence_put(fence); + return out_fence; +} + +static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) +{ + struct dma_fence *fence; + + fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); + if (!fence) + return; + + /* Wait up to 2 seconds for fw to finish all pending requests */ + dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); + dma_fence_put(fence); +} + +static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + + aie2_hwctx_wait_for_idle(hwctx); + aie2_hwctx_stop(xdna, hwctx, NULL); + + return 0; +} + +void aie2_hwctx_suspend(struct amdxdna_client *client) +{ + struct amdxdna_dev *xdna = client->xdna; + + /* + * Command timeout is unlikely. But if it happens, it doesn't + * break the system. aie2_hwctx_stop() will destroy mailbox + * and abort all commands. + */ + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + amdxdna_hwctx_walk(client, NULL, aie2_hwctx_suspend_cb); +} + +static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + + return aie2_hwctx_restart(xdna, hwctx); +} + +int aie2_hwctx_resume(struct amdxdna_client *client) +{ + /* + * The resume path cannot guarantee that mailbox channel can be + * regenerated. If this happen, when submit message to this + * mailbox channel, error will return. + */ + return amdxdna_hwctx_walk(client, NULL, aie2_hwctx_resume_cb); +} + +static void +aie2_sched_notify(struct amdxdna_sched_job *job) +{ + struct dma_fence *fence = job->fence; + + trace_xdna_job(&job->base, job->hwctx->name, "signaling fence", + job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); + + aie2_tdr_signal(job->hwctx->client->xdna); + job->hwctx->priv->completed++; + dma_fence_signal(fence); + + up(&job->hwctx->priv->job_sem); + job->job_done = true; + mmput_async(job->mm); + aie2_job_put(job); +} + +static void aie2_set_cmd_timeout(struct amdxdna_sched_job *job) +{ + struct aie2_ctx_health *aie2_health __free(kfree) = NULL; + struct amdxdna_dev *xdna = job->hwctx->client->xdna; + struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; + struct app_health_report *report = job->aie2_job_health; + u32 fail_cmd_idx = 0; + + if (!report) + goto set_timeout; + + XDNA_ERR(xdna, "Firmware timeout state capture:"); + XDNA_ERR(xdna, "\tVersion: %d.%d", report->major, report->minor); + XDNA_ERR(xdna, "\tReport size: 0x%x", report->size); + XDNA_ERR(xdna, "\tContext ID: %d", report->context_id); + XDNA_ERR(xdna, "\tDPU PC: 0x%x", report->dpu_pc); + XDNA_ERR(xdna, "\tTXN OP ID: 0x%x", report->txn_op_id); + XDNA_ERR(xdna, "\tContext PC: 0x%x", report->ctx_pc); + XDNA_ERR(xdna, "\tFatal error type: 0x%x", report->fatal_info.fatal_type); + XDNA_ERR(xdna, "\tFatal error exception type: 0x%x", report->fatal_info.exception_type); + XDNA_ERR(xdna, "\tFatal error exception PC: 0x%x", report->fatal_info.exception_pc); + XDNA_ERR(xdna, "\tFatal error app module: 0x%x", report->fatal_info.app_module); + XDNA_ERR(xdna, "\tFatal error task ID: %d", report->fatal_info.task_index); + XDNA_ERR(xdna, "\tTimed out sub command ID: %d", report->run_list_id); + + fail_cmd_idx = report->run_list_id; + aie2_health = kzalloc_obj(*aie2_health); + if (!aie2_health) + goto set_timeout; + + aie2_health->header.version = AMDXDNA_CMD_CTX_HEALTH_V1; + aie2_health->header.npu_gen = AMDXDNA_CMD_CTX_HEALTH_AIE2; + aie2_health->txn_op_idx = report->txn_op_id; + aie2_health->ctx_pc = report->ctx_pc; + aie2_health->fatal_error_type = report->fatal_info.fatal_type; + aie2_health->fatal_error_exception_type = report->fatal_info.exception_type; + aie2_health->fatal_error_exception_pc = report->fatal_info.exception_pc; + aie2_health->fatal_error_app_module = report->fatal_info.app_module; + +set_timeout: + amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_TIMEOUT, + aie2_health, sizeof(*aie2_health)); +} + +static int +aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) +{ + struct amdxdna_sched_job *job = handle; + struct amdxdna_gem_obj *cmd_abo; + int ret = 0; + u32 status; + + cmd_abo = job->cmd_bo; + + if (unlikely(job->job_timeout)) { + aie2_set_cmd_timeout(job); + ret = -EINVAL; + goto out; + } + + if (unlikely(!data) || unlikely(size != sizeof(u32))) { + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); + ret = -EINVAL; + goto out; + } + + status = readl(data); + XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); + if (status == AIE2_STATUS_SUCCESS) + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); + else + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR, NULL, 0); + +out: + aie2_sched_notify(job); + return ret; +} + +static int +aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) +{ + struct amdxdna_sched_job *job = handle; + int ret = 0; + + if (unlikely(!data || size != sizeof(u32))) { + job->drv_cmd->result = U32_MAX; + ret = -EINVAL; + } else { + job->drv_cmd->result = readl(data); + } + + aie2_sched_notify(job); + return ret; +} + +static int +aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) +{ + struct amdxdna_sched_job *job = handle; + struct amdxdna_gem_obj *cmd_abo; + struct amdxdna_dev *xdna; + u32 fail_cmd_idx = 0; + u32 fail_cmd_status; + u32 cmd_status; + int ret = 0; + + cmd_abo = job->cmd_bo; + + if (unlikely(job->job_timeout)) { + aie2_set_cmd_timeout(job); + ret = -EINVAL; + goto out; + } + + if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { + amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); + ret = -EINVAL; + goto out; + } + + cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); + xdna = job->hwctx->client->xdna; + XDNA_DBG(xdna, "Status 0x%x", cmd_status); + if (cmd_status == AIE2_STATUS_SUCCESS) { + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); + goto out; + } + + /* Slow path to handle error, read from ringbuf on BAR */ + fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); + fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); + XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", + fail_cmd_idx, fail_cmd_status); + + if (fail_cmd_status == AIE2_STATUS_SUCCESS) { + amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT, NULL, 0); + ret = -EINVAL; + } else { + amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR, NULL, 0); + } + +out: + aie2_sched_notify(job); + return ret; +} + +static struct dma_fence * +aie2_sched_job_run(struct drm_sched_job *sched_job) +{ + struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); + struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; + struct amdxdna_hwctx *hwctx = job->hwctx; + struct dma_fence *fence; + int ret; + + trace_xdna_job(sched_job, hwctx->name, "job run", + job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); + + if (!hwctx->priv->mbox_chann) + return NULL; + + if (!mmget_not_zero(job->mm)) + return ERR_PTR(-ESRCH); + + kref_get(&job->refcnt); + fence = dma_fence_get(job->fence); + + if (job->drv_cmd) { + switch (job->drv_cmd->opcode) { + case SYNC_DEBUG_BO: + ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); + break; + case ATTACH_DEBUG_BO: + case DETACH_DEBUG_BO: + ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); + break; + default: + ret = -EINVAL; + break; + } + goto out; + } + + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW); + + if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) + ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); + else if (force_cmdlist) + ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); + else + ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler); + +out: + if (ret) { + dma_fence_put(job->fence); + aie2_job_put(job); + mmput(job->mm); + fence = ERR_PTR(ret); + } else { + /* + * Command is successfully posted to hardware, update the + * tdr timestamp. The total pending commands are limited. + * So there will not be a case that driver keeps posting + * commands without getting any hardware respond. + */ + aie2_tdr_signal(hwctx->client->xdna); + } + trace_xdna_job(sched_job, hwctx->name, "sent to device", + job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); + + return fence; +} + +static void aie2_sched_job_free(struct drm_sched_job *sched_job) +{ + struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); + struct amdxdna_hwctx *hwctx = job->hwctx; + + /* job->drv_cmd could be freed, so use DEFAULT_IO */ + trace_xdna_job(sched_job, hwctx->name, "job free", + job->seq, DEFAULT_IO); + if (!job->job_done) + up(&hwctx->priv->job_sem); + + drm_sched_job_cleanup(sched_job); + aie2_job_put(job); +} + +static enum drm_gpu_sched_stat +aie2_sched_job_timedout(struct drm_sched_job *sched_job) +{ + struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); + struct amdxdna_hwctx *hwctx = job->hwctx; + struct app_health_report *report; + struct amdxdna_dev *xdna; + int ret; + + xdna = hwctx->client->xdna; + + guard(mutex)(&xdna->dev_lock); + + if (!aie2_tdr_detect(xdna)) + return DRM_GPU_SCHED_STAT_NO_HANG; + + report = kzalloc_obj(*report); + if (!report) + goto reset_hwctx; + + ret = aie2_query_app_health(xdna->dev_handle, hwctx->fw_ctx_id, report); + if (ret) + kfree(report); + else + job->aie2_job_health = report; + +reset_hwctx: + job->job_timeout = true; + aie2_hwctx_stop(xdna, hwctx, sched_job); + + aie2_hwctx_restart(xdna, hwctx); + + return DRM_GPU_SCHED_STAT_RESET; +} + +static const struct drm_sched_backend_ops sched_ops = { + .run_job = aie2_sched_job_run, + .free_job = aie2_sched_job_free, + .timedout_job = aie2_sched_job_timedout, +}; + +static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct amdxdna_dev_hdl *ndev; + int start, end, first, last; + u32 width = 1, entries = 0; + int i; + + if (!hwctx->num_tiles) { + XDNA_ERR(xdna, "Number of tiles is zero"); + return -EINVAL; + } + + ndev = xdna->dev_handle; + if (unlikely(!ndev->aie.metadata.core.row_count)) { + XDNA_WARN(xdna, "Core tile row count is zero"); + return -EINVAL; + } + + hwctx->num_col = hwctx->num_tiles / ndev->aie.metadata.core.row_count; + if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { + XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); + return -EINVAL; + } + + if (ndev->priv->col_align == COL_ALIGN_NATURE) + width = hwctx->num_col; + + /* + * In range [start, end], find out columns that is multiple of width. + * 'first' is the first column, + * 'last' is the last column, + * 'entries' is the total number of columns. + */ + start = xdna->dev_info->first_col; + end = ndev->total_col - hwctx->num_col; + if (start > 0 && end == 0) { + XDNA_DBG(xdna, "Force start from col 0"); + start = 0; + } + first = start + (width - start % width) % width; + last = end - end % width; + if (last >= first) + entries = (last - first) / width + 1; + XDNA_DBG(xdna, "start %d end %d first %d last %d", + start, end, first, last); + + if (unlikely(!entries)) { + XDNA_ERR(xdna, "Start %d end %d width %d", + start, end, width); + return -EINVAL; + } + + hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); + if (!hwctx->col_list) + return -ENOMEM; + + hwctx->col_list_len = entries; + hwctx->col_list[0] = first; + for (i = 1; i < entries; i++) + hwctx->col_list[i] = hwctx->col_list[i - 1] + width; + + print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, + entries * sizeof(*hwctx->col_list), false); + return 0; +} + +static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct alloc_requests *xrs_req; + u32 temporal_only_col = 0; + int ret; + + xrs_req = kzalloc_obj(*xrs_req); + if (!xrs_req) + return -ENOMEM; + + if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) { + xrs_req->cdo.start_cols = &temporal_only_col; + xrs_req->cdo.cols_len = 1; + xrs_req->cdo.ncols = xdna->dev_handle->total_col; + } else { + xrs_req->cdo.start_cols = hwctx->col_list; + xrs_req->cdo.cols_len = hwctx->col_list_len; + xrs_req->cdo.ncols = hwctx->num_col; + } + /* Use platform opc */ + xrs_req->cdo.qos_cap.opc = xdna->dev_handle->priv->col_opc * hwctx->num_col; + + xrs_req->rqos.gops = hwctx->qos.gops; + xrs_req->rqos.fps = hwctx->qos.fps; + xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; + xrs_req->rqos.latency = hwctx->qos.latency; + xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; + xrs_req->rqos.priority = hwctx->qos.priority; + + xrs_req->rid = (uintptr_t)hwctx; + + ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); + if (ret) + XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret); + + kfree(xrs_req); + return ret; +} + +static void aie2_release_resource(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + int ret; + + ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); + if (ret) + XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); +} + +static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct drm_file *filp = hwctx->client->filp; + struct drm_syncobj *syncobj; + u32 hdl; + int ret; + + hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; + + ret = drm_syncobj_create(&syncobj, 0, NULL); + if (ret) { + XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret); + return ret; + } + ret = drm_syncobj_get_handle(filp, syncobj, &hdl); + if (ret) { + drm_syncobj_put(syncobj); + XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret); + return ret; + } + hwctx->priv->syncobj = syncobj; + hwctx->syncobj_hdl = hdl; + + return 0; +} + +static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx) +{ + /* + * The syncobj_hdl is owned by user space and will be cleaned up + * separately. + */ + drm_syncobj_put(hwctx->priv->syncobj); +} + +int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + const struct drm_sched_init_args args = { + .ops = &sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, + .credit_limit = HWCTX_MAX_CMDS, + .timeout = tdr_timeout_ms ? + msecs_to_jiffies(tdr_timeout_ms) : + MAX_SCHEDULE_TIMEOUT, + .name = "amdxdna_js", + .dev = xdna->ddev.dev, + }; + struct drm_gpu_scheduler *sched; + struct amdxdna_hwctx_priv *priv; + struct amdxdna_gem_obj *heap; + int i, ret; + + priv = kzalloc_obj(*hwctx->priv); + if (!priv) + return -ENOMEM; + hwctx->priv = priv; + + mutex_lock(&client->mm_lock); + heap = xa_load(&client->dev_heap_xa, 0); + if (!heap) { + XDNA_ERR(xdna, "The client dev heap object not exist"); + mutex_unlock(&client->mm_lock); + ret = -ENOENT; + goto free_priv; + } + drm_gem_object_get(to_gobj(heap)); + mutex_unlock(&client->mm_lock); + priv->heap = heap; + sema_init(&priv->job_sem, HWCTX_MAX_CMDS); + + ret = amdxdna_gem_pin(heap); + if (ret) { + XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret); + goto put_heap; + } + + for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { + struct amdxdna_gem_obj *abo; + struct amdxdna_drm_create_bo args = { + .flags = 0, + .type = AMDXDNA_BO_DEV, + .vaddr = 0, + .size = MAX_CHAIN_CMDBUF_SIZE, + }; + + abo = amdxdna_drm_create_dev_bo(&xdna->ddev, &args, client->filp); + if (IS_ERR(abo)) { + ret = PTR_ERR(abo); + goto free_cmd_bufs; + } + + XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx", + i, amdxdna_gem_dev_addr(abo), abo->mem.size); + priv->cmd_buf[i] = abo; + } + + sched = &priv->sched; + mutex_init(&priv->io_lock); + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&priv->io_lock); + fs_reclaim_release(GFP_KERNEL); + + ret = drm_sched_init(sched, &args); + if (ret) { + XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); + goto free_cmd_bufs; + } + + ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, + &sched, 1, NULL); + if (ret) { + XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret); + goto free_sched; + } + + ret = aie2_hwctx_col_list(hwctx); + if (ret) { + XDNA_ERR(xdna, "Create col list failed, ret %d", ret); + goto free_entity; + } + + ret = amdxdna_pm_resume_get_locked(xdna); + if (ret) + goto free_col_list; + + ret = aie2_alloc_resource(hwctx); + if (ret) { + XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret); + goto suspend_put; + } + + ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, + amdxdna_obj_dma_addr(heap), + heap->mem.size); + if (ret) { + XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret); + goto release_resource; + } + + ret = amdxdna_update_heap(client, hwctx); + if (ret) { + XDNA_ERR(xdna, "Update heap failed, ret %d", ret); + goto release_resource; + } + + ret = aie2_ctx_syncobj_create(hwctx); + if (ret) { + XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret); + goto release_resource; + } + amdxdna_pm_suspend_put(xdna); + + init_waitqueue_head(&priv->job_free_wq); + + XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); + + return 0; + +release_resource: + aie2_release_resource(hwctx); +suspend_put: + amdxdna_pm_suspend_put(xdna); +free_col_list: + kfree(hwctx->col_list); +free_entity: + drm_sched_entity_destroy(&priv->entity); +free_sched: + drm_sched_fini(&priv->sched); +free_cmd_bufs: + for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { + if (!priv->cmd_buf[i]) + continue; + drm_gem_object_put(to_gobj(priv->cmd_buf[i])); + } + amdxdna_gem_unpin(heap); +put_heap: + drm_gem_object_put(to_gobj(heap)); +free_priv: + kfree(priv); + return ret; +} + +void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_dev *xdna; + int idx; + + xdna = hwctx->client->xdna; + + XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); + aie2_hwctx_wait_for_idle(hwctx); + + /* Request fw to destroy hwctx and cancel the rest pending requests */ + drm_sched_stop(&hwctx->priv->sched, NULL); + aie2_release_resource(hwctx); + drm_sched_start(&hwctx->priv->sched, 0); + + mutex_unlock(&xdna->dev_lock); + drm_sched_entity_destroy(&hwctx->priv->entity); + + /* Wait for all submitted jobs to be completed or canceled */ + wait_event(hwctx->priv->job_free_wq, + atomic64_read(&hwctx->job_submit_cnt) == + atomic64_read(&hwctx->job_free_cnt)); + mutex_lock(&xdna->dev_lock); + + drm_sched_fini(&hwctx->priv->sched); + aie2_ctx_syncobj_destroy(hwctx); + + for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) + drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); + amdxdna_gem_unpin(hwctx->priv->heap); + drm_gem_object_put(to_gobj(hwctx->priv->heap)); + + mutex_destroy(&hwctx->priv->io_lock); + kfree(hwctx->col_list); + kfree(hwctx->priv); + kfree(hwctx->cus); +} + +static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size) +{ + struct amdxdna_hwctx *hwctx = handle; + + amdxdna_pm_suspend_put(hwctx->client->xdna); + return 0; +} + +static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size) +{ + struct amdxdna_hwctx_param_config_cu *config = buf; + struct amdxdna_dev *xdna = hwctx->client->xdna; + u32 total_size; + int ret; + + XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); + if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) + return -EINVAL; + + if (hwctx->cus) { + XDNA_ERR(xdna, "Not support re-config CU"); + return -EINVAL; + } + + if (!config->num_cus) { + XDNA_ERR(xdna, "Number of CU is zero"); + return -EINVAL; + } + + total_size = struct_size(config, cu_configs, config->num_cus); + if (total_size > size) { + XDNA_ERR(xdna, "CU config larger than size"); + return -EINVAL; + } + + hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); + if (!hwctx->cus) + return -ENOMEM; + + ret = amdxdna_pm_resume_get(xdna); + if (ret) + goto free_cus; + + ret = aie2_config_cu(hwctx, aie2_config_cu_resp_handler); + if (ret) { + XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret); + goto pm_suspend_put; + } + + wmb(); /* To avoid locking in command submit when check status */ + + return 0; + +pm_suspend_put: + amdxdna_pm_suspend_put(xdna); +free_cus: + kfree(hwctx->cus); + hwctx->cus = NULL; + return ret; +} + +static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq) +{ + struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq); + struct amdxdna_dev *xdna = hwctx->client->xdna; + + if (!out_fence) { + XDNA_ERR(xdna, "Failed to get fence"); + return; + } + + mutex_unlock(&xdna->dev_lock); + dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT); + mutex_lock(&xdna->dev_lock); + dma_fence_put(out_fence); +} + +static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, + bool attach) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_drv_cmd *cmd; + struct amdxdna_gem_obj *abo; + u64 seq; + int ret; + + abo = amdxdna_gem_get_obj(client, bo_hdl, AMDXDNA_BO_DEV); + if (!abo) { + XDNA_ERR(xdna, "Get bo %d failed", bo_hdl); + return -EINVAL; + } + + cmd = kzalloc_obj(*cmd); + if (!cmd) { + ret = -ENOMEM; + goto put_obj; + } + kref_init(&cmd->refcnt); + + if (attach) { + if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) { + ret = -EBUSY; + goto put_cmd; + } + cmd->opcode = ATTACH_DEBUG_BO; + } else { + if (abo->assigned_hwctx != hwctx->id) { + ret = -EINVAL; + goto put_cmd; + } + cmd->opcode = DETACH_DEBUG_BO; + } + + ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, + &bo_hdl, 1, hwctx->id, &seq); + if (ret) { + XDNA_ERR(xdna, "Submit command failed"); + goto put_cmd; + } + + aie2_cmd_wait(hwctx, seq); + if (cmd->result) { + XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); + ret = -EINVAL; + goto put_cmd; + } + + if (attach) + abo->assigned_hwctx = hwctx->id; + else + abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; + + XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name); + +put_cmd: + aie2_cmd_put(cmd); +put_obj: + amdxdna_gem_put_obj(abo); + return ret; +} + +int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + switch (type) { + case DRM_AMDXDNA_HWCTX_CONFIG_CU: + return aie2_hwctx_cu_config(hwctx, buf, size); + case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: + return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, true); + case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: + return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, false); + default: + XDNA_DBG(xdna, "Not supported type %d", type); + return -EOPNOTSUPP; + } +} + +int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_drv_cmd *cmd; + u64 seq; + int ret; + + cmd = kzalloc_obj(*cmd); + if (!cmd) + return -ENOMEM; + kref_init(&cmd->refcnt); + + cmd->opcode = SYNC_DEBUG_BO; + ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, + &debug_bo_hdl, 1, hwctx->id, &seq); + if (ret) { + XDNA_ERR(xdna, "Submit command failed"); + goto put_cmd; + } + + aie2_cmd_wait(hwctx, seq); + if (cmd->result) { + XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); + ret = -EINVAL; + } + +put_cmd: + aie2_cmd_put(cmd); + return ret; +} + +static int aie2_populate_range(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct amdxdna_umap *mapp; + unsigned long timeout; + struct mm_struct *mm; + bool found; + int ret; + + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); +again: + found = false; + down_write(&xdna->notifier_lock); + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + /* + * Skip entries that have already been unmapped. + * + * If userspace unmaps the address and later submits I/O using + * it, the IOMMU will reject the access and report a fault. + * Ignore such entries here. + */ + if (mapp->unmapped) + continue; + + if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { + found = true; + break; + } + } + + if (!found) { + /* + * This also covers the case where all mappings have been + * removed. There are no invalid mappings left to process. + * Any subsequent I/O using the unmapped address will be + * rejected by the IOMMU. + */ + abo->mem.map_invalid = false; + up_write(&xdna->notifier_lock); + return 0; + } + + up_write(&xdna->notifier_lock); + + mm = mapp->notifier.mm; + if (!mmget_not_zero(mm)) { + amdxdna_umap_put(mapp); + return -EFAULT; + } + + ret = hmm_range_fault_unlocked_timeout(&mapp->range, timeout); + if (ret) + goto put_mm; + + down_write(&xdna->notifier_lock); + if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { + up_write(&xdna->notifier_lock); + amdxdna_umap_put(mapp); + mmput(mm); + goto again; + } + mapp->invalid = false; + up_write(&xdna->notifier_lock); + amdxdna_umap_put(mapp); + mmput(mm); + goto again; + +put_mm: + amdxdna_umap_put(mapp); + mmput(mm); + return ret == -EBUSY ? -ETIME : ret; +} + +int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct ww_acquire_ctx acquire_ctx; + struct dma_fence_chain *chain; + struct amdxdna_gem_obj *abo; + unsigned long timeout = 0; + int ret, i; + + ret = down_interruptible(&hwctx->priv->job_sem); + if (ret) { + XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret); + return ret; + } + + chain = dma_fence_chain_alloc(); + if (!chain) { + XDNA_ERR(xdna, "Alloc fence chain failed"); + ret = -ENOMEM; + goto up_sem; + } + + ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx, + hwctx->client->filp->client_id); + if (ret) { + XDNA_ERR(xdna, "DRM job init failed, ret %d", ret); + goto free_chain; + } + +retry: + ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); + if (ret) { + XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); + goto cleanup_job; + } + + for (i = 0; i < job->bo_cnt; i++) { + ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); + if (ret) { + XDNA_WARN(xdna, "Failed to reserve fences %d", ret); + drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); + goto cleanup_job; + } + } + + down_read(&xdna->notifier_lock); + for (i = 0; i < job->bo_cnt; i++) { + abo = to_xdna_obj(job->bos[i]); + if (abo->mem.map_invalid) { + up_read(&xdna->notifier_lock); + drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); + if (!timeout) { + timeout = jiffies + + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + } else if (time_after(jiffies, timeout)) { + ret = -ETIME; + goto cleanup_job; + } + + ret = aie2_populate_range(abo); + if (ret) + goto cleanup_job; + goto retry; + } + } + + mutex_lock(&hwctx->priv->io_lock); + drm_sched_job_arm(&job->base); + job->out_fence = dma_fence_get(&job->base.s_fence->finished); + for (i = 0; i < job->bo_cnt; i++) + dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); + job->seq = hwctx->priv->seq++; + kref_get(&job->refcnt); + if (job->drv_cmd) + kref_get(&job->drv_cmd->refcnt); + drm_sched_entity_push_job(&job->base); + + *seq = job->seq; + drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); + mutex_unlock(&hwctx->priv->io_lock); + + up_read(&xdna->notifier_lock); + drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); + + aie2_job_put(job); + atomic64_inc(&hwctx->job_submit_cnt); + + return 0; + +cleanup_job: + drm_sched_job_cleanup(&job->base); +free_chain: + dma_fence_chain_free(chain); +up_sem: + up(&hwctx->priv->job_sem); + job->job_done = true; + return ret; +} + +void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, + unsigned long cur_seq) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct drm_gem_object *gobj = to_gobj(abo); + long ret; + + ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, + true, MAX_SCHEDULE_TIMEOUT); + if (!ret) + XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); + else if (ret == -ERESTARTSYS) + XDNA_DBG(xdna, "Wait for bo interrupted by signal"); +} + +int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, + struct amdxdna_gem_obj *heap) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + u64 addr; + int ret; + + addr = amdxdna_obj_dma_addr(heap); + ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, + addr, heap->mem.size); + if (ret) { + XDNA_ERR(xdna, "Add heap failed hwctx %s 0x%lx ret %d", + hwctx->name, heap->mem.size, ret); + } + + return ret; +} diff --git a/drivers/accel/amdxdna/aie2_error.c b/drivers/accel/amdxdna/aie2_error.c new file mode 100644 index 000000000..babdac015 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_error.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/drm_cache.h> +#include <drm/drm_device.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/dma-mapping.h> +#include <linux/kthread.h> +#include <linux/kernel.h> + +#include "aie.h" +#include "aie2_msg_priv.h" +#include "aie2_pci.h" +#include "amdxdna_error.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +struct async_event { + struct amdxdna_dev_hdl *ndev; + struct async_event_msg_resp resp; + struct workqueue_struct *wq; + struct work_struct work; + u8 *buf; + dma_addr_t addr; + u32 size; +}; + +struct async_events { + struct workqueue_struct *wq; + u8 *buf; + dma_addr_t addr; + u32 size; + u32 event_cnt; + struct async_event event[] __counted_by(event_cnt); +}; + +/* + * Below enum, struct and lookup tables are porting from XAIE util header file. + * + * Below data is defined by AIE device and it is used for decode error message + * from the device. + */ + +enum aie_module_type { + AIE_MEM_MOD = 0, + AIE_CORE_MOD, + AIE_PL_MOD, + AIE_UNKNOWN_MOD, +}; + +enum aie_error_category { + AIE_ERROR_SATURATION = 0, + AIE_ERROR_FP, + AIE_ERROR_STREAM, + AIE_ERROR_ACCESS, + AIE_ERROR_BUS, + AIE_ERROR_INSTRUCTION, + AIE_ERROR_ECC, + AIE_ERROR_LOCK, + AIE_ERROR_DMA, + AIE_ERROR_MEM_PARITY, + /* Unknown is not from XAIE, added for better category */ + AIE_ERROR_UNKNOWN, +}; + +/* Don't pack, unless XAIE side changed */ +struct aie_error { + __u8 row; + __u8 col; + __u32 mod_type; + __u8 event_id; +}; + +struct aie_err_info { + u32 err_cnt; + u32 ret_code; + u32 rsvd; + struct aie_error payload[] __counted_by(err_cnt); +}; + +struct aie_event_category { + u8 event_id; + enum aie_error_category category; +}; + +#define EVENT_CATEGORY(id, cat) { id, cat } +static const struct aie_event_category aie_ml_mem_event_cat[] = { + EVENT_CATEGORY(88U, AIE_ERROR_ECC), + EVENT_CATEGORY(90U, AIE_ERROR_ECC), + EVENT_CATEGORY(91U, AIE_ERROR_MEM_PARITY), + EVENT_CATEGORY(92U, AIE_ERROR_MEM_PARITY), + EVENT_CATEGORY(93U, AIE_ERROR_MEM_PARITY), + EVENT_CATEGORY(94U, AIE_ERROR_MEM_PARITY), + EVENT_CATEGORY(95U, AIE_ERROR_MEM_PARITY), + EVENT_CATEGORY(96U, AIE_ERROR_MEM_PARITY), + EVENT_CATEGORY(97U, AIE_ERROR_DMA), + EVENT_CATEGORY(98U, AIE_ERROR_DMA), + EVENT_CATEGORY(99U, AIE_ERROR_DMA), + EVENT_CATEGORY(100U, AIE_ERROR_DMA), + EVENT_CATEGORY(101U, AIE_ERROR_LOCK), +}; + +static const struct aie_event_category aie_ml_core_event_cat[] = { + EVENT_CATEGORY(55U, AIE_ERROR_ACCESS), + EVENT_CATEGORY(56U, AIE_ERROR_STREAM), + EVENT_CATEGORY(57U, AIE_ERROR_STREAM), + EVENT_CATEGORY(58U, AIE_ERROR_BUS), + EVENT_CATEGORY(59U, AIE_ERROR_INSTRUCTION), + EVENT_CATEGORY(60U, AIE_ERROR_ACCESS), + EVENT_CATEGORY(62U, AIE_ERROR_ECC), + EVENT_CATEGORY(64U, AIE_ERROR_ECC), + EVENT_CATEGORY(65U, AIE_ERROR_ACCESS), + EVENT_CATEGORY(66U, AIE_ERROR_ACCESS), + EVENT_CATEGORY(67U, AIE_ERROR_LOCK), + EVENT_CATEGORY(70U, AIE_ERROR_INSTRUCTION), + EVENT_CATEGORY(71U, AIE_ERROR_STREAM), + EVENT_CATEGORY(72U, AIE_ERROR_BUS), +}; + +static const struct aie_event_category aie_ml_mem_tile_event_cat[] = { + EVENT_CATEGORY(130U, AIE_ERROR_ECC), + EVENT_CATEGORY(132U, AIE_ERROR_ECC), + EVENT_CATEGORY(133U, AIE_ERROR_DMA), + EVENT_CATEGORY(134U, AIE_ERROR_DMA), + EVENT_CATEGORY(135U, AIE_ERROR_STREAM), + EVENT_CATEGORY(136U, AIE_ERROR_STREAM), + EVENT_CATEGORY(137U, AIE_ERROR_STREAM), + EVENT_CATEGORY(138U, AIE_ERROR_BUS), + EVENT_CATEGORY(139U, AIE_ERROR_LOCK), +}; + +static const struct aie_event_category aie_ml_shim_tile_event_cat[] = { + EVENT_CATEGORY(64U, AIE_ERROR_BUS), + EVENT_CATEGORY(65U, AIE_ERROR_STREAM), + EVENT_CATEGORY(66U, AIE_ERROR_STREAM), + EVENT_CATEGORY(67U, AIE_ERROR_BUS), + EVENT_CATEGORY(68U, AIE_ERROR_BUS), + EVENT_CATEGORY(69U, AIE_ERROR_BUS), + EVENT_CATEGORY(70U, AIE_ERROR_BUS), + EVENT_CATEGORY(71U, AIE_ERROR_BUS), + EVENT_CATEGORY(72U, AIE_ERROR_DMA), + EVENT_CATEGORY(73U, AIE_ERROR_DMA), + EVENT_CATEGORY(74U, AIE_ERROR_LOCK), +}; + +static const enum amdxdna_error_num aie_cat_err_num_map[] = { + [AIE_ERROR_SATURATION] = AMDXDNA_ERROR_NUM_AIE_SATURATION, + [AIE_ERROR_FP] = AMDXDNA_ERROR_NUM_AIE_FP, + [AIE_ERROR_STREAM] = AMDXDNA_ERROR_NUM_AIE_STREAM, + [AIE_ERROR_ACCESS] = AMDXDNA_ERROR_NUM_AIE_ACCESS, + [AIE_ERROR_BUS] = AMDXDNA_ERROR_NUM_AIE_BUS, + [AIE_ERROR_INSTRUCTION] = AMDXDNA_ERROR_NUM_AIE_INSTRUCTION, + [AIE_ERROR_ECC] = AMDXDNA_ERROR_NUM_AIE_ECC, + [AIE_ERROR_LOCK] = AMDXDNA_ERROR_NUM_AIE_LOCK, + [AIE_ERROR_DMA] = AMDXDNA_ERROR_NUM_AIE_DMA, + [AIE_ERROR_MEM_PARITY] = AMDXDNA_ERROR_NUM_AIE_MEM_PARITY, + [AIE_ERROR_UNKNOWN] = AMDXDNA_ERROR_NUM_UNKNOWN, +}; + +static_assert(ARRAY_SIZE(aie_cat_err_num_map) == AIE_ERROR_UNKNOWN + 1); + +static const enum amdxdna_error_module aie_err_mod_map[] = { + [AIE_MEM_MOD] = AMDXDNA_ERROR_MODULE_AIE_MEMORY, + [AIE_CORE_MOD] = AMDXDNA_ERROR_MODULE_AIE_CORE, + [AIE_PL_MOD] = AMDXDNA_ERROR_MODULE_AIE_PL, + [AIE_UNKNOWN_MOD] = AMDXDNA_ERROR_MODULE_UNKNOWN, +}; + +static_assert(ARRAY_SIZE(aie_err_mod_map) == AIE_UNKNOWN_MOD + 1); + +static enum aie_error_category +aie_get_error_category(u8 row, u8 event_id, enum aie_module_type mod_type) +{ + const struct aie_event_category *lut; + int num_entry; + int i; + + switch (mod_type) { + case AIE_PL_MOD: + lut = aie_ml_shim_tile_event_cat; + num_entry = ARRAY_SIZE(aie_ml_shim_tile_event_cat); + break; + case AIE_CORE_MOD: + lut = aie_ml_core_event_cat; + num_entry = ARRAY_SIZE(aie_ml_core_event_cat); + break; + case AIE_MEM_MOD: + if (row == 1) { + lut = aie_ml_mem_tile_event_cat; + num_entry = ARRAY_SIZE(aie_ml_mem_tile_event_cat); + } else { + lut = aie_ml_mem_event_cat; + num_entry = ARRAY_SIZE(aie_ml_mem_event_cat); + } + break; + default: + return AIE_ERROR_UNKNOWN; + } + + for (i = 0; i < num_entry; i++) { + if (event_id != lut[i].event_id) + continue; + + if (lut[i].category > AIE_ERROR_UNKNOWN) + return AIE_ERROR_UNKNOWN; + + return lut[i].category; + } + + return AIE_ERROR_UNKNOWN; +} + +static void aie2_update_last_async_error(struct amdxdna_dev_hdl *ndev, void *err_info, u32 num_err) +{ + struct aie_error *errs = err_info; + enum amdxdna_error_module err_mod; + enum aie_error_category aie_err; + enum amdxdna_error_num err_num; + struct aie_error *last_err; + + last_err = &errs[num_err - 1]; + if (last_err->mod_type >= AIE_UNKNOWN_MOD) { + err_num = aie_cat_err_num_map[AIE_ERROR_UNKNOWN]; + err_mod = aie_err_mod_map[AIE_UNKNOWN_MOD]; + } else { + aie_err = aie_get_error_category(last_err->row, + last_err->event_id, + last_err->mod_type); + err_num = aie_cat_err_num_map[aie_err]; + err_mod = aie_err_mod_map[last_err->mod_type]; + } + + ndev->last_async_err.err_code = AMDXDNA_ERROR_ENCODE(err_num, err_mod); + ndev->last_async_err.ts_us = ktime_to_us(ktime_get_real()); + ndev->last_async_err.ex_err_code = AMDXDNA_EXTRA_ERR_ENCODE(last_err->row, last_err->col); +} + +static u32 aie2_error_backtrack(struct amdxdna_dev_hdl *ndev, void *err_info, u32 num_err) +{ + struct aie_error *errs = err_info; + u32 err_col = 0; /* assume that AIE has less than 32 columns */ + int i; + + /* Get err column bitmap */ + for (i = 0; i < num_err; i++) { + struct aie_error *err = &errs[i]; + enum aie_error_category cat; + + cat = aie_get_error_category(err->row, err->event_id, err->mod_type); + XDNA_ERR(ndev->aie.xdna, "Row: %d, Col: %d, module %d, event ID %d, category %d", + err->row, err->col, err->mod_type, + err->event_id, cat); + + if (err->col >= 32) { + XDNA_WARN(ndev->aie.xdna, "Invalid column number"); + break; + } + + err_col |= (1 << err->col); + } + + return err_col; +} + +static int aie2_error_async_cb(void *handle, void __iomem *data, size_t size) +{ + struct async_event *e = handle; + + if (data) { + e->resp.type = readl(data + offsetof(struct async_event_msg_resp, type)); + wmb(); /* Update status in the end, so that no lock for here */ + e->resp.status = readl(data + offsetof(struct async_event_msg_resp, status)); + } + queue_work(e->wq, &e->work); + return 0; +} + +static int aie2_error_event_send(struct async_event *e) +{ + drm_clflush_virt_range(e->buf, e->size); /* device can access */ + return aie2_register_asyn_event_msg(e->ndev, e->addr, e->size, e, + aie2_error_async_cb); +} + +static void aie2_error_worker(struct work_struct *err_work) +{ + struct aie_err_info *info; + struct amdxdna_dev *xdna; + struct async_event *e; + u32 max_err; + u32 err_col; + + e = container_of(err_work, struct async_event, work); + + xdna = e->ndev->aie.xdna; + + if (e->resp.status == MAX_AIE2_STATUS_CODE) + return; + + e->resp.status = MAX_AIE2_STATUS_CODE; + + print_hex_dump_debug("AIE error: ", DUMP_PREFIX_OFFSET, 16, 4, + e->buf, 0x100, false); + + info = (struct aie_err_info *)e->buf; + XDNA_DBG(xdna, "Error count %d return code %d", info->err_cnt, info->ret_code); + + max_err = (e->size - sizeof(*info)) / sizeof(struct aie_error); + if (unlikely(info->err_cnt > max_err)) { + WARN_ONCE(1, "Error count too large %d\n", info->err_cnt); + return; + } + err_col = aie2_error_backtrack(e->ndev, info->payload, info->err_cnt); + if (!err_col) { + XDNA_WARN(xdna, "Did not get error column"); + return; + } + + mutex_lock(&xdna->dev_lock); + aie2_update_last_async_error(e->ndev, info->payload, info->err_cnt); + + /* Re-sent this event to firmware */ + if (aie2_error_event_send(e)) + XDNA_WARN(xdna, "Unable to register async event"); + mutex_unlock(&xdna->dev_lock); +} + +void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct async_events *events; + + events = ndev->async_events; + + mutex_unlock(&xdna->dev_lock); + destroy_workqueue(events->wq); + mutex_lock(&xdna->dev_lock); + + amdxdna_free_msg_buffer(xdna, events->size, events->buf, events->addr); + kfree(events); +} + +int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + u32 total_col = ndev->total_col; + u32 total_size = ASYNC_BUF_SIZE * total_col; + struct async_events *events; + int i, ret; + + events = kzalloc_flex(*events, event, total_col); + if (!events) + return -ENOMEM; + + events->buf = amdxdna_alloc_msg_buffer(xdna, &total_size, &events->addr); + if (IS_ERR(events->buf)) { + ret = PTR_ERR(events->buf); + goto free_events; + } + events->size = total_size; + events->event_cnt = total_col; + + events->wq = alloc_ordered_workqueue("async_wq", 0); + if (!events->wq) { + ret = -ENOMEM; + goto free_buf; + } + + for (i = 0; i < events->event_cnt; i++) { + struct async_event *e = &events->event[i]; + u32 offset = i * ASYNC_BUF_SIZE; + + e->ndev = ndev; + e->wq = events->wq; + e->buf = &events->buf[offset]; + e->addr = events->addr + offset; + e->size = ASYNC_BUF_SIZE; + e->resp.status = MAX_AIE2_STATUS_CODE; + INIT_WORK(&e->work, aie2_error_worker); + + ret = aie2_error_event_send(e); + if (ret) + goto free_wq; + } + + ndev->async_events = events; + + XDNA_DBG(xdna, "Async event count %d, buf total size 0x%x", + events->event_cnt, events->size); + return 0; + +free_wq: + destroy_workqueue(events->wq); +free_buf: + amdxdna_free_msg_buffer(xdna, events->size, events->buf, events->addr); +free_events: + kfree(events); + return ret; +} + +int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, struct amdxdna_drm_get_array *args) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!args->num_element) + return -EINVAL; + + args->num_element = 1; + args->element_size = min(args->element_size, sizeof(ndev->last_async_err)); + if (copy_to_user(u64_to_user_ptr(args->buffer), + &ndev->last_async_err, args->element_size)) + return -EFAULT; + + return 0; +} diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c new file mode 100644 index 000000000..dfe0fbdf0 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_message.c @@ -0,0 +1,1270 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_cache.h> +#include <drm/drm_device.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/bitfield.h> +#include <linux/errno.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <linux/xarray.h> + +#include "aie.h" +#include "aie2_msg_priv.h" +#include "aie2_pci.h" +#include "amdxdna_ctx.h" +#include "amdxdna_gem.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_pci_drv.h" + +#define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) + +int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND); + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret); + return ret; + } + + return aie_psp_waitmode_poll(ndev->aie.psp_hdl); +} + +int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(suspend, MSG_OP_RESUME); + + return aie_send_mgmt_msg_wait(&ndev->aie, &msg); +} + +int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) +{ + DECLARE_AIE_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); + int ret; + + req.type = type; + req.value = value; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Failed to set runtime config, ret %d", ret); + return ret; + } + + return 0; +} + +int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) +{ + DECLARE_AIE_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); + int ret; + + req.type = type; + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Failed to get runtime config, ret %d", ret); + return ret; + } + + *value = resp.value; + return 0; +} + +int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) +{ + DECLARE_AIE_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); + + req.pasid = pasid; + + return aie_send_mgmt_msg_wait(&ndev->aie, &msg); +} + +int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_query_aie_version *version) +{ + DECLARE_AIE_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + return ret; + + XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", + resp.major, resp.minor); + + version->major = resp.major; + version->minor = resp.minor; + + return 0; +} + +int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_query_aie_metadata *metadata) +{ + DECLARE_AIE_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + return ret; + + metadata->col_size = resp.info.size; + metadata->cols = resp.info.cols; + metadata->rows = resp.info.rows; + + metadata->version.major = resp.info.major; + metadata->version.minor = resp.info.minor; + + metadata->core.row_count = resp.info.core_rows; + metadata->core.row_start = resp.info.core_row_start; + metadata->core.dma_channel_count = resp.info.core_dma_channels; + metadata->core.lock_count = resp.info.core_locks; + metadata->core.event_reg_count = resp.info.core_events; + + metadata->mem.row_count = resp.info.mem_rows; + metadata->mem.row_start = resp.info.mem_row_start; + metadata->mem.dma_channel_count = resp.info.mem_dma_channels; + metadata->mem.lock_count = resp.info.mem_locks; + metadata->mem.event_reg_count = resp.info.mem_events; + + metadata->shim.row_count = resp.info.shim_rows; + metadata->shim.row_start = resp.info.shim_row_start; + metadata->shim.dma_channel_count = resp.info.shim_dma_channels; + metadata->shim.lock_count = resp.info.shim_locks; + metadata->shim.event_reg_count = resp.info.shim_events; + + return 0; +} + +int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, + struct amdxdna_fw_ver *fw_ver) +{ + DECLARE_AIE_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + return ret; + + fw_ver->major = resp.major; + fw_ver->minor = resp.minor; + fw_ver->sub = resp.sub; + fw_ver->build = resp.build; + + return 0; +} + +static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) +{ + DECLARE_AIE_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + req.context_id = id; + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret && ret != -ENODEV) + XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); + else if (ret == -ENODEV) + XDNA_DBG(xdna, "Destroy context: device already stopped"); + + return ret; +} + +static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, + struct amdxdna_hwctx *hwctx) +{ + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_PREEMPT)) + return PRIORITY_HIGH; + + switch (hwctx->qos.priority) { + case AMDXDNA_QOS_REALTIME_PRIORITY: + return PRIORITY_REALTIME; + case AMDXDNA_QOS_HIGH_PRIORITY: + return PRIORITY_HIGH; + case AMDXDNA_QOS_NORMAL_PRIORITY: + return PRIORITY_NORMAL; + case AMDXDNA_QOS_LOW_PRIORITY: + return PRIORITY_LOW; + default: + return PRIORITY_HIGH; + } +} + +int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) +{ + DECLARE_AIE_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct xdna_mailbox_chann_res x2i; + struct xdna_mailbox_chann_res i2x; + struct cq_pair *cq_pair; + u32 intr_reg; + int ret; + + req.aie_type = 1; + req.start_col = hwctx->start_col; + req.num_col = hwctx->num_col; + req.num_unused_col = hwctx->num_unused_col; + req.num_cq_pairs_requested = 1; + req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0; + req.context_priority = aie2_get_context_priority(ndev, hwctx); + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + return ret; + + hwctx->fw_ctx_id = resp.context_id; + if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) + return -EINVAL; + + if (ndev->force_preempt_enabled) { + ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); + if (ret) { + XDNA_ERR(xdna, "failed to enable force preempt %d", ret); + goto del_ctx_req; + } + } + + cq_pair = &resp.cq_pair[0]; + x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); + x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); + x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); + x2i.rb_size = cq_pair->x2i_q.buf_size; + + i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); + i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); + i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); + i2x.rb_size = cq_pair->i2x_q.buf_size; + + ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); + if (ret == -EINVAL) { + XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); + goto del_ctx_req; + } + + intr_reg = i2x.mb_head_ptr_reg + 4; + hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox); + if (!hwctx->priv->mbox_chann) { + XDNA_ERR(xdna, "Not able to create channel"); + ret = -EINVAL; + goto del_ctx_req; + } + + ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x, + intr_reg, ret); + if (ret) { + XDNA_ERR(xdna, "Not able to create channel"); + ret = -EINVAL; + goto free_channel; + } + ndev->hwctx_num++; + + XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); + XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); + + return 0; + +free_channel: + xdna_mailbox_free_channel(hwctx->priv->mbox_chann); +del_ctx_req: + aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); + return ret; +} + +int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + if (!hwctx->priv->mbox_chann) + return 0; + + xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); + ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); + xdna_mailbox_free_channel(hwctx->priv->mbox_chann); + XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); + hwctx->priv->mbox_chann = NULL; + hwctx->fw_ctx_id = -1; + ndev->hwctx_num--; + + return ret; +} + +static int aie2_send_host_buf_msgs(struct amdxdna_dev_hdl *ndev, u32 context_id, + u64 addr, u64 size, u32 initial_opcode) +{ + DECLARE_AIE_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); + struct amdxdna_dev *xdna = ndev->aie.xdna; + size_t chunk_size; + int ret; + + chunk_size = xdna->dev_info->dev_mem_size; + if (!size || !IS_ALIGNED(size, chunk_size)) { + XDNA_ERR(xdna, "Invalid size 0x%llx for chunk 0x%lx", + size, chunk_size); + return -EINVAL; + } + + msg.opcode = initial_opcode; + do { + req.context_id = context_id; + req.buf_addr = addr; + req.buf_size = chunk_size; + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "fw ctx %d addr 0x%llx size 0x%lx", + context_id, addr, chunk_size); + return ret; + } + + XDNA_DBG(xdna, "fw ctx %d host buf op 0x%x addr 0x%llx size 0x%lx", + context_id, msg.opcode, addr, chunk_size); + + addr += chunk_size; + size -= chunk_size; + msg.opcode = MSG_OP_ADD_HOST_BUFFER; + } while (size); + + return 0; +} + +int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) +{ + return aie2_send_host_buf_msgs(ndev, context_id, addr, size, + MSG_OP_MAP_HOST_BUFFER); +} + +int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) +{ + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_ADD_HOST_BUFFER)) + return -EOPNOTSUPP; + + return aie2_send_host_buf_msgs(ndev, context_id, addr, size, + MSG_OP_ADD_HOST_BUFFER); +} + +static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) +{ + u32 *bitmap = arg; + + *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col); + + return 0; +} + +int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, + u32 size, u32 *cols_filled) +{ + DECLARE_AIE_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); + struct amdxdna_dev *xdna = ndev->aie.xdna; + u32 buf_sz, aie_bitmap = 0; + struct amdxdna_client *client; + dma_addr_t dma_addr; + u8 *buff_addr; + int ret; + + buf_sz = ndev->aie.metadata.cols * ndev->aie.metadata.col_size; + buff_addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr); + if (IS_ERR(buff_addr)) + return PTR_ERR(buff_addr); + + /* Go through each hardware context and mark the AIE columns that are active */ + list_for_each_entry(client, &xdna->client_list, node) + amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map); + + *cols_filled = 0; + req.dump_buff_addr = dma_addr; + req.dump_buff_size = buf_sz; + req.num_cols = hweight32(aie_bitmap); + req.aie_bitmap = aie_bitmap; + + drm_clflush_virt_range(buff_addr, req.dump_buff_size); /* device can access */ + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "Error during NPU query, status %d", ret); + goto fail; + } + + XDNA_DBG(xdna, "Query NPU status completed"); + + if (buf_sz < resp.size) { + ret = -EINVAL; + XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); + goto fail; + } + + size = min(size, resp.size); + if (copy_to_user(buf, buff_addr, size)) { + ret = -EFAULT; + XDNA_ERR(xdna, "Failed to copy NPU status to user space"); + goto fail; + } + + *cols_filled = aie_bitmap; + +fail: + amdxdna_free_msg_buffer(xdna, buf_sz, buff_addr, dma_addr); + return ret; +} + +int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, + char __user *buf, u32 size, + struct amdxdna_drm_query_telemetry_header *header) +{ + DECLARE_AIE_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); + struct amdxdna_dev *xdna = ndev->aie.xdna; + dma_addr_t dma_addr; + u32 buf_sz; + u8 *addr; + int ret; + + if (header->type >= MAX_TELEMETRY_TYPE) + return -EINVAL; + + buf_sz = min(size, SZ_4M); + addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr); + if (IS_ERR(addr)) + return PTR_ERR(addr); + + req.buf_addr = dma_addr; + req.buf_size = buf_sz; + req.type = header->type; + + drm_clflush_virt_range(addr, req.buf_size); /* device can access */ + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); + goto free_buf; + } + + if (buf_sz < resp.size) { + ret = -EINVAL; + XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); + goto free_buf; + } + + size = min(size, resp.size); + if (copy_to_user(buf, addr, size)) { + ret = -EFAULT; + XDNA_ERR(xdna, "Failed to copy telemetry to user space"); + goto free_buf; + } + + header->major = resp.major; + header->minor = resp.minor; + +free_buf: + amdxdna_free_msg_buffer(xdna, buf_sz, addr, dma_addr); + return ret; +} + +int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, + void *handle, int (*cb)(void*, void __iomem *, size_t)) +{ + struct async_event_msg_req req = { 0 }; + struct xdna_mailbox_msg msg = { + .send_data = (u8 *)&req, + .send_size = sizeof(req), + .handle = handle, + .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, + .notify_cb = cb, + }; + + req.buf_addr = addr; + req.buf_size = size; + + XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size); + return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg, TX_TIMEOUT); +} + +int aie2_config_cu(struct amdxdna_hwctx *hwctx, + int (*notify_cb)(void *, void __iomem *, size_t)) +{ + struct mailbox_channel *chann = hwctx->priv->mbox_chann; + struct amdxdna_dev *xdna = hwctx->client->xdna; + u32 shift = xdna->dev_info->dev_mem_buf_shift; + struct config_cu_req req = { 0 }; + struct xdna_mailbox_msg msg; + struct drm_gem_object *gobj; + struct amdxdna_gem_obj *abo; + int i; + + if (!chann) + return -ENODEV; + + if (!hwctx->cus) + return 0; + + if (hwctx->cus->num_cus > MAX_NUM_CUS) { + XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); + return -EINVAL; + } + + for (i = 0; i < hwctx->cus->num_cus; i++) { + struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; + + if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) + return -EINVAL; + + gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); + if (!gobj) { + XDNA_ERR(xdna, "Lookup GEM object failed"); + return -EINVAL; + } + abo = to_xdna_obj(gobj); + + if (abo->type != AMDXDNA_BO_DEV) { + drm_gem_object_put(gobj); + XDNA_ERR(xdna, "Invalid BO type"); + return -EINVAL; + } + + req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, + amdxdna_gem_dev_addr(abo) >> shift); + req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); + XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, + amdxdna_gem_dev_addr(abo), req.cfgs[i]); + drm_gem_object_put(gobj); + } + req.num_cus = hwctx->cus->num_cus; + + msg.send_data = (u8 *)&req; + msg.send_size = sizeof(req); + msg.handle = hwctx; + msg.opcode = MSG_OP_CONFIG_CU; + msg.notify_cb = notify_cb; + return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); +} + +static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req, + size_t *size, u32 *msg_op) +{ + struct execute_buffer_req *cu_req = req; + u32 cmd_len; + void *cmd; + + cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + if (cmd_len > sizeof(cu_req->payload)) + return -EINVAL; + + cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (cu_req->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + memcpy(cu_req->payload, cmd, cmd_len); + + *size = sizeof(*cu_req); + *msg_op = MSG_OP_EXECUTE_BUFFER_CF; + return 0; +} + +static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req, + size_t *size, u32 *msg_op) +{ + struct exec_dpu_req *dpu_req = req; + struct amdxdna_cmd_start_npu *sn; + u32 cmd_len; + + sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload)) + return -EINVAL; + + dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (dpu_req->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + dpu_req->inst_buf_addr = sn->buffer; + dpu_req->inst_size = sn->buffer_size; + dpu_req->inst_prop_cnt = sn->prop_count; + memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn)); + + *size = sizeof(*dpu_req); + *msg_op = MSG_OP_EXEC_DPU; + return 0; +} + +static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) +{ + struct cmd_chain_req *chain_req = req; + + chain_req->buf_addr = slot_addr; + chain_req->buf_size = size; + chain_req->count = cmd_cnt; +} + +static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) +{ + struct cmd_chain_npu_req *npu_chain_req = req; + + npu_chain_req->flags = 0; + npu_chain_req->reserved = 0; + npu_chain_req->buf_addr = slot_addr; + npu_chain_req->buf_size = size; + npu_chain_req->count = cmd_cnt; +} + +static int +aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + struct cmd_chain_slot_execbuf_cf *cf_slot = slot; + u32 cmd_len; + void *cmd; + + cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + if (*size < sizeof(*cf_slot) + cmd_len) + return -EINVAL; + + cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (cf_slot->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + cf_slot->arg_cnt = cmd_len / sizeof(u32); + memcpy(cf_slot->args, cmd, cmd_len); + /* Accurate slot size to hint firmware to do necessary copy */ + *size = sizeof(*cf_slot) + cmd_len; + return 0; +} + +static int +aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + struct cmd_chain_slot_dpu *dpu_slot = slot; + struct amdxdna_cmd_start_npu *sn; + u32 cmd_len; + u32 arg_sz; + + sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + arg_sz = cmd_len - sizeof(*sn); + if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) + return -EINVAL; + + if (*size < sizeof(*dpu_slot) + arg_sz) + return -EINVAL; + + dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (dpu_slot->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + dpu_slot->inst_buf_addr = sn->buffer; + dpu_slot->inst_size = sn->buffer_size; + dpu_slot->inst_prop_cnt = sn->prop_count; + dpu_slot->arg_cnt = arg_sz / sizeof(u32); + memcpy(dpu_slot->args, sn->prop_args, arg_sz); + + /* Accurate slot size to hint firmware to do necessary copy */ + *size = sizeof(*dpu_slot) + arg_sz; + return 0; +} + +static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + return -EOPNOTSUPP; +} + +static u32 aie2_get_chain_msg_op(u32 cmd_op) +{ + switch (cmd_op) { + case ERT_START_CU: + return MSG_OP_CHAIN_EXEC_BUFFER_CF; + case ERT_START_NPU: + return MSG_OP_CHAIN_EXEC_DPU; + default: + break; + } + + return MSG_OP_MAX_OPCODE; +} + +static struct aie2_exec_msg_ops legacy_exec_message_ops = { + .init_cu_req = aie2_init_exec_cu_req, + .init_dpu_req = aie2_init_exec_dpu_req, + .init_chain_req = aie2_init_exec_chain_req, + .fill_cf_slot = aie2_cmdlist_fill_cf, + .fill_dpu_slot = aie2_cmdlist_fill_dpu, + .fill_preempt_slot = aie2_cmdlist_unsupp, + .fill_elf_slot = aie2_cmdlist_unsupp, + .get_chain_msg_op = aie2_get_chain_msg_op, +}; + +static int +aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + struct cmd_chain_slot_npu *npu_slot = slot; + u32 cmd_len; + void *cmd; + + cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + if (*size < sizeof(*npu_slot) + cmd_len) + return -EINVAL; + + memset(npu_slot, 0, sizeof(*npu_slot)); + npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (npu_slot->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + npu_slot->type = EXEC_NPU_TYPE_NON_ELF; + npu_slot->arg_cnt = cmd_len / sizeof(u32); + memcpy(npu_slot->args, cmd, cmd_len); + + *size = sizeof(*npu_slot) + cmd_len; + return 0; +} + +static int +aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + struct cmd_chain_slot_npu *npu_slot = slot; + struct amdxdna_cmd_start_npu *sn; + u32 cmd_len; + u32 arg_sz; + + sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + arg_sz = cmd_len - sizeof(*sn); + if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) + return -EINVAL; + + if (*size < sizeof(*npu_slot) + arg_sz) + return -EINVAL; + + memset(npu_slot, 0, sizeof(*npu_slot)); + npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (npu_slot->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; + npu_slot->inst_buf_addr = sn->buffer; + npu_slot->inst_size = sn->buffer_size; + npu_slot->inst_prop_cnt = sn->prop_count; + npu_slot->arg_cnt = arg_sz / sizeof(u32); + memcpy(npu_slot->args, sn->prop_args, arg_sz); + + *size = sizeof(*npu_slot) + arg_sz; + return 0; +} + +static int +aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + struct cmd_chain_slot_npu *npu_slot = slot; + struct amdxdna_cmd_preempt_data *pd; + u32 cmd_len; + u32 arg_sz; + + pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + arg_sz = cmd_len - sizeof(*pd); + if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) + return -EINVAL; + + if (*size < sizeof(*npu_slot) + arg_sz) + return -EINVAL; + + memset(npu_slot, 0, sizeof(*npu_slot)); + npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); + if (npu_slot->cu_idx == INVALID_CU_IDX) + return -EINVAL; + + npu_slot->type = EXEC_NPU_TYPE_PREEMPT; + npu_slot->inst_buf_addr = pd->inst_buf; + npu_slot->save_buf_addr = pd->save_buf; + npu_slot->restore_buf_addr = pd->restore_buf; + npu_slot->inst_size = pd->inst_size; + npu_slot->save_size = pd->save_size; + npu_slot->restore_size = pd->restore_size; + npu_slot->inst_prop_cnt = pd->inst_prop_cnt; + npu_slot->arg_cnt = arg_sz / sizeof(u32); + memcpy(npu_slot->args, pd->prop_args, arg_sz); + + *size = sizeof(*npu_slot) + arg_sz; + return 0; +} + +static int +aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) +{ + struct cmd_chain_slot_npu *npu_slot = slot; + struct amdxdna_cmd_preempt_data *pd; + u32 cmd_len; + u32 arg_sz; + + pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); + arg_sz = cmd_len - sizeof(*pd); + if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) + return -EINVAL; + + if (*size < sizeof(*npu_slot) + arg_sz) + return -EINVAL; + + memset(npu_slot, 0, sizeof(*npu_slot)); + npu_slot->type = EXEC_NPU_TYPE_ELF; + npu_slot->inst_buf_addr = pd->inst_buf; + npu_slot->save_buf_addr = pd->save_buf; + npu_slot->restore_buf_addr = pd->restore_buf; + npu_slot->inst_size = pd->inst_size; + npu_slot->save_size = pd->save_size; + npu_slot->restore_size = pd->restore_size; + npu_slot->inst_prop_cnt = pd->inst_prop_cnt; + npu_slot->arg_cnt = 1; + npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN; + + *size = struct_size(npu_slot, args, npu_slot->arg_cnt); + return 0; +} + +static u32 aie2_get_npu_chain_msg_op(u32 cmd_op) +{ + return MSG_OP_CHAIN_EXEC_NPU; +} + +static struct aie2_exec_msg_ops npu_exec_message_ops = { + .init_cu_req = aie2_init_exec_cu_req, + .init_dpu_req = aie2_init_exec_dpu_req, + .init_chain_req = aie2_init_npu_chain_req, + .fill_cf_slot = aie2_cmdlist_fill_npu_cf, + .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu, + .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt, + .fill_elf_slot = aie2_cmdlist_fill_npu_elf, + .get_chain_msg_op = aie2_get_npu_chain_msg_op, +}; + +static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, + size_t *size, u32 *msg_op) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev); + int ret; + u32 op; + + op = amdxdna_cmd_get_op(cmd_abo); + switch (op) { + case ERT_START_CU: + ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op); + if (ret) { + XDNA_DBG(xdna, "Init CU req failed ret %d", ret); + return ret; + } + break; + case ERT_START_NPU: + ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op); + if (ret) { + XDNA_DBG(xdna, "Init DPU req failed ret %d", ret); + return ret; + } + + break; + default: + XDNA_ERR(xdna, "Unsupported op %d", op); + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +static int +aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, + size_t *size, u32 *cmd_op) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev); + int ret; + u32 op; + + op = amdxdna_cmd_get_op(cmd_abo); + if (*cmd_op == ERT_INVALID_CMD) + *cmd_op = op; + else if (op != *cmd_op) + return -EINVAL; + + switch (op) { + case ERT_START_CU: + ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size); + break; + case ERT_START_NPU: + ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); + break; + case ERT_START_NPU_PREEMPT: + if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) + return -EOPNOTSUPP; + ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size); + break; + case ERT_START_NPU_PREEMPT_ELF: + if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) + return -EOPNOTSUPP; + ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size); + break; + default: + XDNA_INFO(xdna, "Unsupported op %d", op); + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +void aie2_msg_init(struct amdxdna_dev_hdl *ndev) +{ + if (AIE_FEATURE_ON(&ndev->aie, AIE2_NPU_COMMAND)) + ndev->exec_msg_ops = &npu_exec_message_ops; + else + ndev->exec_msg_ops = &legacy_exec_message_ops; +} + +static inline struct amdxdna_gem_obj * +aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) +{ + int idx = get_job_idx(job->seq); + + return job->hwctx->priv->cmd_buf[idx]; +} + +int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)) +{ + struct mailbox_channel *chann = hwctx->priv->mbox_chann; + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; + struct xdna_mailbox_msg msg; + union exec_req req; + int ret; + + if (!chann) + return -ENODEV; + + ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode); + if (ret) + return ret; + + msg.handle = job; + msg.notify_cb = notify_cb; + msg.send_data = (u8 *)&req; + print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, + 0x40, false); + + ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); + if (ret) { + XDNA_ERR(xdna, "Send message failed"); + return ret; + } + + return 0; +} + +int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, + struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)) +{ + struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); + struct mailbox_channel *chann = hwctx->priv->mbox_chann; + struct amdxdna_client *client = hwctx->client; + struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; + void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo); + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_cmd_chain *payload; + struct xdna_mailbox_msg msg; + union exec_chain_req req; + u32 payload_len, ccnt; + u32 offset = 0; + size_t size; + int ret; + u32 op; + u32 i; + + if (!cmd_buf) + return -ENOMEM; + + op = amdxdna_cmd_get_op(cmd_abo); + payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); + if (op != ERT_CMD_CHAIN) { + XDNA_DBG(xdna, "Invalid op code %d", op); + return -EINVAL; + } + + if (!payload) { + XDNA_DBG(xdna, "Failed to get command payload"); + return -EINVAL; + } + + ccnt = payload->command_count; + if (payload_len < struct_size(payload, data, ccnt)) { + XDNA_DBG(xdna, "Invalid command count %d", ccnt); + return -EINVAL; + } + + op = ERT_INVALID_CMD; + for (i = 0; i < ccnt; i++) { + u32 boh = (u32)(payload->data[i]); + struct amdxdna_gem_obj *abo; + + abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_SHARE); + if (!abo) { + XDNA_ERR(xdna, "Failed to find cmd BO %d", boh); + return -ENOENT; + } + + size = cmdbuf_abo->mem.size - offset; + ret = aie2_cmdlist_fill_slot(cmd_buf + offset, abo, &size, &op); + amdxdna_gem_put_obj(abo); + if (ret) + return ret; + + offset += size; + } + + XDNA_DBG(xdna, "Total %d commands:", ccnt); + print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4, + cmd_buf, offset, false); + + msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); + if (msg.opcode == MSG_OP_MAX_OPCODE) + return -EOPNOTSUPP; + + /* The offset is the accumulated total size of the cmd buffer */ + EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), + offset, ccnt); + drm_clflush_virt_range(cmd_buf, offset); + + msg.handle = job; + msg.notify_cb = notify_cb; + msg.send_data = (u8 *)&req; + msg.send_size = sizeof(req); + print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, + &req, msg.send_size, false); + ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); + if (ret) { + XDNA_ERR(xdna, "Send message failed"); + return ret; + } + + return 0; +} + +int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, + struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)) +{ + struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); + struct mailbox_channel *chann = hwctx->priv->mbox_chann; + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; + void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo); + struct xdna_mailbox_msg msg; + union exec_chain_req req; + u32 op = ERT_INVALID_CMD; + size_t size; + int ret; + + if (!cmd_buf) + return -ENOMEM; + + size = cmdbuf_abo->mem.size; + ret = aie2_cmdlist_fill_slot(cmd_buf, cmd_abo, &size, &op); + if (ret) + return ret; + + print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4, cmd_buf, size, false); + + msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); + if (msg.opcode == MSG_OP_MAX_OPCODE) + return -EOPNOTSUPP; + + EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), size, 1); + drm_clflush_virt_range(cmd_buf, size); + + msg.handle = job; + msg.notify_cb = notify_cb; + msg.send_data = (u8 *)&req; + msg.send_size = sizeof(req); + print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, + &req, msg.send_size, false); + ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); + if (ret) { + XDNA_ERR(hwctx->client->xdna, "Send message failed"); + return ret; + } + + return 0; +} + +int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)) +{ + struct mailbox_channel *chann = hwctx->priv->mbox_chann; + struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct xdna_mailbox_msg msg; + struct sync_bo_req req; + int ret = 0; + + req.src_addr = 0; + req.dst_addr = amdxdna_dev_bo_offset(abo); + req.size = abo->mem.size; + + /* Device to Host */ + req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | + FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); + + XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", + req.size, req.src_addr, req.dst_addr); + + msg.handle = job; + msg.notify_cb = notify_cb; + msg.send_data = (u8 *)&req; + msg.send_size = sizeof(req); + msg.opcode = MSG_OP_SYNC_BO; + + ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); + if (ret) { + XDNA_ERR(xdna, "Send message failed"); + return ret; + } + + return 0; +} + +int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)) +{ + struct mailbox_channel *chann = hwctx->priv->mbox_chann; + struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct config_debug_bo_req req; + struct xdna_mailbox_msg msg; + + if (job->drv_cmd->opcode == ATTACH_DEBUG_BO) + req.config = DEBUG_BO_REGISTER; + else + req.config = DEBUG_BO_UNREGISTER; + + req.offset = amdxdna_dev_bo_offset(abo); + req.size = abo->mem.size; + + XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d", + req.offset, req.size, req.config); + + msg.handle = job; + msg.notify_cb = notify_cb; + msg.send_data = (u8 *)&req; + msg.send_size = sizeof(req); + msg.opcode = MSG_OP_CONFIG_DEBUG_BO; + + return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); +} + +int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, + struct app_health_report *report) +{ + DECLARE_AIE_MSG(get_app_health, MSG_OP_GET_APP_HEALTH); + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct app_health_report *buf; + dma_addr_t dma_addr; + u32 buf_size; + int ret; + + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_APP_HEALTH)) { + XDNA_DBG(xdna, "App health feature not supported"); + return -EOPNOTSUPP; + } + + buf_size = sizeof(*report); + buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, &dma_addr); + if (IS_ERR(buf)) { + XDNA_ERR(xdna, "Failed to allocate buffer for app health"); + return PTR_ERR(buf); + } + + req.buf_addr = dma_addr; + req.context_id = context_id; + req.buf_size = buf_size; + + drm_clflush_virt_range(buf, req.buf_size); + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status); + goto free_buf; + } + + /* Copy the report to caller's buffer */ + memcpy(report, buf, sizeof(*report)); + +free_buf: + amdxdna_free_msg_buffer(xdna, buf_size, buf, dma_addr); + return ret; +} + +static int +aie2_runtime_update_ctx_prop(struct amdxdna_dev_hdl *ndev, + struct amdxdna_hwctx *ctx, u32 type, u32 value) +{ + DECLARE_AIE_MSG(update_property, MSG_OP_UPDATE_PROPERTY); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_UPDATE_PROPERTY)) + return -EOPNOTSUPP; + + if (ctx) + req.context_id = ctx->fw_ctx_id; + else + req.context_id = AIE2_UPDATE_PROPERTY_ALL_CTX; + + req.time_quota_us = value; + req.type = type; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "%s update property failed, type %d ret %d", + ctx ? ctx->name : "ctx.all", type, ret); + return ret; + } + + return 0; +} + +int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + ret = aie2_runtime_update_ctx_prop(ndev, NULL, UPDATE_PROPERTY_TIME_QUOTA, us); + if (ret == -EOPNOTSUPP) { + XDNA_DBG(xdna, "update time quota not support, skipped"); + ret = 0; + } else if (!ret) { + XDNA_DBG(xdna, "Ctx exec time quantum updated to %u us", us); + } + return ret; +} + +int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev) +{ + DECLARE_AIE_MSG(get_dev_revision, MSG_OP_GET_DEV_REVISION); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + if (!AIE_FEATURE_ON(&ndev->aie, AIE2_GET_DEV_REVISION)) + return -EOPNOTSUPP; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + return ret; + + *rev = resp.rev; + + if (*rev < AIE2_DEV_REVISION_STXA || *rev >= AIE2_DEV_REVISION_UNKN) { + XDNA_ERR(xdna, "Unknown device revision: %d (raw fuse: 0x%x)", + *rev, resp.raw_fuse_data); + return -EINVAL; + } + + XDNA_DBG(xdna, "Device revision: %d (raw fuse: 0x%x)", *rev, resp.raw_fuse_data); + return 0; +} diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h new file mode 100644 index 000000000..fd65a4236 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_msg_priv.h @@ -0,0 +1,546 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE2_MSG_PRIV_H_ +#define _AIE2_MSG_PRIV_H_ + +enum aie2_msg_opcode { + MSG_OP_CREATE_CONTEXT = 0x2, + MSG_OP_DESTROY_CONTEXT = 0x3, + MSG_OP_GET_TELEMETRY = 0x4, + MSG_OP_SYNC_BO = 0x7, + MSG_OP_EXECUTE_BUFFER_CF = 0xC, + MSG_OP_QUERY_COL_STATUS = 0xD, + MSG_OP_QUERY_AIE_TILE_INFO = 0xE, + MSG_OP_QUERY_AIE_VERSION = 0xF, + MSG_OP_EXEC_DPU = 0x10, + MSG_OP_CONFIG_CU = 0x11, + MSG_OP_CHAIN_EXEC_BUFFER_CF = 0x12, + MSG_OP_CHAIN_EXEC_DPU = 0x13, + MSG_OP_CONFIG_DEBUG_BO = 0x14, + MSG_OP_CHAIN_EXEC_NPU = 0x18, + MSG_OP_MAX_XRT_OPCODE, + MSG_OP_SUSPEND = 0x101, + MSG_OP_RESUME = 0x102, + MSG_OP_ASSIGN_MGMT_PASID = 0x103, + MSG_OP_INVOKE_SELF_TEST = 0x104, + MSG_OP_MAP_HOST_BUFFER = 0x106, + MSG_OP_GET_FIRMWARE_VERSION = 0x108, + MSG_OP_SET_RUNTIME_CONFIG = 0x10A, + MSG_OP_GET_RUNTIME_CONFIG = 0x10B, + MSG_OP_REGISTER_ASYNC_EVENT_MSG = 0x10C, + MSG_OP_UPDATE_PROPERTY = 0x113, + MSG_OP_GET_APP_HEALTH = 0x114, + MSG_OP_ADD_HOST_BUFFER = 0x115, + MSG_OP_GET_DEV_REVISION = 0x117, + MSG_OP_MAX_DRV_OPCODE, + MSG_OP_GET_PROTOCOL_VERSION = 0x301, + MSG_OP_MAX_OPCODE +}; + +enum aie2_msg_status { + AIE2_STATUS_SUCCESS = 0x0, + /* AIE Error codes */ + AIE2_STATUS_AIE_SATURATION_ERROR = 0x1000001, + AIE2_STATUS_AIE_FP_ERROR = 0x1000002, + AIE2_STATUS_AIE_STREAM_ERROR = 0x1000003, + AIE2_STATUS_AIE_ACCESS_ERROR = 0x1000004, + AIE2_STATUS_AIE_BUS_ERROR = 0x1000005, + AIE2_STATUS_AIE_INSTRUCTION_ERROR = 0x1000006, + AIE2_STATUS_AIE_ECC_ERROR = 0x1000007, + AIE2_STATUS_AIE_LOCK_ERROR = 0x1000008, + AIE2_STATUS_AIE_DMA_ERROR = 0x1000009, + AIE2_STATUS_AIE_MEM_PARITY_ERROR = 0x100000a, + AIE2_STATUS_AIE_PWR_CFG_ERROR = 0x100000b, + AIE2_STATUS_AIE_BACKTRACK_ERROR = 0x100000c, + AIE2_STATUS_MAX_AIE_STATUS_CODE, + /* MGMT ERT Error codes */ + AIE2_STATUS_MGMT_ERT_SELF_TEST_FAILURE = 0x2000001, + AIE2_STATUS_MGMT_ERT_HASH_MISMATCH, + AIE2_STATUS_MGMT_ERT_NOAVAIL, + AIE2_STATUS_MGMT_ERT_INVALID_PARAM, + AIE2_STATUS_MGMT_ERT_ENTER_SUSPEND_FAILURE, + AIE2_STATUS_MGMT_ERT_BUSY, + AIE2_STATUS_MGMT_ERT_APPLICATION_ACTIVE, + MAX_MGMT_ERT_STATUS_CODE, + /* APP ERT Error codes */ + AIE2_STATUS_APP_ERT_FIRST_ERROR = 0x3000001, + AIE2_STATUS_APP_INVALID_INSTR, + AIE2_STATUS_APP_LOAD_PDI_FAIL, + MAX_APP_ERT_STATUS_CODE, + /* NPU RTOS Error Codes */ + AIE2_STATUS_INVALID_INPUT_BUFFER = 0x4000001, + AIE2_STATUS_INVALID_COMMAND, + AIE2_STATUS_INVALID_PARAM, + AIE2_STATUS_INVALID_OPERATION = 0x4000006, + AIE2_STATUS_ASYNC_EVENT_MSGS_FULL, + AIE2_STATUS_MAX_RTOS_STATUS_CODE, + MAX_AIE2_STATUS_CODE +}; + +struct assign_mgmt_pasid_req { + __u16 pasid; + __u16 reserved; +} __packed; + +struct assign_mgmt_pasid_resp { + enum aie2_msg_status status; +} __packed; + +struct map_host_buffer_req { + __u32 context_id; + __u64 buf_addr; + __u64 buf_size; +} __packed; + +struct map_host_buffer_resp { + enum aie2_msg_status status; +} __packed; + +#define MAX_CQ_PAIRS 2 +struct cq_info { + __u32 head_addr; + __u32 tail_addr; + __u32 buf_addr; + __u32 buf_size; +}; + +struct cq_pair { + struct cq_info x2i_q; + struct cq_info i2x_q; +}; + +#define PRIORITY_REALTIME 1 +#define PRIORITY_HIGH 2 +#define PRIORITY_NORMAL 3 +#define PRIORITY_LOW 4 + +struct create_ctx_req { + __u32 aie_type; + __u8 start_col; + __u8 num_col; + __u8 num_unused_col; + __u8 reserved; + __u8 num_cq_pairs_requested; + __u8 reserved1; + __u16 pasid; + __u32 pad[2]; + __u32 sec_comm_target_type; + __u32 context_priority; +} __packed; + +struct create_ctx_resp { + enum aie2_msg_status status; + __u32 context_id; + __u16 msix_id; + __u8 num_cq_pairs_allocated; + __u8 reserved; + struct cq_pair cq_pair[MAX_CQ_PAIRS]; +} __packed; + +struct destroy_ctx_req { + __u32 context_id; +} __packed; + +struct destroy_ctx_resp { + enum aie2_msg_status status; +} __packed; + +enum telemetry_type { + TELEMETRY_TYPE_DISABLED, + TELEMETRY_TYPE_HEALTH, + TELEMETRY_TYPE_ERROR_INFO, + TELEMETRY_TYPE_PROFILING, + TELEMETRY_TYPE_DEBUG, + MAX_TELEMETRY_TYPE +}; + +struct get_telemetry_req { + enum telemetry_type type; + __u64 buf_addr; + __u32 buf_size; +} __packed; + +struct get_telemetry_resp { + __u32 major; + __u32 minor; + __u32 size; + enum aie2_msg_status status; +} __packed; + +struct execute_buffer_req { + __u32 cu_idx; + __u32 payload[19]; +} __packed; + +struct exec_dpu_req { + __u64 inst_buf_addr; + __u32 inst_size; + __u32 inst_prop_cnt; + __u32 cu_idx; + __u32 payload[35]; +} __packed; + +enum exec_npu_type { + EXEC_NPU_TYPE_NON_ELF = 0x1, + EXEC_NPU_TYPE_PARTIAL_ELF = 0x2, + EXEC_NPU_TYPE_PREEMPT = 0x3, + EXEC_NPU_TYPE_ELF = 0x4, +}; + +union exec_req { + struct execute_buffer_req ebuf; + struct exec_dpu_req dpu_req; +}; + +struct execute_buffer_resp { + enum aie2_msg_status status; +} __packed; + +struct aie_tile_info { + __u32 size; + __u16 major; + __u16 minor; + __u16 cols; + __u16 rows; + __u16 core_rows; + __u16 mem_rows; + __u16 shim_rows; + __u16 core_row_start; + __u16 mem_row_start; + __u16 shim_row_start; + __u16 core_dma_channels; + __u16 mem_dma_channels; + __u16 shim_dma_channels; + __u16 core_locks; + __u16 mem_locks; + __u16 shim_locks; + __u16 core_events; + __u16 mem_events; + __u16 shim_events; + __u16 reserved; +}; + +struct aie_tile_info_req { + __u32 reserved; +} __packed; + +struct aie_tile_info_resp { + enum aie2_msg_status status; + struct aie_tile_info info; +} __packed; + +struct aie_version_info_req { + __u32 reserved; +} __packed; + +struct aie_version_info_resp { + enum aie2_msg_status status; + __u16 major; + __u16 minor; +} __packed; + +struct aie_column_info_req { + __u64 dump_buff_addr; + __u32 dump_buff_size; + __u32 num_cols; + __u32 aie_bitmap; +} __packed; + +struct aie_column_info_resp { + enum aie2_msg_status status; + __u32 size; +} __packed; + +struct suspend_req { + __u32 place_holder; +} __packed; + +struct suspend_resp { + enum aie2_msg_status status; +} __packed; + +struct resume_req { + __u32 place_holder; +} __packed; + +struct resume_resp { + enum aie2_msg_status status; +} __packed; + +struct check_header_hash_req { + __u64 hash_high; + __u64 hash_low; +} __packed; + +struct check_header_hash_resp { + enum aie2_msg_status status; +} __packed; + +struct query_error_req { + __u64 buf_addr; + __u32 buf_size; + __u32 next_row; + __u32 next_column; + __u32 next_module; +} __packed; + +struct query_error_resp { + enum aie2_msg_status status; + __u32 num_err; + __u32 has_next_err; + __u32 next_row; + __u32 next_column; + __u32 next_module; +} __packed; + +struct protocol_version_req { + __u32 reserved; +} __packed; + +struct protocol_version_resp { + enum aie2_msg_status status; + __u32 major; + __u32 minor; +} __packed; + +struct firmware_version_req { + __u32 reserved; +} __packed; + +struct firmware_version_resp { + enum aie2_msg_status status; + __u32 major; + __u32 minor; + __u32 sub; + __u32 build; +} __packed; + +#define MAX_NUM_CUS 32 +#define AIE2_MSG_CFG_CU_PDI_ADDR GENMASK(16, 0) +#define AIE2_MSG_CFG_CU_FUNC GENMASK(24, 17) +struct config_cu_req { + __u32 num_cus; + __u32 cfgs[MAX_NUM_CUS]; +} __packed; + +struct config_cu_resp { + enum aie2_msg_status status; +} __packed; + +struct set_runtime_cfg_req { + __u32 type; + __u64 value; +} __packed; + +struct set_runtime_cfg_resp { + enum aie2_msg_status status; +} __packed; + +struct get_runtime_cfg_req { + __u32 type; +} __packed; + +struct get_runtime_cfg_resp { + enum aie2_msg_status status; + __u64 value; +} __packed; + +enum async_event_type { + ASYNC_EVENT_TYPE_AIE_ERROR, + ASYNC_EVENT_TYPE_EXCEPTION, + MAX_ASYNC_EVENT_TYPE +}; + +#define ASYNC_BUF_SIZE SZ_8K +struct async_event_msg_req { + __u64 buf_addr; + __u32 buf_size; +} __packed; + +struct async_event_msg_resp { + enum aie2_msg_status status; + enum async_event_type type; +} __packed; + +#define MAX_CHAIN_CMDBUF_SIZE SZ_4K + +struct cmd_chain_slot_execbuf_cf { + __u32 cu_idx; + __u32 arg_cnt; + __u32 args[] __counted_by(arg_cnt); +}; + +struct cmd_chain_slot_dpu { + __u64 inst_buf_addr; + __u32 inst_size; + __u32 inst_prop_cnt; + __u32 cu_idx; + __u32 arg_cnt; +#define MAX_DPU_ARGS_SIZE (34 * sizeof(__u32)) + __u32 args[] __counted_by(arg_cnt); +}; + +#define MAX_NPU_ARGS_SIZE (26 * sizeof(__u32)) +#define AIE2_EXEC_BUFFER_KERNEL_OP_TXN 3 +struct cmd_chain_slot_npu { + enum exec_npu_type type; + u64 inst_buf_addr; + u64 save_buf_addr; + u64 restore_buf_addr; + u32 inst_size; + u32 save_size; + u32 restore_size; + u32 inst_prop_cnt; + u32 cu_idx; + u32 arg_cnt; + u32 args[] __counted_by(arg_cnt); +} __packed; + +struct cmd_chain_req { + __u64 buf_addr; + __u32 buf_size; + __u32 count; +} __packed; + +struct cmd_chain_npu_req { + u32 flags; + u32 reserved; + u64 buf_addr; + u32 buf_size; + u32 count; +} __packed; + +union exec_chain_req { + struct cmd_chain_npu_req npu_req; + struct cmd_chain_req req; +}; + +struct cmd_chain_resp { + enum aie2_msg_status status; + __u32 fail_cmd_idx; + enum aie2_msg_status fail_cmd_status; +} __packed; + +#define AIE2_MSG_SYNC_BO_SRC_TYPE GENMASK(3, 0) +#define AIE2_MSG_SYNC_BO_DST_TYPE GENMASK(7, 4) +struct sync_bo_req { + __u64 src_addr; + __u64 dst_addr; + __u32 size; +#define SYNC_BO_DEV_MEM 0 +#define SYNC_BO_HOST_MEM 2 + __u32 type; +} __packed; + +struct sync_bo_resp { + enum aie2_msg_status status; +} __packed; + +#define DEBUG_BO_UNREGISTER 0 +#define DEBUG_BO_REGISTER 1 +struct config_debug_bo_req { + __u64 offset; + __u64 size; + /* + * config operations. + * DEBUG_BO_REGISTER: Register debug buffer + * DEBUG_BO_UNREGISTER: Unregister debug buffer + */ + __u32 config; +} __packed; + +struct config_debug_bo_resp { + enum aie2_msg_status status; +} __packed; + +struct fatal_error_info { + __u32 fatal_type; /* Fatal error type */ + __u32 exception_type; /* Only valid if fatal_type is a specific value */ + __u32 exception_argument; /* Argument based on exception type */ + __u32 exception_pc; /* Program Counter at the time of the exception */ + __u32 app_module; /* Error module name */ + __u32 task_index; /* Index of the task in which the error occurred */ + __u32 reserved[127]; +}; + +struct app_health_report { + __u16 major; + __u16 minor; + __u32 size; + __u32 context_id; + /* + * Program Counter (PC) of the last initiated DPU opcode, as reported by the ERT + * application. Before execution begins or after successful completion, the value is set + * to UINT_MAX. If execution halts prematurely due to an error, this field retains the + * opcode's PC value. + * Note: To optimize performance, the ERT may simplify certain aspects of reporting. + * Proper interpretation requires familiarity with the implementation details. + */ + __u32 dpu_pc; + /* + * Index of the last initiated TXN opcode. + * Before execution starts or after successful completion, the value is set to UINT_MAX. + * If execution halts prematurely due to an error, this field retains the opcode's ID. + * Note: To optimize performance, the ERT may simplify certain aspects of reporting. + * Proper interpretation requires familiarity with the implementation details. + */ + __u32 txn_op_id; + /* The PC of the context at the time of the report */ + __u32 ctx_pc; + struct fatal_error_info fatal_info; + /* Index of the most recently executed run list entry. */ + __u32 run_list_id; +}; + +struct get_app_health_req { + __u32 context_id; + __u32 buf_size; + __u64 buf_addr; +} __packed; + +struct get_app_health_resp { + enum aie2_msg_status status; + __u32 required_buffer_size; + __u32 reserved[7]; +} __packed; + +struct update_property_req { +#define UPDATE_PROPERTY_TIME_QUOTA 0 + __u32 type; +#define AIE2_UPDATE_PROPERTY_ALL_CTX 0xFF + __u8 context_id; + __u8 reserved[7]; + __u32 time_quota_us; + __u32 reserved1; +} __packed; + +struct update_property_resp { + enum aie2_msg_status status; +} __packed; + +enum aie2_dev_revision { + AIE2_DEV_REVISION_STXA = 1, + AIE2_DEV_REVISION_STXB, + AIE2_DEV_REVISION_KRK1, + AIE2_DEV_REVISION_KRK2, + AIE2_DEV_REVISION_HALO, + AIE2_DEV_REVISION_GPT1, + AIE2_DEV_REVISION_GPT2, + AIE2_DEV_REVISION_GPT3, + AIE2_DEV_REVISION_UNKN, +}; + +struct get_dev_revision_req { + __u32 place_holder; +} __packed; + +struct get_dev_revision_resp { + enum aie2_msg_status status; + enum aie2_dev_revision rev; + __u32 raw_fuse_data; +} __packed; + +#endif /* _AIE2_MSG_PRIV_H_ */ diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c new file mode 100644 index 000000000..daec1f6b4 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_pci.c @@ -0,0 +1,1251 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_drv.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_managed.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/amd-pmf-io.h> +#include <linux/cleanup.h> +#include <linux/errno.h> +#include <linux/firmware.h> +#include <linux/iommu.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <linux/xarray.h> +#include <asm/hypervisor.h> + +#include "aie2_msg_priv.h" +#include "aie2_pci.h" +#include "aie2_solver.h" +#include "amdxdna_ctx.h" +#include "amdxdna_gem.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" + +static int aie2_max_col = XRS_MAX_COL; +module_param(aie2_max_col, uint, 0600); +MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used"); + +#define DEFAULT_TIME_QUANTUM 30000 /* microseconds */ + +static char *npu_fw[] = { + "npu_7.sbin", + "npu.sbin" +}; + +/* + * The management mailbox channel is allocated by firmware. + * The related register and ring buffer information is on SRAM BAR. + * This struct is the register layout. + */ +#define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */ +struct mgmt_mbox_chann_info { + __u32 x2i_tail; + __u32 x2i_head; + __u32 x2i_buf; + __u32 x2i_buf_sz; + __u32 i2x_tail; + __u32 i2x_head; + __u32 i2x_buf; + __u32 i2x_buf_sz; + __u32 magic; + __u32 msi_id; + __u32 prot_major; + __u32 prot_minor; + __u32 rsvd[4]; +}; + +static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev) +{ + struct mgmt_mbox_chann_info info_regs; + struct xdna_mailbox_chann_res *i2x; + struct xdna_mailbox_chann_res *x2i; + u32 addr, off; + u32 *reg; + int ret; + int i; + + /* + * Once firmware is alive, it will write management channel + * information in SRAM BAR and write the address of that information + * at FW_ALIVE_OFF offset in SRMA BAR. + * + * Read a non-zero value from FW_ALIVE_OFF implies that firmware + * is alive. + */ + ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF), + addr, addr, AIE_INTERVAL, AIE_TIMEOUT); + if (ret || !addr) + return -ETIME; + + off = AIE2_SRAM_OFF(ndev, addr); + reg = (u32 *)&info_regs; + for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++) + reg[i] = readl(ndev->sram_base + off + i * sizeof(u32)); + + if (info_regs.magic != MGMT_MBOX_MAGIC) { + XDNA_ERR(ndev->aie.xdna, "Invalid mbox magic 0x%x", info_regs.magic); + ret = -EINVAL; + goto done; + } + + i2x = &ndev->aie.mgmt_i2x; + x2i = &ndev->aie.mgmt_x2i; + + i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head); + i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail); + i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf); + i2x->rb_size = info_regs.i2x_buf_sz; + + x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head); + x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail); + x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf); + x2i->rb_size = info_regs.x2i_buf_sz; + + ndev->aie.mgmt_chan_idx = info_regs.msi_id; + ndev->aie.mgmt_prot_major = info_regs.prot_major; + ndev->aie.mgmt_prot_minor = info_regs.prot_minor; + + ret = aie_check_protocol(&ndev->aie, ndev->aie.mgmt_prot_major, + ndev->aie.mgmt_prot_minor); + +done: + aie_dump_mgmt_chann_debug(&ndev->aie); + + /* Must clear address at FW_ALIVE_OFF */ + writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF)); + + return ret; +} + +int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, + enum rt_config_category category, u32 *val) +{ + const struct rt_config *cfg; + u32 value; + int ret; + + for (cfg = ndev->priv->rt_config; cfg->type; cfg++) { + if (cfg->category != category) + continue; + + if (cfg->feature_mask && + bitmap_subset(&cfg->feature_mask, &ndev->aie.feature_mask, + AIE2_FEATURE_MAX)) + continue; + + value = val ? *val : cfg->value; + ret = aie2_set_runtime_cfg(ndev, cfg->type, value); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Set type %d value %d failed", + cfg->type, value); + return ret; + } + } + + return 0; +} + +static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + ret = aie2_suspend_fw(ndev); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Suspend firmware failed"); + return ret; + } + + ret = aie2_resume_fw(ndev); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Resume firmware failed"); + return ret; + } + + return 0; +} + +static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Runtime config failed"); + return ret; + } + + ret = aie2_assign_mgmt_pasid(ndev, 0); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Can not assign PASID"); + return ret; + } + + ret = aie2_update_prop_time_quota(ndev, DEFAULT_TIME_QUANTUM); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Failed to update execution time quantum"); + return ret; + } + + ret = aie2_xdna_reset(ndev); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Reset firmware failed"); + return ret; + } + + return 0; +} + +static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + ret = aie2_query_firmware_version(ndev, &ndev->aie.xdna->fw_ver); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "query firmware version failed"); + return ret; + } + + ret = aie2_query_aie_version(ndev, &ndev->version); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Query AIE version failed"); + return ret; + } + + ret = aie2_query_aie_metadata(ndev, &ndev->aie.metadata); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "Query AIE metadata failed"); + return ret; + } + + ndev->total_col = min(aie2_max_col, ndev->aie.metadata.cols); + + return 0; +} + +static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev) +{ + if (aie2_suspend_fw(ndev)) + XDNA_ERR(ndev->aie.xdna, "Suspend_fw failed"); + XDNA_DBG(ndev->aie.xdna, "Firmware suspended"); +} + +static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action) +{ + struct amdxdna_hwctx *hwctx = cb_arg; + struct amdxdna_dev *xdna; + int ret; + + xdna = hwctx->client->xdna; + + hwctx->start_col = action->part.start_col; + hwctx->num_unused_col = action->part.ncols - hwctx->num_col; + hwctx->num_col = action->part.ncols; + ret = aie2_create_context(xdna->dev_handle, hwctx); + if (ret) + XDNA_ERR(xdna, "create context failed, ret %d", ret); + + return ret; +} + +static int aie2_xrs_unload(void *cb_arg) +{ + struct amdxdna_hwctx *hwctx = cb_arg; + struct amdxdna_dev *xdna; + int ret; + + xdna = hwctx->client->xdna; + + ret = aie2_destroy_context(xdna->dev_handle, hwctx); + if (ret) + XDNA_ERR(xdna, "destroy context failed, ret %d", ret); + + return ret; +} + +static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level) +{ + struct amdxdna_dev *xdna = to_xdna_dev(ddev); + struct amdxdna_dev_hdl *ndev; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + ndev = xdna->dev_handle; + ndev->dft_dpm_level = dpm_level; + if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level) + return 0; + + return aie2_pm_set_dpm(ndev, dpm_level); +} + +static struct xrs_action_ops aie2_xrs_actions = { + .load = aie2_xrs_load, + .unload = aie2_xrs_unload, + .set_dft_dpm_level = aie2_xrs_set_dft_dpm_level, +}; + +static void aie2_smu_fini(struct amdxdna_dev_hdl *ndev) +{ + ndev->priv->hw_ops->set_dpm(ndev, 0); + aie_smu_fini(ndev->aie.smu_hdl); +} + +static void aie2_hw_stop(struct amdxdna_dev *xdna) +{ + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + + if (ndev->dev_status <= AIE2_DEV_INIT) { + XDNA_ERR(xdna, "device is already stopped"); + return; + } + + aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL); + aie2_mgmt_fw_fini(ndev); + aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann); + drmm_kfree(&xdna->ddev, ndev->mbox); + ndev->mbox = NULL; + aie_psp_stop(ndev->aie.psp_hdl); + aie2_smu_fini(ndev); + aie2_error_async_events_free(ndev); + pci_disable_device(pdev); + + ndev->dev_status = AIE2_DEV_INIT; +} + +static int aie2_hw_start(struct amdxdna_dev *xdna) +{ + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + struct xdna_mailbox_res mbox_res; + u32 xdna_mailbox_intr_reg; + int mgmt_mb_irq, ret; + + if (ndev->dev_status >= AIE2_DEV_START) { + XDNA_INFO(xdna, "device is already started"); + return 0; + } + + ret = pci_enable_device(pdev); + if (ret) { + XDNA_ERR(xdna, "failed to enable device, ret %d", ret); + return ret; + } + pci_set_master(pdev); + + mbox_res.ringbuf_base = ndev->sram_base; + mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar); + mbox_res.mbox_base = ndev->mbox_base; + mbox_res.mbox_size = MBOX_SIZE(ndev); + mbox_res.name = "xdna_mailbox"; + ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res); + if (!ndev->mbox) { + XDNA_ERR(xdna, "failed to create mailbox device"); + ret = -ENODEV; + goto disable_dev; + } + + ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox); + if (!ndev->aie.mgmt_chann) { + XDNA_ERR(xdna, "failed to alloc channel"); + ret = -ENODEV; + goto disable_dev; + } + + ret = aie_smu_init(ndev->aie.smu_hdl); + if (ret) { + XDNA_ERR(xdna, "failed to init smu, ret %d", ret); + goto free_channel; + } + + ret = aie_psp_start(ndev->aie.psp_hdl); + if (ret) { + XDNA_ERR(xdna, "failed to start psp, ret %d", ret); + goto fini_smu; + } + + ret = aie2_get_mgmt_chann_info(ndev); + if (ret) { + XDNA_ERR(xdna, "firmware is not alive"); + goto stop_psp; + } + + mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx); + if (mgmt_mb_irq < 0) { + ret = mgmt_mb_irq; + XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret); + goto stop_psp; + } + + xdna_mailbox_intr_reg = ndev->aie.mgmt_i2x.mb_head_ptr_reg + 4; + ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann, + &ndev->aie.mgmt_x2i, + &ndev->aie.mgmt_i2x, + xdna_mailbox_intr_reg, + mgmt_mb_irq); + if (ret) { + XDNA_ERR(xdna, "failed to start management mailbox channel"); + ret = -EINVAL; + goto stop_psp; + } + + ret = aie2_mgmt_fw_init(ndev); + if (ret) { + XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret); + goto stop_fw; + } + + ret = aie2_pm_init(ndev); + if (ret) { + XDNA_ERR(xdna, "failed to init pm, ret %d", ret); + goto stop_fw; + } + + ret = aie2_mgmt_fw_query(ndev); + if (ret) { + XDNA_ERR(xdna, "failed to query fw, ret %d", ret); + goto stop_fw; + } + + ret = aie2_error_async_events_alloc(ndev); + if (ret) { + XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret); + goto stop_fw; + } + + WRITE_ONCE(ndev->last_signal_ts, jiffies); + ndev->dev_status = AIE2_DEV_START; + + return 0; + +stop_fw: + aie2_suspend_fw(ndev); + xdna_mailbox_stop_channel(ndev->aie.mgmt_chann); +stop_psp: + aie_psp_stop(ndev->aie.psp_hdl); +fini_smu: + aie2_smu_fini(ndev); +free_channel: + xdna_mailbox_free_channel(ndev->aie.mgmt_chann); + ndev->aie.mgmt_chann = NULL; +disable_dev: + pci_disable_device(pdev); + + return ret; +} + +static int aie2_hw_suspend(struct amdxdna_dev *xdna) +{ + struct amdxdna_client *client; + + list_for_each_entry(client, &xdna->client_list, node) + aie2_hwctx_suspend(client); + + aie2_hw_stop(xdna); + + return 0; +} + +static int aie2_hw_resume(struct amdxdna_dev *xdna) +{ + struct amdxdna_client *client; + int ret; + + ret = aie2_hw_start(xdna); + if (ret) { + XDNA_ERR(xdna, "Start hardware failed, %d", ret); + return ret; + } + + list_for_each_entry(client, &xdna->client_list, node) { + ret = aie2_hwctx_resume(client); + if (ret) + break; + } + + return ret; +} + +static int aie2_init(struct amdxdna_dev *xdna) +{ + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + void __iomem *tbl[PCI_NUM_RESOURCES] = {0}; + struct init_config xrs_cfg = { 0 }; + struct amdxdna_dev_hdl *ndev; + struct psp_config psp_conf = { 0 }; + struct smu_config smu_conf; + const struct firmware *fw; + unsigned long bars = 0; + char *fw_full_path; + int i, nvec, ret; + + if (!hypervisor_is_type(X86_HYPER_NATIVE)) { + XDNA_ERR(xdna, "Running under hypervisor not supported"); + return -EINVAL; + } + + if (!xdna->group) { + XDNA_ERR(xdna, "Running without IOMMU not supported"); + return -EINVAL; + } + + ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL); + if (!ndev) + return -ENOMEM; + + ndev->priv = xdna->dev_info->dev_priv; + ndev->aie.xdna = xdna; + + for (i = 0; i < ARRAY_SIZE(npu_fw); i++) { + fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]); + if (!fw_full_path) + return -ENOMEM; + + ret = firmware_request_nowarn(&fw, fw_full_path, &pdev->dev); + kfree(fw_full_path); + if (!ret) { + XDNA_INFO(xdna, "Load firmware %s%s", ndev->priv->fw_path, npu_fw[i]); + break; + } + } + + if (ret) { + XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", + ndev->priv->fw_path, ret); + return ret; + } + + ret = pcim_enable_device(pdev); + if (ret) { + XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret); + goto release_fw; + } + + for (i = 0; i < PSP_MAX_REGS; i++) + set_bit(PSP_REG_BAR(ndev, i), &bars); + for (i = 0; i < SMU_MAX_REGS; i++) + set_bit(SMU_REG_BAR(ndev, i), &bars); + + set_bit(xdna->dev_info->sram_bar, &bars); + set_bit(xdna->dev_info->mbox_bar, &bars); + + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + if (!test_bit(i, &bars)) + continue; + tbl[i] = pcim_iomap(pdev, i, 0); + if (!tbl[i]) { + XDNA_ERR(xdna, "map bar %d failed", i); + ret = -ENOMEM; + goto release_fw; + } + } + + ndev->sram_base = tbl[xdna->dev_info->sram_bar]; + ndev->mbox_base = tbl[xdna->dev_info->mbox_bar]; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret); + goto release_fw; + } + + nvec = pci_msix_vec_count(pdev); + if (nvec <= 0) { + XDNA_ERR(xdna, "does not get number of interrupt vector"); + ret = -EINVAL; + goto release_fw; + } + + ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); + if (ret < 0) { + XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret); + goto release_fw; + } + + psp_conf.fw_size = fw->size; + psp_conf.fw_buf = fw->data; + psp_conf.arg2_mask = GENMASK(23, 0); + psp_conf.notify_val = 1; + for (i = 0; i < PSP_MAX_REGS; i++) + psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i); + ndev->aie.psp_hdl = aiem_psp_create(&xdna->ddev, &psp_conf); + if (!ndev->aie.psp_hdl) { + XDNA_ERR(xdna, "failed to create psp"); + ret = -ENOMEM; + goto release_fw; + } + + for (i = 0; i < SMU_MAX_REGS; i++) + smu_conf.smu_regs[i] = tbl[SMU_REG_BAR(ndev, i)] + SMU_REG_OFF(ndev, i); + ndev->aie.smu_hdl = aiem_smu_create(&xdna->ddev, &smu_conf); + if (!ndev->aie.smu_hdl) { + XDNA_ERR(xdna, "failed to create smu"); + ret = -ENOMEM; + goto release_fw; + } + xdna->dev_handle = ndev; + + ret = aie2_hw_start(xdna); + if (ret) { + XDNA_ERR(xdna, "start npu failed, ret %d", ret); + goto release_fw; + } + + xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1; + for (i = 0; i < xrs_cfg.clk_list.num_levels; i++) + xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk; + xrs_cfg.sys_eff_factor = 2; + xrs_cfg.ddev = &xdna->ddev; + xrs_cfg.actions = &aie2_xrs_actions; + xrs_cfg.total_col = ndev->total_col; + + xdna->xrs_hdl = xrsm_init(&xrs_cfg); + if (!xdna->xrs_hdl) { + XDNA_ERR(xdna, "Initialize resolver failed"); + ret = -EINVAL; + goto stop_hw; + } + + release_firmware(fw); + aie2_msg_init(ndev); + amdxdna_vbnv_init(xdna); + amdxdna_pm_init(xdna); + return 0; + +stop_hw: + aie2_hw_stop(xdna); +release_fw: + release_firmware(fw); + + return ret; +} + +static void aie2_fini(struct amdxdna_dev *xdna) +{ + amdxdna_pm_fini(xdna); + aie2_hw_stop(xdna); +} + +static int aie2_get_aie_status(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_query_aie_status status = {}; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev; + u32 buf_sz; + int ret; + + ndev = xdna->dev_handle; + buf_sz = min(args->buffer_size, sizeof(status)); + if (copy_from_user(&status, u64_to_user_ptr(args->buffer), buf_sz)) { + XDNA_ERR(xdna, "Failed to copy AIE request into kernel"); + return -EFAULT; + } + + ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer), + status.buffer_size, &status.cols_filled); + if (ret) { + XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret); + return ret; + } + + if (copy_to_user(u64_to_user_ptr(args->buffer), &status, buf_sz)) { + XDNA_ERR(xdna, "Failed to copy AIE request info to user space"); + return -EFAULT; + } + + return 0; +} + +static int aie2_get_aie_version(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_query_aie_version version; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev; + u32 buf_sz; + + ndev = xdna->dev_handle; + version.major = ndev->version.major; + version.minor = ndev->version.minor; + + buf_sz = min(args->buffer_size, sizeof(version)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &version, buf_sz)) + return -EFAULT; + + return 0; +} + +static int aie2_get_firmware_version(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_query_firmware_version version; + struct amdxdna_dev *xdna = client->xdna; + u32 buf_sz; + + version.major = xdna->fw_ver.major; + version.minor = xdna->fw_ver.minor; + version.patch = xdna->fw_ver.sub; + version.build = xdna->fw_ver.build; + + buf_sz = min(args->buffer_size, sizeof(version)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &version, buf_sz)) + return -EFAULT; + + return 0; +} + +static int aie2_get_power_mode(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_get_power_mode mode = {}; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev; + u32 buf_sz; + + ndev = xdna->dev_handle; + mode.power_mode = ndev->pw_mode; + + buf_sz = min(args->buffer_size, sizeof(mode)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, buf_sz)) + return -EFAULT; + + return 0; +} + +static int aie2_get_clock_metadata(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_query_clock_metadata *clock; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev; + int ret = 0; + u32 buf_sz; + + ndev = xdna->dev_handle; + clock = kzalloc_obj(*clock); + if (!clock) + return -ENOMEM; + + aie2_update_counters(ndev); + snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name), + "MP-NPU Clock"); + clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq; + snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock"); + clock->h_clock.freq_mhz = ndev->hclk_freq; + + buf_sz = min(args->buffer_size, sizeof(*clock)); + if (copy_to_user(u64_to_user_ptr(args->buffer), clock, buf_sz)) + ret = -EFAULT; + + kfree(clock); + return ret; +} + +static int aie2_get_sensors(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle; + struct amdxdna_drm_query_sensor sensor = {}; + struct amd_pmf_npu_metrics npu_metrics; + u32 sensors_count = 0, i; + int ret; + + ret = AIE2_GET_PMF_NPU_METRICS(&npu_metrics); + if (ret) + return ret; + + sensor.type = AMDXDNA_SENSOR_TYPE_POWER; + sensor.input = npu_metrics.npu_power; + sensor.unitm = -3; + scnprintf(sensor.label, sizeof(sensor.label), "Total Power"); + scnprintf(sensor.units, sizeof(sensor.units), "mW"); + + if (args->buffer_size < sizeof(sensor)) + goto out; + + if (copy_to_user(u64_to_user_ptr(args->buffer), &sensor, sizeof(sensor))) + return -EFAULT; + + args->buffer_size -= sizeof(sensor); + sensors_count++; + + for (i = 0; i < min_t(u32, ndev->total_col, 8); i++) { + memset(&sensor, 0, sizeof(sensor)); + sensor.input = npu_metrics.npu_busy[i]; + sensor.type = AMDXDNA_SENSOR_TYPE_COLUMN_UTILIZATION; + sensor.unitm = 0; + scnprintf(sensor.label, sizeof(sensor.label), "Column %d Utilization", i); + scnprintf(sensor.units, sizeof(sensor.units), "%%"); + + if (args->buffer_size < sizeof(sensor)) + goto out; + + if (copy_to_user(u64_to_user_ptr(args->buffer) + sensors_count * sizeof(sensor), + &sensor, sizeof(sensor))) + return -EFAULT; + + args->buffer_size -= sizeof(sensor); + sensors_count++; + } + +out: + args->buffer_size = sensors_count * sizeof(sensor); + + return 0; +} + +static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg) +{ + struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL; + struct amdxdna_drm_get_array *array_args = arg; + struct amdxdna_drm_hwctx_entry __user *buf; + struct app_health_report report; + struct amdxdna_dev_hdl *ndev; + u32 size; + int ret; + + if (!array_args->num_element) + return -EINVAL; + + tmp = kzalloc_obj(*tmp); + if (!tmp) + return -ENOMEM; + + tmp->pid = hwctx->client->pid; + tmp->context_id = hwctx->id; + tmp->start_col = hwctx->start_col; + tmp->num_col = hwctx->num_col; + tmp->command_submissions = hwctx->priv->seq; + tmp->command_completions = hwctx->priv->completed; + tmp->pasid = hwctx->client->pasid; + tmp->heap_usage = hwctx->client->heap_usage; + tmp->priority = hwctx->qos.priority; + tmp->gops = hwctx->qos.gops; + tmp->fps = hwctx->qos.fps; + tmp->dma_bandwidth = hwctx->qos.dma_bandwidth; + tmp->latency = hwctx->qos.latency; + tmp->frame_exec_time = hwctx->qos.frame_exec_time; + tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE; + ndev = hwctx->client->xdna->dev_handle; + ret = aie2_query_app_health(ndev, hwctx->fw_ctx_id, &report); + if (!ret) { + /* Fill in app health report fields */ + tmp->txn_op_idx = report.txn_op_id; + tmp->ctx_pc = report.ctx_pc; + tmp->fatal_error_type = report.fatal_info.fatal_type; + tmp->fatal_error_exception_type = report.fatal_info.exception_type; + tmp->fatal_error_exception_pc = report.fatal_info.exception_pc; + tmp->fatal_error_app_module = report.fatal_info.app_module; + } + + buf = u64_to_user_ptr(array_args->buffer); + size = min(sizeof(*tmp), array_args->element_size); + + if (copy_to_user(buf, tmp, size)) + return -EFAULT; + + array_args->buffer += size; + array_args->num_element--; + + return 0; +} + +static int aie2_get_hwctx_status(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_get_array array_args; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_client *tmp_client; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx); + array_args.buffer = args->buffer; + array_args.num_element = args->buffer_size / array_args.element_size; + list_for_each_entry(tmp_client, &xdna->client_list, node) { + ret = amdxdna_hwctx_walk(tmp_client, &array_args, + aie2_hwctx_status_cb); + if (ret) + break; + } + + args->buffer_size -= (u32)(array_args.buffer - args->buffer); + return 0; +} + +static int aie2_query_resource_info(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_get_resource_info res_info; + const struct amdxdna_dev_priv *priv; + struct amdxdna_dev_hdl *ndev; + struct amdxdna_dev *xdna; + u32 buf_sz; + + xdna = client->xdna; + ndev = xdna->dev_handle; + priv = ndev->priv; + + aie2_update_counters(ndev); + res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk; + res_info.npu_tops_max = ndev->max_tops; + res_info.npu_task_max = priv->hwctx_limit; + res_info.npu_tops_curr = ndev->curr_tops; + res_info.npu_task_curr = ndev->hwctx_num; + + buf_sz = min(args->buffer_size, sizeof(res_info)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, buf_sz)) + return -EFAULT; + + return 0; +} + +static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg) +{ + struct amdxdna_dev *xdna = hwctx->client->xdna; + u32 *map = arg; + + if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) { + XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id, + xdna->dev_handle->priv->hwctx_limit); + return -EINVAL; + } + + map[hwctx->fw_ctx_id] = hwctx->id; + return 0; +} + +static int aie2_get_telemetry(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL; + u32 telemetry_data_sz, header_sz, elem_num; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_client *tmp_client; + int ret; + + elem_num = xdna->dev_handle->priv->hwctx_limit; + header_sz = struct_size(header, map, elem_num); + if (args->buffer_size <= header_sz) { + XDNA_ERR(xdna, "Invalid buffer size"); + return -EINVAL; + } + telemetry_data_sz = args->buffer_size - header_sz; + + header = kzalloc(header_sz, GFP_KERNEL); + if (!header) + return -ENOMEM; + + if (copy_from_user(header, u64_to_user_ptr(args->buffer), sizeof(*header))) { + XDNA_ERR(xdna, "Failed to copy telemetry header from user"); + return -EFAULT; + } + + header->map_num_elements = elem_num; + list_for_each_entry(tmp_client, &xdna->client_list, node) { + ret = amdxdna_hwctx_walk(tmp_client, &header->map, + aie2_fill_hwctx_map); + if (ret) + return ret; + } + + ret = aie2_query_telemetry(xdna->dev_handle, + u64_to_user_ptr(args->buffer + header_sz), + telemetry_data_sz, header); + if (ret) { + XDNA_ERR(xdna, "Query telemetry failed ret %d", ret); + return ret; + } + + if (copy_to_user(u64_to_user_ptr(args->buffer), header, header_sz)) { + XDNA_ERR(xdna, "Copy header failed"); + return -EFAULT; + } + + return 0; +} + +static int aie2_get_preempt_state(struct amdxdna_client *client, + struct amdxdna_drm_get_info *args) +{ + struct amdxdna_drm_attribute_state state = {}; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev; + u32 buf_sz; + + ndev = xdna->dev_handle; + if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE) + state.state = ndev->force_preempt_enabled; + else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE) + state.state = ndev->frame_boundary_preempt; + + buf_sz = min(args->buffer_size, sizeof(state)); + if (copy_to_user(u64_to_user_ptr(args->buffer), &state, buf_sz)) + return -EFAULT; + + return 0; +} + +static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) +{ + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + int ret, idx; + + if (!drm_dev_enter(&xdna->ddev, &idx)) + return -ENODEV; + + ret = amdxdna_pm_resume_get_locked(xdna); + if (ret) + goto dev_exit; + + switch (args->param) { + case DRM_AMDXDNA_QUERY_AIE_STATUS: + ret = aie2_get_aie_status(client, args); + break; + case DRM_AMDXDNA_QUERY_AIE_METADATA: + ret = amdxdna_get_metadata(&ndev->aie, client, args); + break; + case DRM_AMDXDNA_QUERY_AIE_VERSION: + ret = aie2_get_aie_version(client, args); + break; + case DRM_AMDXDNA_QUERY_CLOCK_METADATA: + ret = aie2_get_clock_metadata(client, args); + break; + case DRM_AMDXDNA_QUERY_SENSORS: + ret = aie2_get_sensors(client, args); + break; + case DRM_AMDXDNA_QUERY_HW_CONTEXTS: + ret = aie2_get_hwctx_status(client, args); + break; + case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION: + ret = aie2_get_firmware_version(client, args); + break; + case DRM_AMDXDNA_GET_POWER_MODE: + ret = aie2_get_power_mode(client, args); + break; + case DRM_AMDXDNA_QUERY_TELEMETRY: + ret = aie2_get_telemetry(client, args); + break; + case DRM_AMDXDNA_QUERY_RESOURCE_INFO: + ret = aie2_query_resource_info(client, args); + break; + case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE: + case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE: + ret = aie2_get_preempt_state(client, args); + break; + default: + XDNA_ERR(xdna, "Not supported request parameter %u", args->param); + ret = -EOPNOTSUPP; + } + + amdxdna_pm_suspend_put(xdna); + XDNA_DBG(xdna, "Got param %d", args->param); + +dev_exit: + drm_dev_exit(idx); + return ret; +} + +static int aie2_query_ctx_status_array(struct amdxdna_client *client, + struct amdxdna_drm_get_array *args) +{ + struct amdxdna_drm_get_array array_args; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_client *tmp_client; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (args->element_size > SZ_4K || args->num_element > SZ_1K) { + XDNA_DBG(xdna, "Invalid element size %d or number of element %d", + args->element_size, args->num_element); + return -EINVAL; + } + + array_args.element_size = min(args->element_size, + sizeof(struct amdxdna_drm_hwctx_entry)); + array_args.buffer = args->buffer; + array_args.num_element = args->num_element * args->element_size / + array_args.element_size; + list_for_each_entry(tmp_client, &xdna->client_list, node) { + ret = amdxdna_hwctx_walk(tmp_client, &array_args, + aie2_hwctx_status_cb); + if (ret) + break; + } + + args->element_size = array_args.element_size; + args->num_element = (u32)((array_args.buffer - args->buffer) / + args->element_size); + + return 0; +} + +static int aie2_get_array(struct amdxdna_client *client, + struct amdxdna_drm_get_array *args) +{ + struct amdxdna_dev *xdna = client->xdna; + int ret, idx; + + if (!drm_dev_enter(&xdna->ddev, &idx)) + return -ENODEV; + + ret = amdxdna_pm_resume_get_locked(xdna); + if (ret) + goto dev_exit; + + switch (args->param) { + case DRM_AMDXDNA_HW_CONTEXT_ALL: + ret = aie2_query_ctx_status_array(client, args); + break; + case DRM_AMDXDNA_HW_LAST_ASYNC_ERR: + ret = aie2_get_array_async_error(xdna->dev_handle, args); + break; + case DRM_AMDXDNA_BO_USAGE: + ret = amdxdna_drm_get_bo_usage(&xdna->ddev, args); + break; + default: + XDNA_ERR(xdna, "Not supported request parameter %u", args->param); + ret = -EOPNOTSUPP; + } + + amdxdna_pm_suspend_put(xdna); + XDNA_DBG(xdna, "Got param %d", args->param); + +dev_exit: + drm_dev_exit(idx); + return ret; +} + +static int aie2_set_power_mode(struct amdxdna_client *client, + struct amdxdna_drm_set_state *args) +{ + struct amdxdna_drm_set_power_mode power_state; + enum amdxdna_power_mode_type power_mode; + struct amdxdna_dev *xdna = client->xdna; + + if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer), + sizeof(power_state))) { + XDNA_ERR(xdna, "Failed to copy power mode request into kernel"); + return -EFAULT; + } + + if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad))) + return -EINVAL; + + power_mode = power_state.power_mode; + if (power_mode > POWER_MODE_TURBO) { + XDNA_ERR(xdna, "Invalid power mode %d", power_mode); + return -EINVAL; + } + + return aie2_pm_set_mode(xdna->dev_handle, power_mode); +} + +static int aie2_set_preempt_state(struct amdxdna_client *client, + struct amdxdna_drm_set_state *args) +{ + struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle; + struct amdxdna_drm_attribute_state state; + u32 val; + int ret; + + if (copy_from_user(&state, u64_to_user_ptr(args->buffer), sizeof(state))) + return -EFAULT; + + if (state.state > 1) + return -EINVAL; + + if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad))) + return -EINVAL; + + if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) { + ndev->force_preempt_enabled = state.state; + } else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) { + val = state.state; + ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, + &val); + if (ret) + return ret; + + ndev->frame_boundary_preempt = state.state; + } + + return 0; +} + +static int aie2_set_state(struct amdxdna_client *client, + struct amdxdna_drm_set_state *args) +{ + struct amdxdna_dev *xdna = client->xdna; + int ret, idx; + + if (!drm_dev_enter(&xdna->ddev, &idx)) + return -ENODEV; + + ret = amdxdna_pm_resume_get_locked(xdna); + if (ret) + goto dev_exit; + + switch (args->param) { + case DRM_AMDXDNA_SET_POWER_MODE: + ret = aie2_set_power_mode(client, args); + break; + case DRM_AMDXDNA_SET_FORCE_PREEMPT: + case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT: + ret = aie2_set_preempt_state(client, args); + break; + default: + XDNA_ERR(xdna, "Not supported request parameter %u", args->param); + ret = -EOPNOTSUPP; + break; + } + + amdxdna_pm_suspend_put(xdna); +dev_exit: + drm_dev_exit(idx); + return ret; +} + +static int aie2_get_dev_rev(struct amdxdna_dev *xdna, u32 *rev) +{ + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + enum aie2_dev_revision aie2_rev; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + ret = aie2_get_dev_revision(ndev, &aie2_rev); + + if (!ret) + *rev = (u32)aie2_rev; + + return ret; +} + +const struct amdxdna_dev_ops aie2_ops = { + .init = aie2_init, + .fini = aie2_fini, + .resume = aie2_hw_resume, + .suspend = aie2_hw_suspend, + .get_aie_info = aie2_get_info, + .set_aie_state = aie2_set_state, + .hwctx_init = aie2_hwctx_init, + .hwctx_fini = aie2_hwctx_fini, + .hwctx_config = aie2_hwctx_config, + .hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo, + .cmd_submit = aie2_cmd_submit, + .hmm_invalidate = aie2_hmm_invalidate, + .get_array = aie2_get_array, + .get_dev_revision = aie2_get_dev_rev, + .hwctx_heap_expand = aie2_hwctx_heap_expand, +}; diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h new file mode 100644 index 000000000..ea1dac106 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_pci.h @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023-2026, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE2_PCI_H_ +#define _AIE2_PCI_H_ + +#include <drm/amdxdna_accel.h> +#include <linux/limits.h> +#include <linux/semaphore.h> + +#include "aie.h" +#include "aie2_msg_priv.h" +#include "amdxdna_mailbox.h" + +/* Firmware determines device memory base address and size */ +#define AIE2_DEVM_BASE 0x4000000 +#define AIE2_DEVM_SIZE SZ_64M +#define AIE2_DEVM_MAX_SIZE SZ_512M + +#define NDEV2PDEV(ndev) (to_pci_dev((ndev)->aie.xdna->ddev.dev)) + +#define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr) +#define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr) + +#define SRAM_REG_OFF(ndev, idx) ((ndev)->priv->sram_offs[(idx)].offset) + +#define SRAM_GET_ADDR(ndev, idx) \ +({ \ + typeof(ndev) _ndev = ndev; \ + ((_ndev)->sram_base + SRAM_REG_OFF((_ndev), (idx))); \ +}) + +#define CHAN_SLOT_SZ SZ_8K +#define MBOX_SIZE(ndev) \ +({ \ + typeof(ndev) _ndev = (ndev); \ + ((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \ + pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->aie.xdna->dev_info->mbox_bar); \ +}) + +#if IS_ENABLED(CONFIG_AMD_PMF) +#define AIE2_GET_PMF_NPU_METRICS(metrics) amd_pmf_get_npu_data(metrics) +#define AIE2_GET_PMF_NPU_DATA(field, val) \ +({ \ + struct amd_pmf_npu_metrics _npu_metrics; \ + int _ret; \ + \ + _ret = amd_pmf_get_npu_data(&_npu_metrics); \ + val = _ret ? U32_MAX : _npu_metrics.field; \ + (_ret); \ +}) +#else +#define AIE2_GET_PMF_NPU_METRICS(metrics) \ +({ \ + typeof(metrics) _m = metrics; \ + memset(_m, 0xff, sizeof(*_m)); \ + (-EOPNOTSUPP); \ +}) + +#define SENSOR_DEFAULT_npu_power U32_MAX +#define AIE2_GET_PMF_NPU_DATA(field, val) \ +({ \ + val = SENSOR_DEFAULT_##field; \ + (-EOPNOTSUPP); \ +}) +#endif + +enum aie2_sram_reg_idx { + MBOX_CHANN_OFF = 0, + FW_ALIVE_OFF, + SRAM_MAX_INDEX /* Keep this at the end */ +}; + +struct amdxdna_client; +struct amdxdna_fw_ver; +struct amdxdna_hwctx; +struct amdxdna_sched_job; + +enum rt_config_category { + AIE2_RT_CFG_INIT, + AIE2_RT_CFG_CLK_GATING, + AIE2_RT_CFG_FORCE_PREEMPT, + AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, +}; + +struct rt_config { + u32 type; + u32 value; + u32 category; + unsigned long feature_mask; +}; + +struct dpm_clk_freq { + u32 npuclk; + u32 hclk; +}; + +/* + * Define the maximum number of pending commands in a hardware context. + * Must be power of 2! + */ +#define HWCTX_MAX_CMDS 4 +#define get_job_idx(seq) ((seq) & (HWCTX_MAX_CMDS - 1)) +struct amdxdna_hwctx_priv { + struct amdxdna_gem_obj *heap; + void *mbox_chann; + + struct drm_gpu_scheduler sched; + struct drm_sched_entity entity; + + struct mutex io_lock; /* protect seq and cmd order */ + struct wait_queue_head job_free_wq; + u32 num_pending; + u64 seq; + struct semaphore job_sem; + bool job_done; + + /* Completed job counter */ + u64 completed; + + struct amdxdna_gem_obj *cmd_buf[HWCTX_MAX_CMDS]; + struct drm_syncobj *syncobj; +}; + +enum aie2_dev_status { + AIE2_DEV_UNINIT, + AIE2_DEV_INIT, + AIE2_DEV_START, +}; + +struct aie2_exec_msg_ops { + int (*init_cu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, + size_t *size, u32 *msg_op); + int (*init_dpu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, + size_t *size, u32 *msg_op); + void (*init_chain_req)(void *req, u64 slot_addr, size_t size, u32 cmd_cnt); + int (*fill_cf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); + int (*fill_dpu_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); + int (*fill_preempt_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); + int (*fill_elf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); + u32 (*get_chain_msg_op)(u32 cmd_op); +}; + +struct amdxdna_dev_hdl { + struct aie_device aie; + const struct amdxdna_dev_priv *priv; + void __iomem *sram_base; + void __iomem *mbox_base; + + u32 total_col; + struct amdxdna_drm_query_aie_version version; + struct aie2_exec_msg_ops *exec_msg_ops; + + /* power management and clock*/ + enum amdxdna_power_mode_type pw_mode; + u32 dpm_level; + u32 dft_dpm_level; + u32 max_dpm_level; + u32 clk_gating; + u32 npuclk_freq; + u32 hclk_freq; + u32 max_tops; + u32 curr_tops; + u32 force_preempt_enabled; + u32 frame_boundary_preempt; + + /* Mailbox and the management channel */ + struct mailbox *mbox; + struct async_events *async_events; + + enum aie2_dev_status dev_status; + u32 hwctx_num; + + struct amdxdna_async_error last_async_err; + unsigned long last_signal_ts; +}; + +struct aie2_hw_ops { + int (*set_dpm)(struct amdxdna_dev_hdl *ndev, u32 dpm_level); + int (*update_counters)(struct amdxdna_dev_hdl *ndev); +}; + +#define aie2_update_counters(ndev) \ +({ \ + typeof(ndev) _ndev = ndev; \ + if (_ndev->priv->hw_ops->update_counters) \ + _ndev->priv->hw_ops->update_counters(_ndev); \ +}) + +enum aie2_fw_feature { + AIE2_NPU_COMMAND, + AIE2_PREEMPT, + AIE2_TEMPORAL_ONLY, + AIE2_APP_HEALTH, + AIE2_ADD_HOST_BUFFER, + AIE2_UPDATE_PROPERTY, + AIE2_GET_DEV_REVISION, + AIE2_FEATURE_MAX +}; + +#define AIE2_ALL_FEATURES GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND) + +struct amdxdna_dev_priv { + const char *fw_path; + const struct rt_config *rt_config; + const struct dpm_clk_freq *dpm_clk_tbl; + +#define COL_ALIGN_NONE 0 +#define COL_ALIGN_NATURE 1 + u32 col_align; + u32 col_opc; + u32 mbox_dev_addr; + /* If mbox_size is 0, use BAR size. See MBOX_SIZE macro */ + u32 mbox_size; + u32 hwctx_limit; + u32 sram_dev_addr; + struct aie_bar_off_pair sram_offs[SRAM_MAX_INDEX]; + struct aie_bar_off_pair psp_regs_off[PSP_MAX_REGS]; + struct aie_bar_off_pair smu_regs_off[SMU_MAX_REGS]; + const struct aie2_hw_ops *hw_ops; +}; + +extern const struct amdxdna_dev_ops aie2_ops; + +int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, + enum rt_config_category category, u32 *val); + +/* aie2 npu hw config */ +extern const struct dpm_clk_freq npu1_dpm_clk_table[]; +extern const struct dpm_clk_freq npu4_dpm_clk_table[]; +extern const struct rt_config npu1_default_rt_cfg[]; +extern const struct rt_config npu4_default_rt_cfg[]; +extern const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[]; +extern const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[]; +extern const struct aie2_hw_ops npu4_hw_ops; + +/* aie2_pm.c */ +int aie2_pm_init(struct amdxdna_dev_hdl *ndev); +int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target); +int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); + +/* aie2_error.c */ +int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev); +void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev); +int aie2_error_async_msg_thread(void *data); +int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_get_array *args); + +/* aie2_message.c */ +void aie2_msg_init(struct amdxdna_dev_hdl *ndev); +void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev); +int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev); +int aie2_resume_fw(struct amdxdna_dev_hdl *ndev); +int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value); +int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value); +int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid); +int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_query_aie_version *version); +int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_query_aie_metadata *metadata); +int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, + struct amdxdna_fw_ver *fw_ver); +int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, + struct app_health_report *report); +int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev); +int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); +int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); +int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); +int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); +int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled); +int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, + char __user *buf, u32 size, + struct amdxdna_drm_query_telemetry_header *header); +int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, + void *handle, int (*cb)(void*, void __iomem *, size_t)); +int aie2_config_cu(struct amdxdna_hwctx *hwctx, + int (*notify_cb)(void *, void __iomem *, size_t)); +int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)); +int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, + struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)); +int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, + struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)); +int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)); +int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, + int (*notify_cb)(void *, void __iomem *, size_t)); +int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us); + +/* aie2_hwctx.c */ +int aie2_hwctx_init(struct amdxdna_hwctx *hwctx); +void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx); +int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size); +int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl); +void aie2_hwctx_suspend(struct amdxdna_client *client); +int aie2_hwctx_resume(struct amdxdna_client *client); +int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq); +void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq); +int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap); + +#endif /* _AIE2_PCI_H_ */ diff --git a/drivers/accel/amdxdna/aie2_pm.c b/drivers/accel/amdxdna/aie2_pm.c new file mode 100644 index 000000000..4fe6030d2 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_pm.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> + +#include "aie2_pci.h" +#include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" + +#define AIE2_CLK_GATING_ENABLE 1 +#define AIE2_CLK_GATING_DISABLE 0 + +static int aie2_pm_set_clk_gating(struct amdxdna_dev_hdl *ndev, u32 val) +{ + int ret; + + ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, &val); + if (ret) + return ret; + + ndev->clk_gating = val; + return 0; +} + +int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) +{ + int ret; + + ret = amdxdna_pm_resume_get_locked(ndev->aie.xdna); + if (ret) + return ret; + + ret = ndev->priv->hw_ops->set_dpm(ndev, dpm_level); + if (!ret) + ndev->dpm_level = dpm_level; + amdxdna_pm_suspend_put(ndev->aie.xdna); + + return ret; +} + +int aie2_pm_init(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + if (ndev->dev_status != AIE2_DEV_UNINIT) { + /* Resume device */ + ret = ndev->priv->hw_ops->set_dpm(ndev, ndev->dpm_level); + if (ret) + return ret; + + ret = aie2_pm_set_clk_gating(ndev, ndev->clk_gating); + if (ret) + return ret; + + return 0; + } + + while (ndev->priv->dpm_clk_tbl[ndev->max_dpm_level].hclk) + ndev->max_dpm_level++; + ndev->max_dpm_level--; + + ret = ndev->priv->hw_ops->set_dpm(ndev, ndev->max_dpm_level); + if (ret) + return ret; + ndev->dpm_level = ndev->max_dpm_level; + + ret = aie2_pm_set_clk_gating(ndev, AIE2_CLK_GATING_ENABLE); + if (ret) + return ret; + + ndev->pw_mode = POWER_MODE_DEFAULT; + ndev->dft_dpm_level = 0; + + return 0; +} + +int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + u32 clk_gating, dpm_level; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (ndev->pw_mode == target) + return 0; + + switch (target) { + case POWER_MODE_TURBO: + if (ndev->hwctx_num) { + XDNA_ERR(xdna, "Can not set turbo when there is active hwctx"); + return -EINVAL; + } + + clk_gating = AIE2_CLK_GATING_DISABLE; + dpm_level = ndev->max_dpm_level; + break; + case POWER_MODE_HIGH: + clk_gating = AIE2_CLK_GATING_ENABLE; + dpm_level = ndev->max_dpm_level; + break; + case POWER_MODE_DEFAULT: + clk_gating = AIE2_CLK_GATING_ENABLE; + dpm_level = ndev->dft_dpm_level; + break; + case POWER_MODE_LOW: + clk_gating = AIE2_CLK_GATING_ENABLE; + dpm_level = 0; + break; + case POWER_MODE_MEDIUM: + clk_gating = AIE2_CLK_GATING_ENABLE; + dpm_level = ndev->max_dpm_level / 2; + break; + default: + return -EOPNOTSUPP; + } + + ret = aie2_pm_set_dpm(ndev, dpm_level); + if (ret) + return ret; + + ret = aie2_pm_set_clk_gating(ndev, clk_gating); + if (ret) + return ret; + + ndev->pw_mode = target; + + return 0; +} diff --git a/drivers/accel/amdxdna/aie2_solver.c b/drivers/accel/amdxdna/aie2_solver.c new file mode 100644 index 000000000..6f3ee77d5 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_solver.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/drm_device.h> +#include <drm/drm_managed.h> +#include <drm/drm_print.h> +#include <linux/bitops.h> +#include <linux/bitmap.h> +#include <linux/slab.h> + +#include "aie2_solver.h" + +struct partition_node { + struct list_head list; + u32 nshared; /* # shared requests */ + u32 start_col; /* start column */ + u32 ncols; /* # columns */ + bool exclusive; /* can not be shared if set */ +}; + +struct solver_node { + struct list_head list; + u64 rid; /* Request ID from consumer */ + + struct partition_node *pt_node; + void *cb_arg; + u32 dpm_level; + u32 cols_len; + u32 start_cols[] __counted_by(cols_len); +}; + +struct solver_rgroup { + u32 rgid; + u32 nnode; + u32 npartition_node; + + DECLARE_BITMAP(resbit, XRS_MAX_COL); + struct list_head node_list; + struct list_head pt_node_list; +}; + +struct solver_state { + struct solver_rgroup rgp; + struct init_config cfg; + struct xrs_action_ops *actions; +}; + +static u32 calculate_gops(struct aie_qos *rqos) +{ + u32 service_rate = 0; + + if (rqos->latency) + service_rate = max_t(u32, 1000 / rqos->latency, 1); + + if (rqos->fps > service_rate) + return rqos->fps * rqos->gops; + + return service_rate * rqos->gops; +} + +/* + * qos_meet() - Check the QOS request can be met. + */ +static int qos_meet(struct solver_state *xrs, struct aie_qos *rqos, u32 cgops) +{ + u32 request_gops = calculate_gops(rqos) * xrs->cfg.sys_eff_factor; + + if (request_gops <= cgops) + return 0; + + return -EINVAL; +} + +/* + * sanity_check() - Do a basic sanity check on allocation request. + */ +static int sanity_check(struct solver_state *xrs, struct alloc_requests *req) +{ + struct cdo_parts *cdop = &req->cdo; + struct aie_qos *rqos = &req->rqos; + u32 cu_clk_freq; + + if (cdop->ncols > xrs->cfg.total_col) + return -EINVAL; + + /* + * We can find at least one CDOs groups that meet the + * GOPs requirement. + */ + cu_clk_freq = xrs->cfg.clk_list.cu_clk_list[xrs->cfg.clk_list.num_levels - 1]; + + if (qos_meet(xrs, rqos, cdop->qos_cap.opc * cu_clk_freq / 1000)) + return -EINVAL; + + return 0; +} + +static bool is_valid_qos_dpm_params(struct aie_qos *rqos) +{ + /* + * gops is retrieved from the xmodel, so it's always set + * fps and latency are the configurable params from the application + */ + if (rqos->gops > 0 && (rqos->fps > 0 || rqos->latency > 0)) + return true; + + return false; +} + +static int set_dpm_level(struct solver_state *xrs, struct alloc_requests *req, u32 *dpm_level) +{ + struct solver_rgroup *rgp = &xrs->rgp; + struct cdo_parts *cdop = &req->cdo; + struct aie_qos *rqos = &req->rqos; + u32 freq, max_dpm_level, level; + struct solver_node *node; + + max_dpm_level = xrs->cfg.clk_list.num_levels - 1; + /* If no QoS parameters are passed, set it to the max DPM level */ + if (!is_valid_qos_dpm_params(rqos)) { + level = max_dpm_level; + goto set_dpm; + } + + /* Find one CDO group that meet the GOPs requirement. */ + for (level = 0; level < max_dpm_level; level++) { + freq = xrs->cfg.clk_list.cu_clk_list[level]; + if (!qos_meet(xrs, rqos, cdop->qos_cap.opc * freq / 1000)) + break; + } + + /* set the dpm level which fits all the sessions */ + list_for_each_entry(node, &rgp->node_list, list) { + if (node->dpm_level > level) + level = node->dpm_level; + } + +set_dpm: + *dpm_level = level; + return xrs->cfg.actions->set_dft_dpm_level(xrs->cfg.ddev, level); +} + +static struct solver_node *rg_search_node(struct solver_rgroup *rgp, u64 rid) +{ + struct solver_node *node; + + list_for_each_entry(node, &rgp->node_list, list) { + if (node->rid == rid) + return node; + } + + return NULL; +} + +static void remove_partition_node(struct solver_rgroup *rgp, + struct partition_node *pt_node) +{ + pt_node->nshared--; + if (pt_node->nshared > 0) + return; + + list_del(&pt_node->list); + rgp->npartition_node--; + + bitmap_clear(rgp->resbit, pt_node->start_col, pt_node->ncols); + kfree(pt_node); +} + +static void remove_solver_node(struct solver_rgroup *rgp, + struct solver_node *node) +{ + list_del(&node->list); + rgp->nnode--; + + if (node->pt_node) + remove_partition_node(rgp, node->pt_node); + + kfree(node); +} + +static int get_free_partition(struct solver_state *xrs, + struct solver_node *snode, + struct alloc_requests *req) +{ + struct partition_node *pt_node; + u32 ncols = req->cdo.ncols; + u32 col, i; + + for (i = 0; i < snode->cols_len; i++) { + col = snode->start_cols[i]; + if (find_next_bit(xrs->rgp.resbit, XRS_MAX_COL, col) >= col + ncols) + break; + } + + if (i == snode->cols_len) + return -ENODEV; + + pt_node = kzalloc_obj(*pt_node); + if (!pt_node) + return -ENOMEM; + + pt_node->nshared = 1; + pt_node->start_col = col; + pt_node->ncols = ncols; + + /* + * Always set exclusive to false for now. + */ + pt_node->exclusive = false; + + list_add_tail(&pt_node->list, &xrs->rgp.pt_node_list); + xrs->rgp.npartition_node++; + bitmap_set(xrs->rgp.resbit, pt_node->start_col, pt_node->ncols); + + snode->pt_node = pt_node; + + return 0; +} + +static int allocate_partition(struct solver_state *xrs, + struct solver_node *snode, + struct alloc_requests *req) +{ + struct partition_node *pt_node, *rpt_node = NULL; + int idx, ret; + + ret = get_free_partition(xrs, snode, req); + if (!ret) + return ret; + + /* try to get a share-able partition */ + list_for_each_entry(pt_node, &xrs->rgp.pt_node_list, list) { + if (pt_node->exclusive) + continue; + + if (rpt_node && pt_node->nshared >= rpt_node->nshared) + continue; + + for (idx = 0; idx < snode->cols_len; idx++) { + if (snode->start_cols[idx] != pt_node->start_col) + continue; + + if (req->cdo.ncols != pt_node->ncols) + continue; + + rpt_node = pt_node; + break; + } + } + + if (!rpt_node) + return -ENODEV; + + rpt_node->nshared++; + snode->pt_node = rpt_node; + + return 0; +} + +static struct solver_node *create_solver_node(struct solver_state *xrs, + struct alloc_requests *req) +{ + struct cdo_parts *cdop = &req->cdo; + struct solver_node *node; + int ret; + + node = kzalloc_flex(*node, start_cols, cdop->cols_len); + if (!node) + return ERR_PTR(-ENOMEM); + + node->rid = req->rid; + node->cols_len = cdop->cols_len; + memcpy(node->start_cols, cdop->start_cols, cdop->cols_len * sizeof(u32)); + + ret = allocate_partition(xrs, node, req); + if (ret) + goto free_node; + + list_add_tail(&node->list, &xrs->rgp.node_list); + xrs->rgp.nnode++; + return node; + +free_node: + kfree(node); + return ERR_PTR(ret); +} + +static void fill_load_action(struct solver_state *xrs, + struct solver_node *snode, + struct xrs_action_load *action) +{ + action->rid = snode->rid; + action->part.start_col = snode->pt_node->start_col; + action->part.ncols = snode->pt_node->ncols; +} + +int xrs_allocate_resource(void *hdl, struct alloc_requests *req, void *cb_arg) +{ + struct xrs_action_load load_act; + struct solver_node *snode; + struct solver_state *xrs; + u32 dpm_level; + int ret; + + xrs = (struct solver_state *)hdl; + + ret = sanity_check(xrs, req); + if (ret) { + drm_err(xrs->cfg.ddev, "invalid request"); + return ret; + } + + if (rg_search_node(&xrs->rgp, req->rid)) { + drm_err(xrs->cfg.ddev, "rid %lld is in-use", req->rid); + return -EEXIST; + } + + snode = create_solver_node(xrs, req); + if (IS_ERR(snode)) + return PTR_ERR(snode); + + fill_load_action(xrs, snode, &load_act); + ret = xrs->cfg.actions->load(cb_arg, &load_act); + if (ret) + goto free_node; + + ret = set_dpm_level(xrs, req, &dpm_level); + if (ret) + goto free_node; + + snode->dpm_level = dpm_level; + snode->cb_arg = cb_arg; + + drm_dbg(xrs->cfg.ddev, "start col %d ncols %d\n", + snode->pt_node->start_col, snode->pt_node->ncols); + + return 0; + +free_node: + remove_solver_node(&xrs->rgp, snode); + + return ret; +} + +int xrs_release_resource(void *hdl, u64 rid) +{ + struct solver_state *xrs = hdl; + struct solver_node *node; + u32 level = 0; + + node = rg_search_node(&xrs->rgp, rid); + if (!node) { + drm_err(xrs->cfg.ddev, "node not exist"); + return -ENODEV; + } + + xrs->cfg.actions->unload(node->cb_arg); + remove_solver_node(&xrs->rgp, node); + + /* set the dpm level which fits all the sessions */ + list_for_each_entry(node, &xrs->rgp.node_list, list) { + if (node->dpm_level > level) + level = node->dpm_level; + } + xrs->cfg.actions->set_dft_dpm_level(xrs->cfg.ddev, level); + + return 0; +} + +void *xrsm_init(struct init_config *cfg) +{ + struct solver_rgroup *rgp; + struct solver_state *xrs; + + xrs = drmm_kzalloc(cfg->ddev, sizeof(*xrs), GFP_KERNEL); + if (!xrs) + return NULL; + + memcpy(&xrs->cfg, cfg, sizeof(*cfg)); + + rgp = &xrs->rgp; + INIT_LIST_HEAD(&rgp->node_list); + INIT_LIST_HEAD(&rgp->pt_node_list); + + return xrs; +} diff --git a/drivers/accel/amdxdna/aie2_solver.h b/drivers/accel/amdxdna/aie2_solver.h new file mode 100644 index 000000000..a2e3c5222 --- /dev/null +++ b/drivers/accel/amdxdna/aie2_solver.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE2_SOLVER_H +#define _AIE2_SOLVER_H + +#define XRS_MAX_COL 128 + +/* + * Structure used to describe a partition. A partition is column based + * allocation unit described by its start column and number of columns. + */ +struct aie_part { + u32 start_col; + u32 ncols; +}; + +/* + * The QoS capabilities of a given AIE partition. + */ +struct aie_qos_cap { + u32 opc; /* operations per cycle */ + u32 dma_bw; /* DMA bandwidth */ +}; + +/* + * QoS requirement of a resource allocation. + */ +struct aie_qos { + u32 gops; /* Giga operations */ + u32 fps; /* Frames per second */ + u32 dma_bw; /* DMA bandwidth */ + u32 latency; /* Frame response latency */ + u32 exec_time; /* Frame execution time */ + u32 priority; /* Request priority */ +}; + +/* + * Structure used to describe a relocatable CDO (Configuration Data Object). + */ +struct cdo_parts { + u32 *start_cols; /* Start column array */ + u32 cols_len; /* Length of start column array */ + u32 ncols; /* # of column */ + struct aie_qos_cap qos_cap; /* CDO QoS capabilities */ +}; + +/* + * Structure used to describe a request to allocate. + */ +struct alloc_requests { + u64 rid; + struct cdo_parts cdo; + struct aie_qos rqos; /* Requested QoS */ +}; + +/* + * Load callback argument + */ +struct xrs_action_load { + u32 rid; + struct aie_part part; +}; + +/* + * Define the power level available + * + * POWER_LEVEL_MIN: + * Lowest power level. Usually set when all actions are unloaded. + * + * POWER_LEVEL_n + * Power levels 0 - n, is a step increase in system frequencies + */ +enum power_level { + POWER_LEVEL_MIN = 0x0, + POWER_LEVEL_0 = 0x1, + POWER_LEVEL_1 = 0x2, + POWER_LEVEL_2 = 0x3, + POWER_LEVEL_3 = 0x4, + POWER_LEVEL_4 = 0x5, + POWER_LEVEL_5 = 0x6, + POWER_LEVEL_6 = 0x7, + POWER_LEVEL_7 = 0x8, + POWER_LEVEL_NUM, +}; + +/* + * Structure used to describe the frequency table. + * Resource solver chooses the frequency from the table + * to meet the QOS requirements. + */ +struct clk_list_info { + u32 num_levels; /* available power levels */ + u32 cu_clk_list[POWER_LEVEL_NUM]; /* available aie clock frequencies in Mhz*/ +}; + +struct xrs_action_ops { + int (*load)(void *cb_arg, struct xrs_action_load *action); + int (*unload)(void *cb_arg); + int (*set_dft_dpm_level)(struct drm_device *ddev, u32 level); +}; + +/* + * Structure used to describe information for solver during initialization. + */ +struct init_config { + u32 total_col; + u32 sys_eff_factor; /* system efficiency factor */ + u32 latency_adj; /* latency adjustment in ms */ + struct clk_list_info clk_list; /* List of frequencies available in system */ + struct drm_device *ddev; + struct xrs_action_ops *actions; +}; + +/* + * xrsm_init() - Register resource solver. Resource solver client needs + * to call this function to register itself. + * + * @cfg: The system metrics for resource solver to use + * + * Return: A resource solver handle + * + * Note: We should only create one handle per AIE array to be managed. + */ +void *xrsm_init(struct init_config *cfg); + +/* + * xrs_allocate_resource() - Request to allocate resources for a given context + * and a partition metadata. (See struct part_meta) + * + * @hdl: Resource solver handle obtained from xrs_init() + * @req: Input to the Resource solver including request id + * and partition metadata. + * @cb_arg: callback argument pointer + * + * Return: 0 when successful. + * Or standard error number when failing + * + * Note: + * There is no lock mechanism inside resource solver. So it is + * the caller's responsibility to lock down XCLBINs and grab + * necessary lock. + */ +int xrs_allocate_resource(void *hdl, struct alloc_requests *req, void *cb_arg); + +/* + * xrs_release_resource() - Request to free resources for a given context. + * + * @hdl: Resource solver handle obtained from xrs_init() + * @rid: The Request ID to identify the requesting context + */ +int xrs_release_resource(void *hdl, u64 rid); +#endif /* _AIE2_SOLVER_H */ diff --git a/drivers/accel/amdxdna/aie4_ctx.c b/drivers/accel/amdxdna/aie4_ctx.c new file mode 100644 index 000000000..8408b0d26 --- /dev/null +++ b/drivers/accel/amdxdna/aie4_ctx.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/types.h> + +#include "aie.h" +#include "aie4_host_queue.h" +#include "aie4_msg_priv.h" +#include "aie4_pci.h" +#include "amdxdna_ctx.h" +#include "amdxdna_gem.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_pci_drv.h" + +static irqreturn_t cert_comp_isr(int irq, void *p) +{ + struct cert_comp *cert_comp = p; + + wake_up_all(&cert_comp->waitq); + return IRQ_HANDLED; +} + +static struct cert_comp *aie4_lookup_cert_comp(struct amdxdna_dev_hdl *ndev, u32 msix_idx) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + struct cert_comp *cert_comp; + int ret; + + guard(mutex)(&ndev->cert_comp_lock); + + cert_comp = xa_load(&ndev->cert_comp_xa, msix_idx); + if (cert_comp) { + kref_get(&cert_comp->kref); + return cert_comp; + } + + cert_comp = kzalloc_obj(*cert_comp); + if (!cert_comp) + return NULL; + + cert_comp->ndev = ndev; + cert_comp->msix_idx = msix_idx; + init_waitqueue_head(&cert_comp->waitq); + kref_init(&cert_comp->kref); + + ret = pci_irq_vector(pdev, cert_comp->msix_idx); + if (ret < 0) { + XDNA_ERR(xdna, "MSI-X idx %u is invalid, ret:%d", msix_idx, ret); + goto free_cert_comp; + } + cert_comp->irq = ret; + + ret = request_irq(cert_comp->irq, cert_comp_isr, 0, "xdna_hsa", cert_comp); + if (ret) { + XDNA_ERR(xdna, "request irq %d failed %d", cert_comp->irq, ret); + goto free_cert_comp; + } + + ret = xa_err(xa_store(&ndev->cert_comp_xa, msix_idx, cert_comp, GFP_KERNEL)); + if (ret) { + XDNA_ERR(xdna, "store cert_comp for msix idx %d failed %d", msix_idx, ret); + goto free_irq; + } + + return cert_comp; + +free_irq: + free_irq(cert_comp->irq, cert_comp); +free_cert_comp: + kfree(cert_comp); + return NULL; +} + +static void cert_comp_release(struct kref *kref) +{ + struct cert_comp *cert_comp = container_of(kref, struct cert_comp, kref); + struct amdxdna_dev_hdl *ndev = cert_comp->ndev; + + drm_WARN_ON(&ndev->aie.xdna->ddev, !mutex_is_locked(&ndev->cert_comp_lock)); + + xa_erase(&ndev->cert_comp_xa, cert_comp->msix_idx); + free_irq(cert_comp->irq, cert_comp); + kfree(cert_comp); +} + +static void aie4_put_cert_comp(struct cert_comp *cert_comp) +{ + struct amdxdna_dev_hdl *ndev; + + ndev = cert_comp->ndev; + guard(mutex)(&ndev->cert_comp_lock); + kref_put(&cert_comp->kref, cert_comp_release); +} + +static int aie4_msg_destroy_context(struct amdxdna_dev_hdl *ndev, u32 hw_context_id) +{ + DECLARE_AIE_MSG(aie4_msg_destroy_hw_context, AIE4_MSG_OP_DESTROY_HW_CONTEXT); + + req.hw_context_id = hw_context_id; + return aie_send_mgmt_msg_wait(&ndev->aie, &msg); +} + +static int aie4_hwctx_create(struct amdxdna_hwctx *hwctx) +{ + DECLARE_AIE_MSG(aie4_msg_create_hw_context, AIE4_MSG_OP_CREATE_HW_CONTEXT); + struct amdxdna_client *client = hwctx->client; + struct amdxdna_hwctx_priv *priv = hwctx->priv; + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + int ret; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + if (!ndev->partition_id || !hwctx->num_tiles) { + XDNA_ERR(xdna, "invalid request partition_id %d, num_tiles %d", + ndev->partition_id, hwctx->num_tiles); + return -EINVAL; + } + + req.partition_id = ndev->partition_id; + req.request_num_tiles = hwctx->num_tiles; + req.pasid = FIELD_PREP(AIE4_MSG_PASID, client->pasid) | + FIELD_PREP(AIE4_MSG_PASID_VLD, 1); + req.priority_band = hwctx->qos.priority; + + req.hsa_addr_high = upper_32_bits(amdxdna_gem_dev_addr(priv->umq_bo)); + req.hsa_addr_low = lower_32_bits(amdxdna_gem_dev_addr(priv->umq_bo)); + + XDNA_DBG(xdna, "pasid 0x%x, num_tiles %d, hsa[0x%x 0x%x]", + req.pasid, req.request_num_tiles, req.hsa_addr_high, req.hsa_addr_low); + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "create ctx failed: %d", ret); + return ret; + } + + XDNA_DBG(xdna, "resp msix: %d, ctx id: %d, doorbell: %d", + resp.job_complete_msix_idx, + resp.hw_context_id, + resp.doorbell_offset); + + /* setup interrupt completion per msix index */ + priv->cert_comp = aie4_lookup_cert_comp(ndev, resp.job_complete_msix_idx); + if (!priv->cert_comp) { + aie4_msg_destroy_context(ndev, resp.hw_context_id); + return -EINVAL; + } + + priv->hw_ctx_id = resp.hw_context_id; + hwctx->doorbell_offset = resp.doorbell_offset; + + return 0; +} + +static void aie4_hwctx_destroy(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_hwctx_priv *priv = hwctx->priv; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + aie4_msg_destroy_context(ndev, priv->hw_ctx_id); + aie4_put_cert_comp(priv->cert_comp); +} + +static void aie4_hwctx_umq_fini(struct amdxdna_hwctx *hwctx) +{ + if (hwctx->priv && hwctx->priv->umq_bo) + amdxdna_gem_put_obj(hwctx->priv->umq_bo); +} + +static int aie4_hwctx_umq_init(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_hwctx_priv *priv = hwctx->priv; + struct amdxdna_dev *xdna = hwctx->client->xdna; + struct amdxdna_gem_obj *umq_bo; + struct host_queue_header *qhdr; + int ret; + + umq_bo = amdxdna_gem_get_obj(hwctx->client, hwctx->umq_bo_hdl, AMDXDNA_BO_SHARE); + if (!umq_bo) { + XDNA_ERR(xdna, "cannot find umq_bo handle %d", hwctx->umq_bo_hdl); + return -ENOENT; + } + if (umq_bo->mem.size < sizeof(*qhdr)) { + XDNA_ERR(xdna, "umq_bo size is too small"); + ret = -EINVAL; + goto put_umq_bo; + } + + /* get kva address for host queue read index and write index */ + qhdr = amdxdna_gem_vmap(umq_bo); + if (!qhdr) { + ret = -ENOMEM; + goto put_umq_bo; + } + + priv->umq_bo = umq_bo; + priv->umq_read_index = &qhdr->read_index; + priv->umq_write_index = &qhdr->write_index; + + return 0; + +put_umq_bo: + amdxdna_gem_put_obj(umq_bo); + return ret; +} + +int aie4_hwctx_init(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_hwctx_priv *priv; + int ret; + + priv = kzalloc_obj(*priv); + if (!priv) + return -ENOMEM; + hwctx->priv = priv; + + ret = aie4_hwctx_umq_init(hwctx); + if (ret) + goto free_priv; + + ret = aie4_hwctx_create(hwctx); + if (ret) + goto umq_fini; + + XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); + return 0; + +umq_fini: + aie4_hwctx_umq_fini(hwctx); +free_priv: + kfree(priv); + hwctx->priv = NULL; + return ret; +} + +void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx) +{ + aie4_hwctx_destroy(hwctx); + aie4_hwctx_umq_fini(hwctx); + kfree(hwctx->priv); +} + +static inline bool valid_queue_index(u64 read, u64 write, u32 capacity) +{ + return (write >= read) && ((write - read) <= capacity); +} + +static u64 get_read_index(struct amdxdna_hwctx *hwctx) +{ + u64 wi = READ_ONCE(*hwctx->priv->umq_write_index); + u64 ri = READ_ONCE(*hwctx->priv->umq_read_index); + struct amdxdna_dev *xdna = hwctx->client->xdna; + + /* + * CERT cannot update read index as uint64 atomically. Driver may read + * half-updated read index when it has bits in high 32bit. In case read + * index is not valid, wait for some time and retry once. It should + * allow CERT to complete the read index update. + */ + if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) { + XDNA_WARN(xdna, "Invalid index, ri %llu, wi %llu", ri, wi); + usleep_range(100, 200); + ri = READ_ONCE(*hwctx->priv->umq_read_index); + if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) { + XDNA_ERR(xdna, "Invalid index after retry, ri %llu, wi %llu", ri, wi); + ri = 0; + } + } + + return ri; +} + +static inline bool check_cmd_done(struct amdxdna_hwctx *hwctx, u64 seq) +{ + u64 read_idx = get_read_index(hwctx); + + return read_idx > seq; +} + +int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout) +{ + unsigned long wait_jifs = MAX_SCHEDULE_TIMEOUT; + struct amdxdna_hwctx_priv *priv = hwctx->priv; + struct cert_comp *cert_comp = priv->cert_comp; + long ret; + + if (timeout) + wait_jifs = msecs_to_jiffies(timeout); + + ret = wait_event_interruptible_timeout(cert_comp->waitq, + (check_cmd_done(hwctx, seq)), + wait_jifs); + + if (!ret) + ret = -ETIME; + + return ret <= 0 ? ret : 0; +} + +int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff) +{ + struct amdxdna_hwctx *hwctx; + unsigned long hwctx_id; + int idx; + + idx = srcu_read_lock(&client->hwctx_srcu); + amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { + if (vm_pgoff == (hwctx->doorbell_offset >> PAGE_SHIFT)) { + srcu_read_unlock(&client->hwctx_srcu, idx); + return 1; + } + } + srcu_read_unlock(&client->hwctx_srcu, idx); + + return 0; +} diff --git a/drivers/accel/amdxdna/aie4_host_queue.h b/drivers/accel/amdxdna/aie4_host_queue.h new file mode 100644 index 000000000..1b33eda3f --- /dev/null +++ b/drivers/accel/amdxdna/aie4_host_queue.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE4_HOST_QUEUE_H_ +#define _AIE4_HOST_QUEUE_H_ + +#include <linux/types.h> + +#define CTX_MAX_CMDS 32 + +struct host_queue_header { + __u64 read_index; + struct { + __u16 major; + __u16 minor; + } version; + __u32 capacity; /* Queue capacity, must be power of two. */ + __u64 write_index; + __u64 data_address; /* The xdna dev addr for payload. */ +}; + +#endif /* _AIE4_HOST_QUEUE_H_ */ diff --git a/drivers/accel/amdxdna/aie4_message.c b/drivers/accel/amdxdna/aie4_message.c new file mode 100644 index 000000000..88037edbb --- /dev/null +++ b/drivers/accel/amdxdna/aie4_message.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_print.h> +#include <linux/mutex.h> + +#include "aie.h" +#include "aie4_msg_priv.h" +#include "aie4_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_pci_drv.h" + +int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(aie4_msg_suspend, AIE4_MSG_OP_SUSPEND); + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret); + + return ret; +} + +int aie4_query_aie_metadata(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_query_aie_metadata *metadata) +{ + DECLARE_AIE_MSG(aie4_msg_aie4_tile_info, AIE4_MSG_OP_AIE_TILE_INFO); + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + return ret; + + metadata->col_size = resp.info.size; + metadata->cols = resp.info.cols; + metadata->rows = resp.info.rows; + + metadata->version.major = resp.info.major; + metadata->version.minor = resp.info.minor; + + metadata->core.row_count = resp.info.core_rows; + metadata->core.row_start = resp.info.core_row_start; + metadata->core.dma_channel_count = resp.info.core_dma_channels; + metadata->core.lock_count = resp.info.core_locks; + metadata->core.event_reg_count = resp.info.core_events; + + metadata->mem.row_count = resp.info.mem_rows; + metadata->mem.row_start = resp.info.mem_row_start; + metadata->mem.dma_channel_count = resp.info.mem_dma_channels; + metadata->mem.lock_count = resp.info.mem_locks; + metadata->mem.event_reg_count = resp.info.mem_events; + + metadata->shim.row_count = resp.info.shim_rows; + metadata->shim.row_start = resp.info.shim_row_start; + metadata->shim.dma_channel_count = resp.info.shim_dma_channels; + metadata->shim.lock_count = resp.info.shim_locks; + metadata->shim.event_reg_count = resp.info.shim_events; + + return 0; +} + +int aie4_attach_work_buffer(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(aie4_msg_attach_work_buffer, AIE4_MSG_OP_ATTACH_WORK_BUFFER); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + req.buff_addr = ndev->work_buf_addr; + req.buff_size = AIE4_WORK_BUFFER_MIN_SIZE; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + XDNA_ERR(xdna, "Failed to attach work buffer, ret %d", ret); + else + XDNA_DBG(xdna, "Attached work buffer"); + + return ret; +} diff --git a/drivers/accel/amdxdna/aie4_msg_priv.h b/drivers/accel/amdxdna/aie4_msg_priv.h new file mode 100644 index 000000000..af0866045 --- /dev/null +++ b/drivers/accel/amdxdna/aie4_msg_priv.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE4_MSG_PRIV_H_ +#define _AIE4_MSG_PRIV_H_ + +#include <linux/sizes.h> +#include <linux/types.h> + +enum aie4_msg_opcode { + AIE4_MSG_OP_SUSPEND = 0x10003, + AIE4_MSG_OP_ATTACH_WORK_BUFFER = 0x1000D, + + AIE4_MSG_OP_CREATE_VFS = 0x20001, + AIE4_MSG_OP_DESTROY_VFS = 0x20002, + + AIE4_MSG_OP_CREATE_PARTITION = 0x30001, + AIE4_MSG_OP_DESTROY_PARTITION = 0x30002, + AIE4_MSG_OP_CREATE_HW_CONTEXT = 0x30003, + AIE4_MSG_OP_DESTROY_HW_CONTEXT = 0x30004, + AIE4_MSG_OP_AIE_TILE_INFO = 0x30006, +}; + +enum aie4_msg_status { + AIE4_MSG_STATUS_SUCCESS = 0x0, + AIE4_MSG_STATUS_ERROR = 0x1, + AIE4_MSG_STATUS_NOTSUPP = 0x2, + MAX_AIE4_MSG_STATUS_CODE = 0x4, +}; + +struct aie4_msg_suspend_req { + __u32 rsvd; +} __packed; + +struct aie4_msg_suspend_resp { + enum aie4_msg_status status; +} __packed; + +struct aie4_msg_create_vfs_req { + __u32 vf_cnt; +} __packed; + +struct aie4_msg_create_vfs_resp { + enum aie4_msg_status status; +} __packed; + +struct aie4_msg_destroy_vfs_req { + __u32 rsvd; +} __packed; + +struct aie4_msg_destroy_vfs_resp { + enum aie4_msg_status status; +} __packed; + +struct aie4_msg_create_partition_req { + __u32 partition_col_start; + __u32 partition_col_count; +} __packed; + +struct aie4_msg_create_partition_resp { + enum aie4_msg_status status; + __u32 partition_id; +} __packed; + +struct aie4_msg_destroy_partition_req { + __u32 partition_id; +} __packed; + +struct aie4_msg_destroy_partition_resp { + enum aie4_msg_status status; +} __packed; + +struct aie4_msg_create_hw_context_req { + __u32 partition_id; + __u32 request_num_tiles; + __u32 hsa_addr_high; + __u32 hsa_addr_low; +#define AIE4_MSG_PASID GENMASK(19, 0) +#define AIE4_MSG_PASID_VLD GENMASK(31, 31) + __u32 pasid; + __u32 priority_band; +} __packed; + +struct aie4_msg_create_hw_context_resp { + enum aie4_msg_status status; + __u32 hw_context_id; + __u32 doorbell_offset; + __u32 job_complete_msix_idx; +} __packed; + +struct aie4_msg_destroy_hw_context_req { + __u32 hw_context_id; + __u32 resvd1; +} __packed; + +struct aie4_msg_destroy_hw_context_resp { + enum aie4_msg_status status; +} __packed; + +struct aie4_tile_info { + __u32 size; + __u16 major; + __u16 minor; + __u16 cols; + __u16 rows; + __u16 core_rows; + __u16 mem_rows; + __u16 shim_rows; + __u16 core_row_start; + __u16 mem_row_start; + __u16 shim_row_start; + __u16 core_dma_channels; + __u16 mem_dma_channels; + __u16 shim_dma_channels; + __u16 core_locks; + __u16 mem_locks; + __u16 shim_locks; + __u16 core_events; + __u16 mem_events; + __u16 shim_events; + __u16 resvd; +} __packed; + +struct aie4_msg_aie4_tile_info_req { + __u32 resvd; +} __packed; + +struct aie4_msg_aie4_tile_info_resp { + enum aie4_msg_status status; + struct aie4_tile_info info; +} __packed; + +#define AIE4_WORK_BUFFER_MIN_SIZE SZ_4M + +struct aie4_msg_attach_work_buffer_req { + __u64 buff_addr; + __u32 reserved; + __u32 buff_size; +} __packed; + +struct aie4_msg_attach_work_buffer_resp { + enum aie4_msg_status status; +} __packed; + +#endif /* _AIE4_MSG_PRIV_H_ */ diff --git a/drivers/accel/amdxdna/aie4_pci.c b/drivers/accel/amdxdna/aie4_pci.c new file mode 100644 index 000000000..a58a83af4 --- /dev/null +++ b/drivers/accel/amdxdna/aie4_pci.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_managed.h> +#include <drm/drm_print.h> +#include <linux/firmware.h> +#include <linux/sizes.h> + +#include "aie.h" +#include "aie4_msg_priv.h" +#include "aie4_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_pci_drv.h" + +#define NO_IOHUB 0 +#define PSP_NOTIFY_INTR 0xD007BE11 +#define AIE4_TOTAL_COLUMN 3 + +/* + * The management mailbox channel is allocated by firmware. + * The related register and ring buffer information is on SRAM BAR. + * This struct is the register layout. + */ +struct mailbox_info { + __u32 valid; + __u32 protocol_major; + __u32 protocol_minor; + __u32 x2i_tail_offset; + __u32 x2i_head_offset; + __u32 x2i_buffer_addr; + __u32 x2i_buffer_size; + __u32 i2x_tail_offset; + __u32 i2x_head_offset; + __u32 i2x_buffer_addr; + __u32 i2x_buffer_size; + __u32 i2x_msi_idx; + __u32 reserved[4]; +}; + +static int aie4_fw_is_alive(struct amdxdna_dev *xdna) +{ + const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + u32 __iomem *src; + u32 fw_is_valid; + int ret; + + src = ndev->rbuf_base + npriv->mbox_info_off; + + ret = readx_poll_timeout(readl, src + offsetof(struct mailbox_info, valid), + fw_is_valid, (fw_is_valid == 0x1), + AIE_INTERVAL, AIE_TIMEOUT); + if (ret) + XDNA_ERR(xdna, "fw_is_valid=%d after %d ms", + fw_is_valid, DIV_ROUND_CLOSEST(AIE_TIMEOUT, 1000000)); + + return ret; +} + +static void aie4_read_mbox_info(struct amdxdna_dev *xdna, + struct mailbox_info *mbox_info) +{ + const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + u32 *dst = (u32 *)mbox_info; + u32 __iomem *src; + int i; + + src = ndev->rbuf_base + npriv->mbox_info_off; + + for (i = 0; i < sizeof(*mbox_info) / sizeof(u32); i++) + dst[i] = readl(&src[i]); +} + +static int aie4_mailbox_info(struct amdxdna_dev *xdna, + struct mailbox_info *mbox_info) +{ + int ret; + + ret = aie4_fw_is_alive(xdna); + if (ret) + return ret; + + aie4_read_mbox_info(xdna, mbox_info); + + ret = aie_check_protocol(&xdna->dev_handle->aie, + mbox_info->protocol_major, + mbox_info->protocol_minor); + if (ret) + XDNA_ERR(xdna, "mailbox major.minor %d.%d is not supported", + mbox_info->protocol_major, mbox_info->protocol_minor); + + return ret; +} + +static void aie4_mailbox_fini(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + + aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann); + drmm_kfree(&xdna->ddev, ndev->mbox); + ndev->mbox = NULL; +} + +static int aie4_irq_init(struct amdxdna_dev *xdna) +{ + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + int ret, nvec; + + nvec = pci_msix_vec_count(pdev); + XDNA_DBG(xdna, "irq vectors:%d", nvec); + if (nvec <= 0) { + XDNA_ERR(xdna, "does not get number of interrupt vector"); + return -EINVAL; + } + + ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); + if (ret < 0) { + XDNA_ERR(xdna, "failed to alloc irq vector, ret: %d", ret); + return ret; + } + + return 0; +} + +static int aie4_mailbox_start(struct amdxdna_dev *xdna, + struct mailbox_info *mbi) +{ + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; + struct xdna_mailbox_chann_res *i2x; + struct xdna_mailbox_chann_res *x2i; + int mgmt_mb_irq; + int ret; + + struct xdna_mailbox_res mbox_res = { + .ringbuf_base = ndev->rbuf_base, + .ringbuf_size = pci_resource_len(pdev, npriv->mbox_rbuf_bar), + .mbox_base = ndev->mbox_base, + .mbox_size = pci_resource_len(pdev, npriv->mbox_bar), + .name = "xdna_aie4_mailbox", + }; + + i2x = &ndev->aie.mgmt_i2x; + x2i = &ndev->aie.mgmt_x2i; + + x2i->mb_head_ptr_reg = mbi->x2i_head_offset; + x2i->mb_tail_ptr_reg = mbi->x2i_tail_offset; + x2i->rb_start_addr = mbi->x2i_buffer_addr; + x2i->rb_size = mbi->x2i_buffer_size; + + i2x->rb_start_addr = mbi->i2x_buffer_addr; + i2x->rb_size = mbi->i2x_buffer_size; + i2x->mb_head_ptr_reg = mbi->i2x_head_offset; + i2x->mb_tail_ptr_reg = mbi->i2x_tail_offset; + + ndev->aie.mgmt_chan_idx = mbi->i2x_msi_idx; + aie_dump_mgmt_chann_debug(&ndev->aie); + + ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res); + if (!ndev->mbox) { + XDNA_ERR(xdna, "failed to create mailbox device"); + return -ENODEV; + } + + ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox); + if (!ndev->aie.mgmt_chann) { + XDNA_ERR(xdna, "failed to alloc mailbox channel"); + return -ENODEV; + } + + mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx); + if (mgmt_mb_irq < 0) { + XDNA_ERR(xdna, "failed to alloc irq vector, return %d", mgmt_mb_irq); + ret = mgmt_mb_irq; + goto free_channel; + } + + ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann, + &ndev->aie.mgmt_x2i, + &ndev->aie.mgmt_i2x, + NO_IOHUB, + mgmt_mb_irq); + if (ret) { + XDNA_ERR(xdna, "failed to start management mailbox channel"); + ret = -EINVAL; + goto free_channel; + } + + XDNA_DBG(xdna, "Mailbox management channel created"); + return 0; + +free_channel: + xdna_mailbox_free_channel(ndev->aie.mgmt_chann); + ndev->aie.mgmt_chann = NULL; + return ret; +} + +static int aie4_mailbox_init(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct mailbox_info mbox_info; + int ret; + + ret = aie4_mailbox_info(xdna, &mbox_info); + if (ret) + return ret; + + return aie4_mailbox_start(xdna, &mbox_info); +} + +static void aie4_fw_stop(struct amdxdna_dev_hdl *ndev) +{ + aie_psp_stop(ndev->aie.psp_hdl); + aie_smu_fini(ndev->aie.smu_hdl); +} + +static int aie4_fw_start(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + ret = aie_smu_init(ndev->aie.smu_hdl); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "failed to init smu, ret %d", ret); + return ret; + } + + ret = aie_psp_start(ndev->aie.psp_hdl); + if (ret) { + XDNA_ERR(ndev->aie.xdna, "failed to start psp, ret %d", ret); + aie_smu_fini(ndev->aie.smu_hdl); + } + + return ret; +} + +static int aie4_partition_init(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(aie4_msg_create_partition, AIE4_MSG_OP_CREATE_PARTITION); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + req.partition_col_start = 0; + req.partition_col_count = AIE4_TOTAL_COLUMN; + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) { + XDNA_ERR(xdna, "partition init failed: %d", ret); + return ret; + } + + ndev->partition_id = resp.partition_id; + return 0; +} + +static void aie4_partition_fini(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(aie4_msg_destroy_partition, AIE4_MSG_OP_DESTROY_PARTITION); + struct amdxdna_dev *xdna = ndev->aie.xdna; + int ret; + + req.partition_id = ndev->partition_id; + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + XDNA_ERR(xdna, "partition fini failed: %d", ret); +} + +static int aie4_query(struct amdxdna_dev_hdl *ndev) +{ + return aie4_query_aie_metadata(ndev, &ndev->aie.metadata); +} + +static int aie4_pf_hw_start(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + ret = aie4_fw_start(ndev); + if (ret) + return ret; + + ret = aie4_mailbox_init(ndev); + if (ret) + goto stop_fw; + + ret = aie4_attach_work_buffer(ndev); + if (ret) + goto mbox_fini; + + return 0; + +mbox_fini: + aie4_mailbox_fini(ndev); +stop_fw: + aie4_fw_stop(ndev); + + return ret; +} + +static void aie4_pf_hw_stop(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + aie4_suspend_fw(ndev); + aie4_mailbox_fini(ndev); + aie4_fw_stop(ndev); +} + +static int aie4_vf_hw_start(struct amdxdna_dev_hdl *ndev) +{ + int ret; + + ret = aie4_mailbox_init(ndev); + if (ret) + return ret; + + ret = aie4_query(ndev); + if (ret) + goto mailbox_fini; + + ret = aie4_partition_init(ndev); + if (ret) + goto mailbox_fini; + + return 0; + +mailbox_fini: + aie4_mailbox_fini(ndev); + return ret; +} + +static void aie4_vf_hw_stop(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + aie4_partition_fini(ndev); + aie4_mailbox_fini(ndev); +} + +static int aie4_request_firmware(struct amdxdna_dev_hdl *ndev, + const struct firmware **npufw, + const struct firmware **certfw) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + char fw_name[128]; + int ret; + + ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s", + pdev->device, pdev->revision, ndev->priv->npufw_path); + if (ret >= sizeof(fw_name)) { + XDNA_ERR(xdna, "npu firmware path is truncated"); + return -EINVAL; + } + + ret = request_firmware(npufw, fw_name, &pdev->dev); + if (ret) { + XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret); + return ret; + } + + ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s", + pdev->device, pdev->revision, ndev->priv->certfw_path); + if (ret >= sizeof(fw_name)) { + XDNA_ERR(xdna, "cert firmware path is truncated"); + ret = -EINVAL; + goto release_npufw; + } + + ret = request_firmware(certfw, fw_name, &pdev->dev); + if (ret) { + XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret); + goto release_npufw; + } + + return 0; + +release_npufw: + release_firmware(*npufw); + + return ret; +} + +static void aie4_release_firmware(struct amdxdna_dev_hdl *ndev, + const struct firmware *npufw, + const struct firmware *certfw) +{ + release_firmware(certfw); + release_firmware(npufw); +} + +static int aie4_prepare_firmware(struct amdxdna_dev_hdl *ndev, + const struct firmware *npufw, + const struct firmware *certfw, + void __iomem *tbl[PCI_NUM_RESOURCES]) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct psp_config psp_conf; + struct smu_config smu_conf; + int i; + + psp_conf.fw_size = npufw->size; + psp_conf.fw_buf = npufw->data; + psp_conf.certfw_size = certfw->size; + psp_conf.certfw_buf = certfw->data; + psp_conf.arg2_mask = ~0; + psp_conf.notify_val = PSP_NOTIFY_INTR; + for (i = 0; i < PSP_MAX_REGS; i++) + psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i); + ndev->aie.psp_hdl = aiem_psp_create(&xdna->ddev, &psp_conf); + if (!ndev->aie.psp_hdl) { + XDNA_ERR(xdna, "failed to create psp"); + return -ENOMEM; + } + + for (i = 0; i < SMU_MAX_REGS; i++) + smu_conf.smu_regs[i] = tbl[SMU_REG_BAR(ndev, i)] + SMU_REG_OFF(ndev, i); + ndev->aie.smu_hdl = aiem_smu_create(&xdna->ddev, &smu_conf); + if (!ndev->aie.smu_hdl) { + XDNA_ERR(xdna, "failed to create smu"); + return -ENOMEM; + } + + return 0; +} + +static int aie4_load_fw(struct amdxdna_dev_hdl *ndev, + void __iomem *tbl[PCI_NUM_RESOURCES]) +{ + const struct firmware *npufw, *certfw; + int ret; + + if (!ndev->priv->npufw_path && !ndev->priv->certfw_path) + return 0; + + ret = aie4_request_firmware(ndev, &npufw, &certfw); + if (ret) + return ret; + + ret = aie4_prepare_firmware(ndev, npufw, certfw, tbl); + aie4_release_firmware(ndev, npufw, certfw); + + return ret; +} + +static int aie4m_pcidev_init(struct amdxdna_dev *xdna) +{ + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + struct amdxdna_dev_hdl *ndev; + void __iomem *tbl[PCI_NUM_RESOURCES] = {0}; + unsigned long bars = 0; + int ret, i; + + ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL); + if (!ndev) + return -ENOMEM; + + ndev->priv = xdna->dev_info->dev_priv; + ndev->aie.xdna = xdna; + xdna->dev_handle = ndev; + + xa_init_flags(&ndev->cert_comp_xa, XA_FLAGS_ALLOC); + mutex_init(&ndev->cert_comp_lock); + + /* Enable managed PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret); + return ret; + } + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + XDNA_ERR(xdna, "failed to set DMA mask to 64:%d", ret); + return ret; + } + + for (i = 0; i < PSP_MAX_REGS; i++) + set_bit(PSP_REG_BAR(ndev, i), &bars); + for (i = 0; i < SMU_MAX_REGS; i++) + set_bit(SMU_REG_BAR(ndev, i), &bars); + set_bit(xdna->dev_info->mbox_bar, &bars); + set_bit(xdna->dev_info->sram_bar, &bars); + + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + if (!test_bit(i, &bars)) + continue; + tbl[i] = pcim_iomap(pdev, i, 0); + if (!tbl[i]) { + XDNA_ERR(xdna, "map bar %d failed", i); + return -ENOMEM; + } + } + + ndev->mbox_base = tbl[xdna->dev_info->mbox_bar]; + ndev->rbuf_base = tbl[xdna->dev_info->sram_bar]; + + pci_set_master(pdev); + + ret = aie4_load_fw(ndev, tbl); + if (ret) + return ret; + + ret = aie4_irq_init(xdna); + if (ret) + return ret; + + amdxdna_vbnv_init(xdna); + XDNA_DBG(xdna, "init finished"); + + return 0; +} + +static int aie4_doorbell_mmap(struct amdxdna_client *client, struct vm_area_struct *vma) +{ + struct amdxdna_dev *xdna = client->xdna; + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; + phys_addr_t res_start; + unsigned long pfn; + int ret; + + if (!aie4_hwctx_valid_doorbell(client, vma->vm_pgoff)) { + XDNA_ERR(xdna, "Invalid doorbell page offset 0x%lx", vma->vm_pgoff); + return -EINVAL; + } + + if (vma_pages(vma) != 1) { + XDNA_ERR(xdna, "can only map one page, got %ld", vma_pages(vma)); + return -EINVAL; + } + + res_start = pci_resource_start(pdev, xdna->dev_info->doorbell_bar) + npriv->doorbell_off; + pfn = PHYS_PFN(res_start) + vma->vm_pgoff; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP); + ret = io_remap_pfn_range(vma, vma->vm_start, + pfn, + PAGE_SIZE, + vma->vm_page_prot); + + XDNA_DBG(xdna, "doorbell ret %d", ret); + return ret; +} + +static int aie4_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) +{ + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + int ret; + + switch (args->param) { + case DRM_AMDXDNA_QUERY_AIE_METADATA: + ret = amdxdna_get_metadata(&ndev->aie, client, args); + break; + default: + XDNA_ERR(xdna, "Not supported request parameter %u", args->param); + ret = -EOPNOTSUPP; + } + + XDNA_DBG(xdna, "Got param %d", args->param); + + return ret; +} + +static int aie4_alloc_work_buffer(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + u32 buf_size = AIE4_WORK_BUFFER_MIN_SIZE; + + ndev->work_buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, + &ndev->work_buf_addr); + if (IS_ERR(ndev->work_buf)) { + int ret = PTR_ERR(ndev->work_buf); + + XDNA_ERR(xdna, "Failed to alloc work buffer, size 0x%x", + AIE4_WORK_BUFFER_MIN_SIZE); + ndev->work_buf = NULL; + return ret; + } + + ndev->work_buf_size = buf_size; + XDNA_DBG(xdna, "Work buffer allocated: size 0x%x", buf_size); + + return 0; +} + +static void aie4_free_work_buffer(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + + if (!ndev->work_buf) + return; + + amdxdna_free_msg_buffer(xdna, ndev->work_buf_size, ndev->work_buf, + ndev->work_buf_addr); + ndev->work_buf = NULL; +} + +static int aie4_pf_init(struct amdxdna_dev *xdna) +{ + int ret; + + ret = aie4m_pcidev_init(xdna); + if (ret) + return ret; + + ret = aie4_alloc_work_buffer(xdna->dev_handle); + if (ret) + return ret; + + ret = aie4_pf_hw_start(xdna->dev_handle); + if (ret) + goto free_work_buf; + + return 0; + +free_work_buf: + aie4_free_work_buffer(xdna->dev_handle); + return ret; +} + +static int aie4_vf_init(struct amdxdna_dev *xdna) +{ + int ret; + + ret = aie4m_pcidev_init(xdna); + if (ret) + return ret; + + return aie4_vf_hw_start(xdna->dev_handle); +} + +static void aie4_pf_fini(struct amdxdna_dev *xdna) +{ + aie4_sriov_stop(xdna->dev_handle); + aie4_pf_hw_stop(xdna->dev_handle); + aie4_free_work_buffer(xdna->dev_handle); +} + +static void aie4_vf_fini(struct amdxdna_dev *xdna) +{ + aie4_vf_hw_stop(xdna->dev_handle); +} + +const struct amdxdna_dev_ops aie4_pf_ops = { + .init = aie4_pf_init, + .fini = aie4_pf_fini, + .sriov_configure = aie4_sriov_configure, +}; + +const struct amdxdna_dev_ops aie4_vf_ops = { + .init = aie4_vf_init, + .fini = aie4_vf_fini, + .hwctx_init = aie4_hwctx_init, + .hwctx_fini = aie4_hwctx_fini, + .mmap = aie4_doorbell_mmap, + .cmd_wait = aie4_cmd_wait, + .get_aie_info = aie4_get_info, +}; diff --git a/drivers/accel/amdxdna/aie4_pci.h b/drivers/accel/amdxdna/aie4_pci.h new file mode 100644 index 000000000..3fd5eace3 --- /dev/null +++ b/drivers/accel/amdxdna/aie4_pci.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE4_PCI_H_ +#define _AIE4_PCI_H_ + +#include <linux/device.h> +#include <linux/iopoll.h> +#include <linux/pci.h> + +#include "aie.h" +#include "amdxdna_mailbox.h" + +struct cert_comp { + struct amdxdna_dev_hdl *ndev; + u32 msix_idx; + int irq; + struct kref kref; + wait_queue_head_t waitq; +}; + +struct amdxdna_hwctx_priv { + struct amdxdna_gem_obj *umq_bo; + u64 *umq_read_index; + u64 *umq_write_index; + + struct cert_comp *cert_comp; + u32 hw_ctx_id; +}; + +struct amdxdna_dev_priv { + const char *npufw_path; + const char *certfw_path; + u32 mbox_bar; + u32 mbox_rbuf_bar; + u64 mbox_info_off; + u32 doorbell_off; + + struct aie_bar_off_pair psp_regs_off[PSP_MAX_REGS]; + struct aie_bar_off_pair smu_regs_off[SMU_MAX_REGS]; +}; + +struct amdxdna_dev_hdl { + struct aie_device aie; + const struct amdxdna_dev_priv *priv; + void __iomem *mbox_base; + void __iomem *rbuf_base; + + struct mailbox *mbox; + u32 partition_id; + + struct xarray cert_comp_xa; /* device level indexed by msix id */ + struct mutex cert_comp_lock; /* protects cert_comp operations*/ + + void *work_buf; + dma_addr_t work_buf_addr; + u32 work_buf_size; +}; + +/* aie4_message.c */ +int aie4_query_aie_metadata(struct amdxdna_dev_hdl *ndev, + struct amdxdna_drm_query_aie_metadata *metadata); +int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev); +int aie4_attach_work_buffer(struct amdxdna_dev_hdl *ndev); + +/* aie4_ctx.c */ +int aie4_hwctx_init(struct amdxdna_hwctx *hwctx); +void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx); +int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout); +int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff); + +/* aie4_sriov.c */ +#if IS_ENABLED(CONFIG_PCI_IOV) +int aie4_sriov_configure(struct amdxdna_dev *xdna, int num_vfs); +int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev); +#else +#define aie4_sriov_configure NULL +static inline int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev) +{ + return 0; +} +#endif + +extern const struct amdxdna_dev_ops aie4_pf_ops; +extern const struct amdxdna_dev_ops aie4_vf_ops; + +#endif /* _AIE4_PCI_H_ */ diff --git a/drivers/accel/amdxdna/aie4_sriov.c b/drivers/accel/amdxdna/aie4_sriov.c new file mode 100644 index 000000000..e1ce63376 --- /dev/null +++ b/drivers/accel/amdxdna/aie4_sriov.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_print.h> +#include <linux/pci.h> + +#include "aie.h" +#include "aie4_msg_priv.h" +#include "aie4_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_pci_drv.h" + +static int aie4_destroy_vfs(struct amdxdna_dev_hdl *ndev) +{ + DECLARE_AIE_MSG(aie4_msg_destroy_vfs, AIE4_MSG_OP_DESTROY_VFS); + int ret; + + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + XDNA_ERR(ndev->aie.xdna, "destroy vfs op failed: %d", ret); + + return ret; +} + +static int aie4_create_vfs(struct amdxdna_dev_hdl *ndev, int num_vfs) +{ + DECLARE_AIE_MSG(aie4_msg_create_vfs, AIE4_MSG_OP_CREATE_VFS); + int ret; + + req.vf_cnt = num_vfs; + ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); + if (ret) + XDNA_ERR(ndev->aie.xdna, "create vfs op failed: %d", ret); + + return ret; +} + +int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + int ret; + + if (!pci_num_vf(pdev)) + return 0; + + ret = pci_vfs_assigned(pdev); + if (ret) { + XDNA_ERR(xdna, "VFs are still assigned to VMs"); + return -EPERM; + } + + pci_disable_sriov(pdev); + return aie4_destroy_vfs(ndev); +} + +static int aie4_sriov_start(struct amdxdna_dev_hdl *ndev, int num_vfs) +{ + struct amdxdna_dev *xdna = ndev->aie.xdna; + struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); + int ret; + + ret = aie4_create_vfs(ndev, num_vfs); + if (ret) + return ret; + + ret = pci_enable_sriov(pdev, num_vfs); + if (ret) { + XDNA_ERR(xdna, "configure VFs failed, ret: %d", ret); + aie4_destroy_vfs(ndev); + return ret; + } + + return num_vfs; +} + +int aie4_sriov_configure(struct amdxdna_dev *xdna, int num_vfs) +{ + struct amdxdna_dev_hdl *ndev = xdna->dev_handle; + + drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); + + return (num_vfs) ? aie4_sriov_start(ndev, num_vfs) : aie4_sriov_stop(ndev); +} diff --git a/drivers/accel/amdxdna/aie_psp.c b/drivers/accel/amdxdna/aie_psp.c new file mode 100644 index 000000000..458dca7cc --- /dev/null +++ b/drivers/accel/amdxdna/aie_psp.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/drm_device.h> +#include <drm/drm_managed.h> +#include <drm/drm_print.h> +#include <linux/bitfield.h> +#include <linux/iopoll.h> +#include <linux/slab.h> + +#include "aie.h" + +#define PSP_STATUS_READY BIT(31) + +/* PSP commands */ +#define PSP_VALIDATE 1 +#define PSP_START 2 +#define PSP_RELEASE_TMR 3 +#define PSP_VALIDATE_CERT 4 + +/* PSP special arguments */ +#define PSP_START_COPY_FW 1 + +/* PSP response error code */ +#define PSP_ERROR_CANCEL 0xFFFF0002 +#define PSP_ERROR_BAD_STATE 0xFFFF0007 + +#define PSP_FW_ALIGN 0x10000 +#define PSP_CFW_ALIGN 0x8000 +#define PSP_POLL_INTERVAL 20000 /* us */ +#define PSP_POLL_TIMEOUT 1000000 /* us */ + +#define PSP_REG(p, reg) ((p)->conf.psp_regs[reg]) +#define PSP_SET_CMD(psp, reg_vals, cmd, arg0, arg1, arg2) \ +({ \ + u32 *_regs = reg_vals; \ + u32 _cmd = cmd; \ + _regs[0] = _cmd; \ + _regs[1] = arg0; \ + _regs[2] = arg1; \ + _regs[3] = ((arg2) | ((_cmd) << 24)) & (psp)->conf.arg2_mask; \ +}) + +struct psp_device { + struct drm_device *ddev; + struct psp_config conf; + u32 fw_buf_sz; + u64 fw_paddr; + void *fw_buffer; + u32 certfw_buf_sz; + u64 certfw_paddr; + void *certfw_buffer; +}; + +static int psp_exec(struct psp_device *psp, u32 *reg_vals) +{ + u32 resp_code; + int ret, i; + u32 ready; + + /* Check for PSP ready before any write */ + ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_STATUS_REG), ready, + FIELD_GET(PSP_STATUS_READY, ready), + PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT); + if (ret) { + drm_err(psp->ddev, "PSP is not ready, ret 0x%x", ret); + return ret; + } + + /* Write command and argument registers */ + for (i = 0; i < PSP_NUM_IN_REGS; i++) + writel(reg_vals[i], PSP_REG(psp, i)); + + /* clear and set PSP INTR register to kick off */ + writel(0, PSP_REG(psp, PSP_INTR_REG)); + writel(psp->conf.notify_val, PSP_REG(psp, PSP_INTR_REG)); + + /* PSP should be busy. Wait for ready, so we know task is done. */ + ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_STATUS_REG), ready, + FIELD_GET(PSP_STATUS_READY, ready), + PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT); + if (ret) { + drm_err(psp->ddev, "PSP is not ready, ret 0x%x", ret); + return ret; + } + + resp_code = readl(PSP_REG(psp, PSP_RESP_REG)); + if (resp_code) { + drm_err(psp->ddev, "fw return error 0x%x", resp_code); + return -EIO; + } + + return 0; +} + +int aie_psp_waitmode_poll(struct psp_device *psp) +{ + struct amdxdna_dev *xdna = to_xdna_dev(psp->ddev); + u32 mode_reg; + int ret; + + ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_PWAITMODE_REG), mode_reg, + (mode_reg & 0x1) == 1, + PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT); + if (ret) + XDNA_ERR(xdna, "fw waitmode reg error, ret %d", ret); + + return ret; +} + +void aie_psp_stop(struct psp_device *psp) +{ + u32 reg_vals[PSP_NUM_IN_REGS]; + int ret; + + PSP_SET_CMD(psp, reg_vals, PSP_RELEASE_TMR, 0, 0, 0); + + ret = psp_exec(psp, reg_vals); + if (ret) + drm_err(psp->ddev, "release tmr failed, ret %d", ret); +} + +static int psp_validate_fw(struct psp_device *psp, u8 cmd, u64 paddr, u32 buf_sz) +{ + u32 reg_vals[PSP_NUM_IN_REGS]; + int ret; + + PSP_SET_CMD(psp, reg_vals, cmd, lower_32_bits(paddr), + upper_32_bits(paddr), buf_sz); + + ret = psp_exec(psp, reg_vals); + if (ret) + drm_err(psp->ddev, "failed to validate fw, ret %d", ret); + + return ret; +} + +static int psp_start(struct psp_device *psp) +{ + u32 reg_vals[PSP_NUM_IN_REGS]; + int ret; + + PSP_SET_CMD(psp, reg_vals, PSP_START, PSP_START_COPY_FW, 0, 0); + + ret = psp_exec(psp, reg_vals); + if (ret) + drm_err(psp->ddev, "failed to start fw, ret %d", ret); + + return ret; +} + +int aie_psp_start(struct psp_device *psp) +{ + int ret; + + ret = psp_validate_fw(psp, PSP_VALIDATE, + psp->fw_paddr, psp->fw_buf_sz); + if (ret) + return ret; + + if (!psp->certfw_buf_sz) + goto psp_start; + + ret = psp_validate_fw(psp, PSP_VALIDATE_CERT, + psp->certfw_paddr, psp->certfw_buf_sz); + if (ret) + return ret; +psp_start: + return psp_start(psp); +} + +/* + * PSP requires host physical address to load firmware. + * Allocate a buffer, obtain its physical address, align, and copy data in. + */ +static void *psp_alloc_fw_buf(struct psp_device *psp, const void *fw_data, + u32 fw_size, u32 align, u32 *buf_sz, + u64 *paddr) +{ + u32 alloc_sz; + void *buffer; + u64 offset; + + *buf_sz = ALIGN(fw_size, align); + alloc_sz = *buf_sz + align; + + buffer = drmm_kmalloc(psp->ddev, alloc_sz, GFP_KERNEL); + if (!buffer) + return NULL; + + *paddr = virt_to_phys(buffer); + offset = ALIGN(*paddr, align) - *paddr; + *paddr += offset; + memcpy(buffer + offset, fw_data, fw_size); + + return buffer; +} + +struct psp_device *aiem_psp_create(struct drm_device *ddev, struct psp_config *conf) +{ + struct psp_device *psp; + + psp = drmm_kzalloc(ddev, sizeof(*psp), GFP_KERNEL); + if (!psp) + return NULL; + + psp->ddev = ddev; + psp->fw_buffer = psp_alloc_fw_buf(psp, conf->fw_buf, conf->fw_size, + PSP_FW_ALIGN, &psp->fw_buf_sz, + &psp->fw_paddr); + if (!psp->fw_buffer) + return NULL; + + if (!conf->certfw_size) { + drm_dbg(ddev, "no cert fw"); + goto done; + } + + /* CERT firmware */ + psp->certfw_buffer = psp_alloc_fw_buf(psp, conf->certfw_buf, + conf->certfw_size, PSP_CFW_ALIGN, + &psp->certfw_buf_sz, + &psp->certfw_paddr); + if (!psp->certfw_buffer) { + drm_err(ddev, "no memory for cert fw buffer"); + return NULL; + } + +done: + memcpy(&psp->conf, conf, sizeof(psp->conf)); + + return psp; +} diff --git a/drivers/accel/amdxdna/aie_smu.c b/drivers/accel/amdxdna/aie_smu.c new file mode 100644 index 000000000..62aea550a --- /dev/null +++ b/drivers/accel/amdxdna/aie_smu.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include "drm/amdxdna_accel.h" +#include <drm/drm_device.h> +#include <drm/drm_managed.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/iopoll.h> + +#include "aie.h" + +#define SMU_RESULT_OK 1 + +/* SMU commands */ +#define AIE_SMU_POWER_ON 0x3 +#define AIE_SMU_POWER_OFF 0x4 +#define AIE_SMU_SET_MPNPUCLK_FREQ 0x5 +#define AIE_SMU_SET_HCLK_FREQ 0x6 +#define AIE_SMU_SET_SOFT_DPMLEVEL 0x7 +#define AIE_SMU_SET_HARD_DPMLEVEL 0x8 + +#define SMU_REG(s, reg) ((s)->smu_regs[reg]) + +struct smu_device { + struct drm_device *ddev; + struct smu_config conf; + void __iomem *smu_regs[SMU_MAX_REGS]; +}; + +static int aie_smu_exec(struct smu_device *smu, u32 reg_cmd, u32 reg_arg, u32 *out) +{ + u32 resp; + int ret; + + writel(0, SMU_REG(smu, SMU_RESP_REG)); + writel(reg_arg, SMU_REG(smu, SMU_ARG_REG)); + writel(reg_cmd, SMU_REG(smu, SMU_CMD_REG)); + + /* Clear and set SMU_INTR_REG to kick off */ + writel(0, SMU_REG(smu, SMU_INTR_REG)); + writel(1, SMU_REG(smu, SMU_INTR_REG)); + + ret = readx_poll_timeout(readl, SMU_REG(smu, SMU_RESP_REG), resp, + resp, AIE_INTERVAL, AIE_TIMEOUT); + if (ret) { + drm_err(smu->ddev, "smu cmd %d timed out", reg_cmd); + return ret; + } + + if (out) + *out = readl(SMU_REG(smu, SMU_OUT_REG)); + + if (resp != SMU_RESULT_OK) { + drm_err(smu->ddev, "smu cmd %d failed, 0x%x", reg_cmd, resp); + return -EINVAL; + } + + return 0; +} + +int aie_smu_init(struct smu_device *smu) +{ + int ret; + + /* + * Failing to set power off indicates an unrecoverable hardware or + * firmware error. + */ + ret = aie_smu_exec(smu, AIE_SMU_POWER_OFF, 0, NULL); + if (ret) { + drm_err(smu->ddev, "Access power failed, ret %d", ret); + return ret; + } + + ret = aie_smu_exec(smu, AIE_SMU_POWER_ON, 0, NULL); + if (ret) { + drm_err(smu->ddev, "Power on failed, ret %d", ret); + return ret; + } + + return 0; +} + +void aie_smu_fini(struct smu_device *smu) +{ + int ret; + + ret = aie_smu_exec(smu, AIE_SMU_POWER_OFF, 0, NULL); + if (ret) + drm_err(smu->ddev, "Power off failed, ret %d", ret); +} + +int aie_smu_set_clocks(struct smu_device *smu, u32 *npuclk, u32 *hclk) +{ + int ret; + + if (npuclk) { + ret = aie_smu_exec(smu, AIE_SMU_SET_MPNPUCLK_FREQ, *npuclk, npuclk); + if (ret) { + drm_err(smu->ddev, "Set mpnpu clock to %d failed, ret %d", *npuclk, ret); + return ret; + } + } + + if (hclk) { + ret = aie_smu_exec(smu, AIE_SMU_SET_HCLK_FREQ, *hclk, hclk); + if (ret) { + drm_err(smu->ddev, "Set hclock to %d failed, ret %d", + *hclk, ret); + return ret; + } + } + + return 0; +} + +int aie_smu_set_dpm(struct smu_device *smu, u32 dpm_level) +{ + int ret; + + ret = aie_smu_exec(smu, AIE_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL); + if (ret) { + drm_err(smu->ddev, "Set hard dpm level %d failed, ret %d", + dpm_level, ret); + return ret; + } + + ret = aie_smu_exec(smu, AIE_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL); + if (ret) { + drm_err(smu->ddev, "Set soft dpm level %d failed, ret %d", + dpm_level, ret); + return ret; + } + + return 0; +} + +struct smu_device *aiem_smu_create(struct drm_device *ddev, struct smu_config *conf) +{ + struct smu_device *smu; + + smu = drmm_kzalloc(ddev, sizeof(*smu), GFP_KERNEL); + if (!smu) + return NULL; + + smu->ddev = ddev; + memcpy(smu->smu_regs, conf->smu_regs, sizeof(smu->smu_regs)); + + return smu; +} diff --git a/drivers/accel/amdxdna/amdxdna_cbuf.c b/drivers/accel/amdxdna/amdxdna_cbuf.c new file mode 100644 index 000000000..a504560aa --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_cbuf.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/drm_mm.h> +#include <drm/drm_prime.h> + +#include "amdxdna_cbuf.h" +#include "amdxdna_pci_drv.h" + +/* + * Carveout memory is a chunk of memory which is physically contiguous and + * is reserved during early boot time. There is only one chunk of such memory + * per device. Once available, all BOs accessible from device should be + * allocated from this memory. This is a platform debug/bringup feature. + */ +struct amdxdna_carveout { + u64 addr; + u64 size; + struct drm_mm mm; + struct mutex lock; /* protect mm */ +}; + +bool amdxdna_use_carveout(struct amdxdna_dev *xdna) +{ + return !!xdna->carveout; +} + +void amdxdna_get_carveout_conf(struct amdxdna_dev *xdna, u64 *addr, u64 *size) +{ + if (amdxdna_use_carveout(xdna)) { + *addr = xdna->carveout->addr; + *size = xdna->carveout->size; + } else { + *addr = 0; + *size = 0; + } +} + +int amdxdna_carveout_init(struct amdxdna_dev *xdna, u64 carveout_addr, u64 carveout_size) +{ + struct amdxdna_carveout *carveout; + + /* Only allow carveout memory to be set up once. */ + if (amdxdna_use_carveout(xdna)) { + XDNA_ERR(xdna, "Carveout memory has already been set up."); + return -EBUSY; + } + + carveout = kzalloc_obj(*carveout); + if (!carveout) + return -ENOMEM; + + carveout->addr = carveout_addr; + carveout->size = carveout_size; + mutex_init(&carveout->lock); + drm_mm_init(&carveout->mm, carveout->addr, carveout->size); + + xdna->carveout = carveout; + XDNA_INFO(xdna, "Use carveout mem: 0x%llx@0x%llx\n", carveout->size, carveout->addr); + return 0; +} + +void amdxdna_carveout_fini(struct amdxdna_dev *xdna) +{ + struct amdxdna_carveout *carveout = xdna->carveout; + + if (!amdxdna_use_carveout(xdna)) + return; + + XDNA_INFO(xdna, "Cleanup carveout mem: 0x%llx@0x%llx\n", carveout->size, carveout->addr); + drm_mm_takedown(&carveout->mm); + mutex_destroy(&carveout->lock); + kfree(carveout); + xdna->carveout = NULL; +} + +struct amdxdna_cbuf_priv { + struct amdxdna_dev *xdna; + struct drm_mm_node node; +}; + +static struct sg_table *amdxdna_cbuf_map(struct dma_buf_attachment *attach, + enum dma_data_direction direction) +{ + struct amdxdna_cbuf_priv *cbuf = attach->dmabuf->priv; + struct device *dev = attach->dev; + struct scatterlist *sgl, *sg; + int ret, n_entries, i; + struct sg_table *sgt; + dma_addr_t dma_addr; + size_t dma_size; + size_t max_seg; + + sgt = kzalloc_obj(*sgt); + if (!sgt) + return ERR_PTR(-ENOMEM); + + max_seg = min_t(size_t, UINT_MAX, dma_max_mapping_size(dev)); + n_entries = (cbuf->node.size + max_seg - 1) / max_seg; + sgl = kzalloc_objs(*sg, n_entries); + if (!sgl) { + ret = -ENOMEM; + goto free_sgt; + } + sg_init_table(sgl, n_entries); + sgt->orig_nents = n_entries; + sgt->nents = n_entries; + sgt->sgl = sgl; + + dma_size = cbuf->node.size; + dma_addr = dma_map_resource(dev, cbuf->node.start, dma_size, + direction, DMA_ATTR_SKIP_CPU_SYNC); + ret = dma_mapping_error(dev, dma_addr); + if (ret) { + pr_err("Failed to dma_map_resource carveout dma buf, ret %d\n", ret); + goto free_sgl; + } + + for_each_sgtable_dma_sg(sgt, sg, i) { + size_t len = min_t(size_t, max_seg, dma_size); + + sg_dma_address(sg) = dma_addr; + sg_dma_len(sg) = len; + dma_addr += len; + dma_size -= len; + } + + return sgt; + +free_sgl: + kfree(sgl); +free_sgt: + kfree(sgt); + return ERR_PTR(ret); +} + +static void amdxdna_cbuf_unmap(struct dma_buf_attachment *attach, + struct sg_table *sgt, + enum dma_data_direction direction) +{ + dma_unmap_resource(attach->dev, sg_dma_address(sgt->sgl), + drm_prime_get_contiguous_size(sgt), direction, + DMA_ATTR_SKIP_CPU_SYNC); + sg_free_table(sgt); + kfree(sgt); +} + +static void amdxdna_cbuf_release(struct dma_buf *dbuf) +{ + struct amdxdna_cbuf_priv *cbuf = dbuf->priv; + struct amdxdna_carveout *carveout; + + carveout = cbuf->xdna->carveout; + mutex_lock(&carveout->lock); + drm_mm_remove_node(&cbuf->node); + mutex_unlock(&carveout->lock); + + kfree(cbuf); +} + +static vm_fault_t amdxdna_cbuf_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct amdxdna_cbuf_priv *cbuf; + unsigned long pfn; + pgoff_t pgoff; + + cbuf = vma->vm_private_data; + pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pfn = (cbuf->node.start >> PAGE_SHIFT) + pgoff; + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + +static const struct vm_operations_struct amdxdna_cbuf_vm_ops = { + .fault = amdxdna_cbuf_vm_fault, +}; + +static int amdxdna_cbuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma) +{ + struct amdxdna_cbuf_priv *cbuf = dbuf->priv; + + vma->vm_ops = &amdxdna_cbuf_vm_ops; + vma->vm_private_data = cbuf; + vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); + + return 0; +} + +static int amdxdna_cbuf_vmap(struct dma_buf *dbuf, struct iosys_map *map) +{ + struct amdxdna_cbuf_priv *cbuf = dbuf->priv; + void *kva; + + kva = memremap(cbuf->node.start, cbuf->node.size, MEMREMAP_WB); + if (!kva) { + pr_err("Failed to vmap carveout dma buf\n"); + return -ENOMEM; + } + + iosys_map_set_vaddr(map, kva); + return 0; +} + +static void amdxdna_cbuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map) +{ + memunmap(map->vaddr); +} + +static const struct dma_buf_ops amdxdna_cbuf_dmabuf_ops = { + .map_dma_buf = amdxdna_cbuf_map, + .unmap_dma_buf = amdxdna_cbuf_unmap, + .release = amdxdna_cbuf_release, + .mmap = amdxdna_cbuf_mmap, + .vmap = amdxdna_cbuf_vmap, + .vunmap = amdxdna_cbuf_vunmap, +}; + +static int amdxdna_cbuf_clear(struct dma_buf *dbuf) +{ + struct iosys_map vmap = IOSYS_MAP_INIT_VADDR(NULL); + + dma_buf_vmap(dbuf, &vmap); + if (!vmap.vaddr) + return -EFAULT; + + memset(vmap.vaddr, 0, dbuf->size); + dma_buf_vunmap(dbuf, &vmap); + + return 0; +} + +struct dma_buf *amdxdna_get_cbuf(struct drm_device *dev, size_t size, u64 alignment) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct amdxdna_carveout *carveout; + struct amdxdna_cbuf_priv *cbuf; + struct dma_buf *dbuf; + int ret; + + cbuf = kzalloc_obj(*cbuf); + if (!cbuf) + return ERR_PTR(-ENOMEM); + cbuf->xdna = xdna; + + carveout = xdna->carveout; + mutex_lock(&carveout->lock); + ret = drm_mm_insert_node_generic(&carveout->mm, &cbuf->node, size, + alignment, 0, DRM_MM_INSERT_BEST); + mutex_unlock(&carveout->lock); + if (ret) + goto free_cbuf; + + exp_info.size = size; + exp_info.ops = &amdxdna_cbuf_dmabuf_ops; + exp_info.priv = cbuf; + exp_info.flags = O_RDWR; + dbuf = dma_buf_export(&exp_info); + if (IS_ERR(dbuf)) { + ret = PTR_ERR(dbuf); + goto remove_node; + } + + ret = amdxdna_cbuf_clear(dbuf); + if (ret) { + dma_buf_put(dbuf); + goto out; + } + return dbuf; + +remove_node: + drm_mm_remove_node(&cbuf->node); +free_cbuf: + kfree(cbuf); +out: + return ERR_PTR(ret); +} diff --git a/drivers/accel/amdxdna/amdxdna_cbuf.h b/drivers/accel/amdxdna/amdxdna_cbuf.h new file mode 100644 index 000000000..8e89336ff --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_cbuf.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ +#ifndef _AMDXDNA_CBUF_H_ +#define _AMDXDNA_CBUF_H_ + +#include "amdxdna_pci_drv.h" +#include <drm/drm_device.h> +#include <linux/dma-buf.h> + +bool amdxdna_use_carveout(struct amdxdna_dev *xdna); +int amdxdna_carveout_init(struct amdxdna_dev *xdna, u64 carveout_addr, u64 carveout_size); +void amdxdna_carveout_fini(struct amdxdna_dev *xdna); +void amdxdna_get_carveout_conf(struct amdxdna_dev *xdna, u64 *addr, u64 *size); +struct dma_buf *amdxdna_get_cbuf(struct drm_device *dev, size_t size, u64 alignment); + +#endif diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c new file mode 100644 index 000000000..31a414c3f --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_drv.h> +#include <drm/drm_file.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/xarray.h> +#include <trace/events/amdxdna.h> + +#include "amdxdna_ctx.h" +#include "amdxdna_gem.h" +#include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" + +#define MAX_HWCTX_ID 255 +#define MAX_ARG_COUNT 4095 + +struct amdxdna_fence { + struct dma_fence base; + spinlock_t lock; /* for base */ + struct amdxdna_hwctx *hwctx; +}; + +static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence) +{ + return KBUILD_MODNAME; +} + +static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence) +{ + struct amdxdna_fence *xdna_fence; + + xdna_fence = container_of(fence, struct amdxdna_fence, base); + + return xdna_fence->hwctx->name; +} + +static const struct dma_fence_ops fence_ops = { + .get_driver_name = amdxdna_fence_get_driver_name, + .get_timeline_name = amdxdna_fence_get_timeline_name, +}; + +static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_fence *fence; + + fence = kzalloc_obj(*fence); + if (!fence) + return NULL; + + fence->hwctx = hwctx; + spin_lock_init(&fence->lock); + dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0); + return &fence->base; +} + +static void amdxdna_hwctx_release_expanded_heap(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_gem_obj *heap; + unsigned long heap_id; + + mutex_lock(&client->mm_lock); + if (hwctx->last_attached_heap) { + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, 1, + hwctx->last_attached_heap) { + amdxdna_gem_unpin(heap); + drm_gem_object_put(to_gobj(heap)); + } + } + mutex_unlock(&client->mm_lock); +} + +static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx, + struct srcu_struct *ss) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + + synchronize_srcu(ss); + + /* At this point, user is not able to submit new commands */ + xdna->dev_info->ops->hwctx_fini(hwctx); + + amdxdna_hwctx_release_expanded_heap(hwctx); + kfree(hwctx->name); + kfree(hwctx); +} + +int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg, + int (*walk)(struct amdxdna_hwctx *hwctx, void *arg)) +{ + struct amdxdna_hwctx *hwctx; + unsigned long hwctx_id; + int ret = 0, idx; + + idx = srcu_read_lock(&client->hwctx_srcu); + amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { + ret = walk(hwctx, arg); + if (ret) + break; + } + srcu_read_unlock(&client->hwctx_srcu, idx); + + return ret; +} + +void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) +{ + struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); + u32 num_masks, count; + + if (!cmd) + return NULL; + + if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) + num_masks = 0; + else + num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); + + if (size) { + count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); + if (unlikely(count <= num_masks || + count * sizeof(u32) + + offsetof(struct amdxdna_cmd, data[0]) > + abo->mem.size)) { + *size = 0; + return NULL; + } + *size = (count - num_masks) * sizeof(u32); + } + return &cmd->data[num_masks]; +} + +u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); + u32 num_masks, i; + u32 *cu_mask; + + if (!cmd) + return INVALID_CU_IDX; + + if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) + return INVALID_CU_IDX; + + num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); + cu_mask = cmd->data; + for (i = 0; i < num_masks; i++) { + if (cu_mask[i]) + return ffs(cu_mask[i]) - 1; + } + + return INVALID_CU_IDX; +} + +int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, + struct amdxdna_sched_job *job, u32 cmd_idx, + enum ert_cmd_state error_state, + void *err_data, size_t size) +{ + struct amdxdna_client *client = job->hwctx->client; + struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); + struct amdxdna_cmd_chain *cc = NULL; + + if (!cmd) + return -ENOMEM; + + cmd->header &= ~AMDXDNA_CMD_STATE; + cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state); + + if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) { + cc = amdxdna_cmd_get_payload(abo, NULL); + cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0; + abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_SHARE); + if (!abo) + return -EINVAL; + cmd = amdxdna_gem_vmap(abo); + if (!cmd) + return -ENOMEM; + } + + memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd)); + if (err_data) + memcpy(cmd->data, err_data, min(size, abo->mem.size - sizeof(*cmd))); + + if (cc) + amdxdna_gem_put_obj(abo); + + return 0; +} + +/* + * This should be called in close() and remove(). DO NOT call in other syscalls. + * This guarantee that when hwctx and resources will be released, if user + * doesn't call amdxdna_drm_destroy_hwctx_ioctl. + */ +void amdxdna_hwctx_remove_all(struct amdxdna_client *client) +{ + struct amdxdna_hwctx *hwctx; + unsigned long hwctx_id; + + amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { + XDNA_DBG(client->xdna, "PID %d close HW context %d", + client->pid, hwctx->id); + xa_erase(&client->hwctx_xa, hwctx->id); + amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); + } +} + +int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_drm_create_hwctx *args = data; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_hwctx *hwctx; + int ret, idx; + + if (args->ext || args->ext_flags) + return -EINVAL; + + if (!xdna->dev_info->ops->hwctx_init) + return -EOPNOTSUPP; + + hwctx = kzalloc_obj(*hwctx); + if (!hwctx) + return -ENOMEM; + + if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) { + XDNA_ERR(xdna, "Access QoS info failed"); + kfree(hwctx); + return -EFAULT; + } + + hwctx->client = client; + hwctx->fw_ctx_id = -1; + hwctx->num_tiles = args->num_tiles; + hwctx->umq_bo_hdl = args->umq_bo; + hwctx->doorbell_offset = AMDXDNA_INVALID_DOORBELL_OFFSET; + hwctx->mem_size = args->mem_size; + hwctx->max_opc = args->max_opc; + + guard(mutex)(&xdna->dev_lock); + + if (!drm_dev_enter(dev, &idx)) { + ret = -ENODEV; + goto free_hwctx; + } + + ret = xdna->dev_info->ops->hwctx_init(hwctx); + if (ret) { + XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret); + goto release_expanded_heap; + } + + hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id); + if (!hwctx->name) { + ret = -ENOMEM; + goto fini_hwctx; + } + + ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx, + XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID), + &client->next_hwctxid, GFP_KERNEL); + if (ret < 0) { + XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret); + goto free_name; + } + + args->handle = hwctx->id; + args->syncobj_handle = hwctx->syncobj_hdl; + args->umq_doorbell = hwctx->doorbell_offset; + + atomic64_set(&hwctx->job_submit_cnt, 0); + atomic64_set(&hwctx->job_free_cnt, 0); + XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret); + drm_dev_exit(idx); + return 0; + +free_name: + kfree(hwctx->name); +fini_hwctx: + xdna->dev_info->ops->hwctx_fini(hwctx); +release_expanded_heap: + amdxdna_hwctx_release_expanded_heap(hwctx); + drm_dev_exit(idx); +free_hwctx: + kfree(hwctx); + return ret; +} + +int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_drm_destroy_hwctx *args = data; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_hwctx *hwctx; + int ret = 0, idx; + + if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) + return -EINVAL; + + if (!drm_dev_enter(dev, &idx)) + return -ENODEV; + + mutex_lock(&xdna->client_lock); + mutex_lock(&xdna->dev_lock); + hwctx = xa_erase(&client->hwctx_xa, args->handle); + if (!hwctx) { + ret = -EINVAL; + XDNA_DBG(xdna, "PID %d HW context %d not exist", + client->pid, args->handle); + goto out; + } + + /* + * The pushed jobs are handled by DRM scheduler during destroy. + * SRCU to synchronize with exec command ioctls. + */ + amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); + + XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle); +out: + mutex_unlock(&xdna->dev_lock); + mutex_unlock(&xdna->client_lock); + drm_dev_exit(idx); + return ret; +} + +int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_drm_config_hwctx *args = data; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_hwctx *hwctx; + u32 buf_size; + void *buf; + int ret; + u64 val; + + if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) + return -EINVAL; + + if (!xdna->dev_info->ops->hwctx_config) + return -EOPNOTSUPP; + + val = args->param_val; + buf_size = args->param_val_size; + + switch (args->param_type) { + case DRM_AMDXDNA_HWCTX_CONFIG_CU: + /* For those types that param_val is pointer */ + if (buf_size > PAGE_SIZE) { + XDNA_ERR(xdna, "Config CU param buffer too large"); + return -E2BIG; + } + + /* Hwctx needs to keep buf */ + buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) { + kfree(buf); + return -EFAULT; + } + + break; + case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: + case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: + /* For those types that param_val is a value */ + buf = NULL; + buf_size = 0; + break; + default: + XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type); + return -EINVAL; + } + + ret = amdxdna_pm_resume_get(xdna); + if (ret) { + XDNA_ERR(xdna, "Resume failed, ret %d", ret); + goto free_buf; + } + + mutex_lock(&xdna->client_lock); + mutex_lock(&xdna->dev_lock); + hwctx = xa_load(&client->hwctx_xa, args->handle); + if (!hwctx) { + XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); + ret = -EINVAL; + goto unlock; + } + + ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); + +unlock: + mutex_unlock(&xdna->dev_lock); + mutex_unlock(&xdna->client_lock); + amdxdna_pm_suspend_put(xdna); +free_buf: + kfree(buf); + return ret; +} + +int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) +{ + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_hwctx *hwctx; + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + int ret; + + if (!xdna->dev_info->ops->hwctx_sync_debug_bo) + return -EOPNOTSUPP; + + gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl); + if (!gobj) + return -EINVAL; + + ret = amdxdna_pm_resume_get(xdna); + if (ret) { + XDNA_ERR(xdna, "Resume failed, ret %d", ret); + goto put_obj; + } + + abo = to_xdna_obj(gobj); + mutex_lock(&xdna->client_lock); + mutex_lock(&xdna->dev_lock); + hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx); + if (!hwctx) { + ret = -EINVAL; + goto unlock; + } + + ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl); + +unlock: + mutex_unlock(&xdna->dev_lock); + mutex_unlock(&xdna->client_lock); + amdxdna_pm_suspend_put(xdna); +put_obj: + drm_gem_object_put(gobj); + return ret; +} + +static int amdxdna_hwctx_expand_heap(struct amdxdna_hwctx *hwctx) +{ + struct amdxdna_client *client = hwctx->client; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_gem_obj *heap; + unsigned long heap_id, nid; + int ret = 0; + + nid = hwctx->last_attached_heap + 1; + if (nid == client->dev_heap_nid) + goto out; + + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, + nid, client->dev_heap_nid) { + drm_gem_object_get(to_gobj(heap)); + ret = amdxdna_gem_pin(heap); + if (ret) { + drm_gem_object_put(to_gobj(heap)); + break; + } + + ret = xdna->dev_info->ops->hwctx_heap_expand(hwctx, heap); + if (ret) { + amdxdna_gem_unpin(heap); + drm_gem_object_put(to_gobj(heap)); + break; + } + + hwctx->last_attached_heap = heap_id; + } + +out: + return ret; +} + +int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx) +{ + unsigned long hwctx_id; + int ret; + + ret = amdxdna_pm_resume_get_locked(client->xdna); + if (ret) + return ret; + + mutex_lock(&client->mm_lock); + + if (hwctx) { + ret = amdxdna_hwctx_expand_heap(hwctx); + } else { + amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { + ret = amdxdna_hwctx_expand_heap(hwctx); + if (ret) + break; + } + } + mutex_unlock(&client->mm_lock); + + amdxdna_pm_suspend_put(client->xdna); + + return ret; +} + +static void +amdxdna_arg_bos_put(struct amdxdna_sched_job *job) +{ + int i; + + for (i = 0; i < job->bo_cnt; i++) { + if (!job->bos[i]) + break; + drm_gem_object_put(job->bos[i]); + } +} + +static int +amdxdna_arg_bos_lookup(struct amdxdna_client *client, + struct amdxdna_sched_job *job, + u32 *bo_hdls, u32 bo_cnt) +{ + struct drm_gem_object *gobj; + int i, ret; + + job->bo_cnt = bo_cnt; + for (i = 0; i < job->bo_cnt; i++) { + struct amdxdna_gem_obj *abo; + + gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]); + if (!gobj) { + ret = -ENOENT; + goto put_shmem_bo; + } + abo = to_xdna_obj(gobj); + + mutex_lock(&abo->lock); + if (abo->pinned) { + mutex_unlock(&abo->lock); + job->bos[i] = gobj; + continue; + } + + ret = amdxdna_gem_pin_nolock(abo); + if (ret) { + mutex_unlock(&abo->lock); + drm_gem_object_put(gobj); + goto put_shmem_bo; + } + abo->pinned = true; + mutex_unlock(&abo->lock); + + job->bos[i] = gobj; + } + + return 0; + +put_shmem_bo: + amdxdna_arg_bos_put(job); + return ret; +} + +void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) +{ + trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); + amdxdna_pm_suspend_put(job->hwctx->client->xdna); + amdxdna_arg_bos_put(job); + amdxdna_gem_put_obj(job->cmd_bo); + dma_fence_put(job->fence); + mmdrop(job->mm); +} + +int amdxdna_cmd_submit(struct amdxdna_client *client, + struct amdxdna_drv_cmd *drv_cmd, + u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt, + u32 hwctx_hdl, u64 *seq) +{ + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_sched_job *job; + struct amdxdna_hwctx *hwctx; + int ret, idx; + + XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); + + if (!xdna->dev_info->ops->cmd_submit) + return -EOPNOTSUPP; + + job = kzalloc_flex(*job, bos, arg_bo_cnt); + if (!job) + return -ENOMEM; + + job->drv_cmd = drv_cmd; + + if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) { + job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_SHARE); + if (!job->cmd_bo) { + XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl); + ret = -EINVAL; + goto free_job; + } + } else if (!drv_cmd) { + /* + * Only internal driver commands (drv_cmd != NULL) may omit a + * command BO. A user command submission with the invalid handle + * would leave job->cmd_bo NULL and later fault when the scheduler + * dereferences it in amdxdna_cmd_set_state(). + */ + XDNA_DBG(xdna, "Command BO handle required for user submission"); + ret = -EINVAL; + goto free_job; + } + + ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt); + if (ret) { + XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret); + goto cmd_put; + } + + ret = amdxdna_pm_resume_get(xdna); + if (ret) { + XDNA_ERR(xdna, "Resume failed, ret %d", ret); + goto put_bos; + } + + idx = srcu_read_lock(&client->hwctx_srcu); + hwctx = xa_load(&client->hwctx_xa, hwctx_hdl); + if (!hwctx) { + XDNA_DBG(xdna, "PID %d failed to get hwctx %d", + client->pid, hwctx_hdl); + ret = -EINVAL; + goto unlock_srcu; + } + + job->hwctx = hwctx; + job->mm = current->mm; + mmgrab(job->mm); + + job->fence = amdxdna_fence_create(hwctx); + if (!job->fence) { + XDNA_ERR(xdna, "Failed to create fence"); + ret = -ENOMEM; + goto unlock_srcu; + } + kref_init(&job->refcnt); + + ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq); + if (ret) + goto put_fence; + + /* + * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated + * resource after synchronize_srcu(). The submitted jobs should be + * handled by the queue, for example DRM scheduler, in device layer. + * For here we can unlock SRCU. + */ + srcu_read_unlock(&client->hwctx_srcu, idx); + trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed"); + + return 0; + +put_fence: + dma_fence_put(job->fence); +unlock_srcu: + srcu_read_unlock(&client->hwctx_srcu, idx); + amdxdna_pm_suspend_put(xdna); +put_bos: + amdxdna_arg_bos_put(job); +cmd_put: + amdxdna_gem_put_obj(job->cmd_bo); +free_job: + if (job->mm) + mmdrop(job->mm); + kfree(job); + return ret; +} + +/* + * The submit command ioctl submits a command to firmware. One firmware command + * may contain multiple command BOs for processing as a whole. + * The command sequence number is returned which can be used for wait command ioctl. + */ +static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client, + struct amdxdna_drm_exec_cmd *args) +{ + struct amdxdna_dev *xdna = client->xdna; + u32 *arg_bo_hdls = NULL; + u32 cmd_bo_hdl; + int ret; + + if (args->arg_count > MAX_ARG_COUNT) { + XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count); + return -EINVAL; + } + + /* Only support single command for now. */ + if (args->cmd_count != 1) { + XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count); + return -EINVAL; + } + + cmd_bo_hdl = (u32)args->cmd_handles; + if (args->arg_count) { + arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL); + if (!arg_bo_hdls) + return -ENOMEM; + ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args), + args->arg_count * sizeof(u32)); + if (ret) { + ret = -EFAULT; + goto free_cmd_bo_hdls; + } + } + + ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls, + args->arg_count, args->hwctx, &args->seq); + if (ret) + XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret); + +free_cmd_bo_hdls: + kfree(arg_bo_hdls); + if (!ret) + XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq); + return ret; +} + +int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_drm_exec_cmd *args = data; + + if (args->ext || args->ext_flags) + return -EINVAL; + + trace_amdxdna_debug_point(current->comm, args->type, "job received"); + + switch (args->type) { + case AMDXDNA_CMD_SUBMIT_EXEC_BUF: + return amdxdna_drm_submit_execbuf(client, args); + } + + XDNA_ERR(client->xdna, "Invalid command type %d", args->type); + return -EINVAL; +} + +int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_wait_cmd *args = data; + struct amdxdna_hwctx *hwctx; + int ret, idx; + + XDNA_DBG(xdna, "PID %d ctx %d timeout set %d ms for cmd %llu", + client->pid, args->hwctx, args->timeout, args->seq); + + if (!xdna->dev_info->ops->cmd_wait) + return -EOPNOTSUPP; + + idx = srcu_read_lock(&client->hwctx_srcu); + hwctx = xa_load(&client->hwctx_xa, args->hwctx); + if (!hwctx) { + XDNA_DBG(xdna, "PID %d failed to get ctx %d", client->pid, args->hwctx); + ret = -EINVAL; + goto unlock_ctx_srcu; + } + + ret = xdna->dev_info->ops->cmd_wait(hwctx, args->seq, args->timeout); + + XDNA_DBG(xdna, "PID %d ctx %d cmd %lld wait finished, ret %d", + client->pid, args->hwctx, args->seq, ret); + + trace_amdxdna_debug_point(current->comm, args->seq, "job returned to user"); + +unlock_ctx_srcu: + srcu_read_unlock(&client->hwctx_srcu, idx); + return ret; +} diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h new file mode 100644 index 000000000..b6bef3af7 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_ctx.h @@ -0,0 +1,223 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#ifndef _AMDXDNA_CTX_H_ +#define _AMDXDNA_CTX_H_ + +#include <linux/bitfield.h> + +#include "amdxdna_gem.h" + +struct amdxdna_hwctx_priv; + +enum ert_cmd_opcode { + ERT_START_CU = 0, + ERT_START_DPU = 18, + ERT_CMD_CHAIN = 19, + ERT_START_NPU = 20, + ERT_START_NPU_PREEMPT = 21, + ERT_START_NPU_PREEMPT_ELF = 22, + ERT_INVALID_CMD = ~0U, +}; + +enum ert_cmd_state { + ERT_CMD_STATE_INVALID, + ERT_CMD_STATE_NEW, + ERT_CMD_STATE_QUEUED, + ERT_CMD_STATE_RUNNING, + ERT_CMD_STATE_COMPLETED, + ERT_CMD_STATE_ERROR, + ERT_CMD_STATE_ABORT, + ERT_CMD_STATE_SUBMITTED, + ERT_CMD_STATE_TIMEOUT, + ERT_CMD_STATE_NORESPONSE, +}; + +/* + * Interpretation of the beginning of data payload for ERT_START_NPU in + * amdxdna_cmd. The rest of the payload in amdxdna_cmd is regular kernel args. + */ +struct amdxdna_cmd_start_npu { + u64 buffer; /* instruction buffer address */ + u32 buffer_size; /* size of buffer in bytes */ + u32 prop_count; /* properties count */ + u32 prop_args[]; /* properties and regular kernel arguments */ +}; + +/* + * Interpretation of the beginning of data payload for ERT_CMD_CHAIN in + * amdxdna_cmd. The rest of the payload in amdxdna_cmd is cmd BO handles. + */ +struct amdxdna_cmd_chain { + u32 command_count; + u32 submit_index; + u32 error_index; + u32 reserved[3]; + u64 data[] __counted_by(command_count); +}; + +/* + * Interpretation of the beginning of data payload for ERT_START_NPU_PREEMPT in + * amdxdna_cmd. The rest of the payload in amdxdna_cmd is regular kernel args. + */ +struct amdxdna_cmd_preempt_data { + u64 inst_buf; /* instruction buffer address */ + u64 save_buf; /* save buffer address */ + u64 restore_buf; /* restore buffer address */ + u32 inst_size; /* size of instruction buffer in bytes */ + u32 save_size; /* size of save buffer in bytes */ + u32 restore_size; /* size of restore buffer in bytes */ + u32 inst_prop_cnt; /* properties count */ + u32 prop_args[]; /* properties and regular kernel arguments */ +}; + +#define AMDXDNA_CMD_CTX_HEALTH_V1 1 +#define AMDXDNA_CMD_CTX_HEALTH_AIE2 0 +struct amdxdna_ctx_health { + u32 version; + u32 npu_gen; +}; + +/* Exec buffer command header format */ +#define AMDXDNA_CMD_STATE GENMASK(3, 0) +#define AMDXDNA_CMD_EXTRA_CU_MASK GENMASK(11, 10) +#define AMDXDNA_CMD_COUNT GENMASK(22, 12) +#define AMDXDNA_CMD_OPCODE GENMASK(27, 23) +struct amdxdna_cmd { + u32 header; + u32 data[]; +}; + +#define INVALID_CU_IDX (~0U) + +struct amdxdna_hwctx { + struct amdxdna_client *client; + struct amdxdna_hwctx_priv *priv; + char *name; + + u32 id; + u32 max_opc; + u32 num_tiles; + u32 mem_size; + u32 fw_ctx_id; + u32 col_list_len; + u32 *col_list; + u32 start_col; + u32 num_col; + u32 umq_bo_hdl; + u32 doorbell_offset; + u32 num_unused_col; + u32 last_attached_heap; + + struct amdxdna_qos_info qos; + struct amdxdna_hwctx_param_config_cu *cus; + u32 syncobj_hdl; + + atomic64_t job_submit_cnt; + atomic64_t job_free_cnt ____cacheline_aligned_in_smp; +}; + +#define drm_job_to_xdna_job(j) \ + container_of(j, struct amdxdna_sched_job, base) + +enum amdxdna_job_opcode { + DEFAULT_IO, + SYNC_DEBUG_BO, + ATTACH_DEBUG_BO, + DETACH_DEBUG_BO, +}; + +struct amdxdna_drv_cmd { + enum amdxdna_job_opcode opcode; + u32 result; + struct kref refcnt; +}; + +struct app_health_report; +union amdxdna_job_priv { + struct app_health_report *aie2_health; +}; + +struct amdxdna_sched_job { + struct drm_sched_job base; + struct kref refcnt; + struct amdxdna_hwctx *hwctx; + struct mm_struct *mm; + /* The fence to notice DRM scheduler that job is done by hardware */ + struct dma_fence *fence; + /* user can wait on this fence */ + struct dma_fence *out_fence; + bool job_done; + bool job_timeout; + u64 seq; + struct amdxdna_drv_cmd *drv_cmd; + struct amdxdna_gem_obj *cmd_bo; + union amdxdna_job_priv priv; + size_t bo_cnt; + struct drm_gem_object *bos[] __counted_by(bo_cnt); +}; + +#define aie2_job_health priv.aie2_health + +static inline u32 +amdxdna_cmd_get_op(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); + + if (!cmd) + return ERT_INVALID_CMD; + + return FIELD_GET(AMDXDNA_CMD_OPCODE, cmd->header); +} + +static inline void +amdxdna_cmd_set_state(struct amdxdna_gem_obj *abo, enum ert_cmd_state s) +{ + struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); + + if (!cmd) + return; + + cmd->header &= ~AMDXDNA_CMD_STATE; + cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, s); +} + +static inline enum ert_cmd_state +amdxdna_cmd_get_state(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); + + if (!cmd) + return ERT_CMD_STATE_INVALID; + + return FIELD_GET(AMDXDNA_CMD_STATE, cmd->header); +} + +void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size); +u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo); +int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, + struct amdxdna_sched_job *job, u32 cmd_idx, + enum ert_cmd_state error_state, + void *err_data, size_t size); + +void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job); +void amdxdna_hwctx_remove_all(struct amdxdna_client *client); +int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg, + int (*walk)(struct amdxdna_hwctx *hwctx, void *arg)); +int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl); +int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx); + +int amdxdna_cmd_submit(struct amdxdna_client *client, + struct amdxdna_drv_cmd *drv_cmd, u32 cmd_bo_hdls, + u32 *arg_bo_hdls, u32 arg_bo_cnt, + u32 hwctx_hdl, u64 *seq); + +int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); + +#endif /* _AMDXDNA_CTX_H_ */ diff --git a/drivers/accel/amdxdna/amdxdna_debugfs.c b/drivers/accel/amdxdna/amdxdna_debugfs.c new file mode 100644 index 000000000..a6ec17c63 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_debugfs.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include "amdxdna_cbuf.h" +#include "amdxdna_debugfs.h" + +#include <drm/drm_file.h> +#include <linux/debugfs.h> +#include <linux/pm_runtime.h> +#include <linux/seq_file.h> +#include <linux/string.h> +#include <linux/uaccess.h> + +#define _DBGFS_FOPS(_open, _release, _write) \ +{ \ + .owner = THIS_MODULE, \ + .open = _open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = _release, \ + .write = _write, \ +} + +#define AMDXDNA_DBGFS_FOPS(_name, _show, _write) \ + static int amdxdna_dbgfs_##_name##_open(struct inode *inode, struct file *file) \ + { \ + return single_open(file, _show, inode->i_private); \ + } \ + static int amdxdna_dbgfs_##_name##_release(struct inode *inode, struct file *file) \ + { \ + return single_release(inode, file); \ + } \ + static const struct file_operations amdxdna_fops_##_name = \ + _DBGFS_FOPS(amdxdna_dbgfs_##_name##_open, amdxdna_dbgfs_##_name##_release, _write) + +#define AMDXDNA_DBGFS_FILE(_name, _mode) { #_name, &amdxdna_fops_##_name, _mode } + +#define file_to_xdna(file) (((struct seq_file *)(file)->private_data)->private) + +static ssize_t amdxdna_carveout_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct amdxdna_dev *xdna = file_to_xdna(file); + char kbuf[128]; + u64 size, addr; + char *sep; + int ret; + + if (count == 0 || count >= sizeof(kbuf)) + return -EINVAL; + + if (copy_from_user(kbuf, buf, count)) + return -EFAULT; + kbuf[count] = '\0'; + strim(kbuf); + XDNA_DBG(xdna, "Trying to set carveout to %s", kbuf); + + sep = strchr(kbuf, '@'); + if (!sep) + return -EINVAL; + *sep = '\0'; + sep++; + + ret = kstrtou64(kbuf, 0, &size); + if (ret) + return ret; + + ret = kstrtou64(sep, 0, &addr); + if (ret) + return ret; + + /* Sanity check the addr and size. */ + if (!size) + return -EINVAL; + if (!IS_ALIGNED(addr, PAGE_SIZE) || !IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; + + guard(mutex)(&xdna->dev_lock); + + ret = amdxdna_carveout_init(xdna, addr, size); + if (ret) + return ret; + + return count; +} + +static int amdxdna_carveout_show(struct seq_file *m, void *unused) +{ + struct amdxdna_dev *xdna = m->private; + u64 addr, size; + + guard(mutex)(&xdna->dev_lock); + amdxdna_get_carveout_conf(xdna, &addr, &size); + seq_printf(m, "0x%llx@0x%llx\n", size, addr); + return 0; +} + +/* + * Input/output format: <carveout_size>@<carveout_address> + */ +AMDXDNA_DBGFS_FOPS(carveout, amdxdna_carveout_show, amdxdna_carveout_write); + +static const struct { + const char *name; + const struct file_operations *fops; + umode_t mode; +} amdxdna_dbgfs_files[] = { + AMDXDNA_DBGFS_FILE(carveout, 0600), +}; + +void amdxdna_debugfs_init(struct amdxdna_dev *xdna) +{ + struct drm_minor *minor = xdna->ddev.accel; + int i; + + /* + * It should be okay that debugfs fails to init. + * We rely on DRM framework to finish debugfs. + */ + for (i = 0; i < ARRAY_SIZE(amdxdna_dbgfs_files); i++) { + debugfs_create_file(amdxdna_dbgfs_files[i].name, + amdxdna_dbgfs_files[i].mode, + minor->debugfs_root, + xdna, + amdxdna_dbgfs_files[i].fops); + } +} diff --git a/drivers/accel/amdxdna/amdxdna_debugfs.h b/drivers/accel/amdxdna/amdxdna_debugfs.h new file mode 100644 index 000000000..2abb45de3 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_debugfs.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ +#ifndef _AMDXDNA_DEBUGFS_H_ +#define _AMDXDNA_DEBUGFS_H_ + +#include "amdxdna_pci_drv.h" + +#if defined(CONFIG_DEBUG_FS) +void amdxdna_debugfs_init(struct amdxdna_dev *xdna); +#else +static inline void amdxdna_debugfs_init(struct amdxdna_dev *xdna) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif diff --git a/drivers/accel/amdxdna/amdxdna_error.h b/drivers/accel/amdxdna/amdxdna_error.h new file mode 100644 index 000000000..c51de86ec --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_error.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025, Advanced Micro Devices, Inc. + */ + +#ifndef _AMDXDNA_ERROR_H_ +#define _AMDXDNA_ERROR_H_ + +#include <linux/bitfield.h> +#include <linux/bits.h> + +#define AMDXDNA_ERR_DRV_AIE 4 +#define AMDXDNA_ERR_SEV_CRITICAL 3 +#define AMDXDNA_ERR_CLASS_AIE 2 + +#define AMDXDNA_ERR_NUM_MASK GENMASK_U64(15, 0) +#define AMDXDNA_ERR_DRV_MASK GENMASK_U64(23, 16) +#define AMDXDNA_ERR_SEV_MASK GENMASK_U64(31, 24) +#define AMDXDNA_ERR_MOD_MASK GENMASK_U64(39, 32) +#define AMDXDNA_ERR_CLASS_MASK GENMASK_U64(47, 40) + +enum amdxdna_error_num { + AMDXDNA_ERROR_NUM_AIE_SATURATION = 3, + AMDXDNA_ERROR_NUM_AIE_FP, + AMDXDNA_ERROR_NUM_AIE_STREAM, + AMDXDNA_ERROR_NUM_AIE_ACCESS, + AMDXDNA_ERROR_NUM_AIE_BUS, + AMDXDNA_ERROR_NUM_AIE_INSTRUCTION, + AMDXDNA_ERROR_NUM_AIE_ECC, + AMDXDNA_ERROR_NUM_AIE_LOCK, + AMDXDNA_ERROR_NUM_AIE_DMA, + AMDXDNA_ERROR_NUM_AIE_MEM_PARITY, + AMDXDNA_ERROR_NUM_UNKNOWN = 15, +}; + +enum amdxdna_error_module { + AMDXDNA_ERROR_MODULE_AIE_CORE = 3, + AMDXDNA_ERROR_MODULE_AIE_MEMORY, + AMDXDNA_ERROR_MODULE_AIE_SHIM, + AMDXDNA_ERROR_MODULE_AIE_NOC, + AMDXDNA_ERROR_MODULE_AIE_PL, + AMDXDNA_ERROR_MODULE_UNKNOWN = 8, +}; + +#define AMDXDNA_ERROR_ENCODE(err_num, err_mod) \ + (FIELD_PREP(AMDXDNA_ERR_NUM_MASK, err_num) | \ + FIELD_PREP_CONST(AMDXDNA_ERR_DRV_MASK, AMDXDNA_ERR_DRV_AIE) | \ + FIELD_PREP_CONST(AMDXDNA_ERR_SEV_MASK, AMDXDNA_ERR_SEV_CRITICAL) | \ + FIELD_PREP(AMDXDNA_ERR_MOD_MASK, err_mod) | \ + FIELD_PREP_CONST(AMDXDNA_ERR_CLASS_MASK, AMDXDNA_ERR_CLASS_AIE)) + +#define AMDXDNA_EXTRA_ERR_COL_MASK GENMASK_U64(7, 0) +#define AMDXDNA_EXTRA_ERR_ROW_MASK GENMASK_U64(15, 8) + +#define AMDXDNA_EXTRA_ERR_ENCODE(row, col) \ + (FIELD_PREP(AMDXDNA_EXTRA_ERR_COL_MASK, col) | \ + FIELD_PREP(AMDXDNA_EXTRA_ERR_ROW_MASK, row)) + +#endif /* _AMDXDNA_ERROR_H_ */ diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c new file mode 100644 index 000000000..b01f5d95f --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -0,0 +1,1381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_cache.h> +#include <drm/drm_device.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/dma-buf.h> +#include <linux/dma-direct.h> +#include <linux/iosys-map.h> +#include <linux/pagemap.h> +#include <linux/vmalloc.h> + +#include "amdxdna_cbuf.h" +#include "amdxdna_ctx.h" +#include "amdxdna_gem.h" +#include "amdxdna_pci_drv.h" +#include "amdxdna_ubuf.h" + +MODULE_IMPORT_NS("DMA_BUF"); + +/* + * The dev BO could be across multiple heap BO chunks. The heap chunks should + * be mapped to userspace and the userspace virtual address should be + * contiguous. + */ +static int +amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo) +{ + struct amdxdna_client *client = dev_bo->client; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_gem_obj *heap; + u64 heap_addr, exp_heap_uva; + u32 heap_id; + + if (xa_empty(&client->dev_heap_xa)) { + XDNA_DBG(xdna, "Empty heap xa"); + return -EAGAIN; + } + + for (heap_id = 0; heap_id < client->dev_heap_nid; heap_id++) { + heap = xa_load(&client->dev_heap_xa, heap_id); + if (!heap) { + XDNA_ERR(xdna, "Failed to load heap %d", heap_id); + return -EINVAL; + } + heap_addr = amdxdna_gem_dev_addr(heap); + if (heap_addr > dev_bo->mm_node.start) + break; + } + + heap_id--; + heap = xa_load(&client->dev_heap_xa, heap_id); + exp_heap_uva = amdxdna_gem_uva(heap); + heap_addr = amdxdna_gem_dev_addr(heap); + dev_bo->heap_start_id = heap_id; + dev_bo->mem.uva = dev_bo->mm_node.start - heap_addr + exp_heap_uva; + + for (; heap_id < client->dev_heap_nid; heap_id++) { + heap = xa_load(&client->dev_heap_xa, heap_id); + if (!heap) { + XDNA_ERR(xdna, "Failed to load heap %d", heap_id); + return -EINVAL; + } + heap_addr = amdxdna_gem_uva(heap); + if (heap_addr == AMDXDNA_INVALID_ADDR) { + XDNA_ERR(xdna, "Heap %d is not mapped", heap_id); + return -EAGAIN; + } + + if (heap_addr != exp_heap_uva) { + XDNA_ERR(xdna, "Heap %d uva is not contiguous", heap_id); + return -EINVAL; + } + + if (heap->dev_addr + heap->mem.size >= + dev_bo->mm_node.start + dev_bo->mem.size) + break; + + exp_heap_uva += heap->mem.size; + } + + if (heap_id == client->dev_heap_nid) { + XDNA_DBG(xdna, "Can not find heap end"); + return -EAGAIN; + } + + dev_bo->heap_end_id = heap_id; + + return 0; +} + +static int +amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_client *client = abo->client; + struct amdxdna_dev *xdna = client->xdna; + struct amdxdna_mem *mem = &abo->mem; + struct amdxdna_gem_obj *heap; + unsigned long heap_id; + u32 align; + int ret; + + mutex_lock(&client->mm_lock); + + if (!mem->size || mem->size > xdna->dev_info->dev_heap_max_size) { + XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, max heap 0x%lx", + mem->size, xdna->dev_info->dev_heap_max_size); + ret = -EINVAL; + goto unlock_out; + } + + align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift); + ret = drm_mm_insert_node_generic(&client->dev_heap_mm, + &abo->mm_node, + mem->size, align, + 0, DRM_MM_INSERT_BEST); + if (ret) { + XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); + goto unlock_out; + } + + ret = amdxdna_init_dev_bo(abo); + if (ret) { + drm_mm_remove_node(&abo->mm_node); + goto unlock_out; + } + + client->heap_usage += mem->size; + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, + abo->heap_start_id, abo->heap_end_id) + drm_gem_object_get(to_gobj(heap)); + +unlock_out: + mutex_unlock(&client->mm_lock); + + return ret; +} + +static void +amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_client *client = abo->client; + struct amdxdna_gem_obj *heap; + unsigned long heap_id; + + mutex_lock(&client->mm_lock); + + drm_mm_remove_node(&abo->mm_node); + client->heap_usage -= abo->mem.size; + + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, + abo->heap_start_id, abo->heap_end_id) + drm_gem_object_put(to_gobj(heap)); + + mutex_unlock(&client->mm_lock); +} + +static struct amdxdna_gem_obj * +amdxdna_gem_create_obj(struct drm_device *dev, size_t size) +{ + struct amdxdna_gem_obj *abo; + + abo = kzalloc_obj(*abo); + if (!abo) + return ERR_PTR(-ENOMEM); + + abo->pinned = false; + abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; + mutex_init(&abo->lock); + + abo->mem.dma_addr = AMDXDNA_INVALID_ADDR; + abo->mem.uva = AMDXDNA_INVALID_ADDR; + abo->mem.size = size; + abo->open_ref = 0; + abo->internal = false; + INIT_LIST_HEAD(&abo->mem.umap_list); + + return abo; +} + +static void +amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo) +{ + mutex_destroy(&abo->lock); + kfree(abo); +} + +/* + * Obtains a kernel virtual address on the BO (usually of small size). + * The mapping is established on the first call and stays valid until + * amdxdna_gem_vunmap() is called. + */ +void *amdxdna_gem_vmap(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); + int ret; + + if (abo->mem.kva) + return abo->mem.kva; + + /* The first call to get the kva, taking slow path. */ + guard(mutex)(&abo->lock); + + if (!abo->mem.kva) { + ret = drm_gem_vmap(to_gobj(abo), &map); + if (ret) + XDNA_ERR(xdna, "Vmap bo failed, ret %d", ret); + else + abo->mem.kva = map.vaddr; + } + return abo->mem.kva; +} + +/* + * Free mapping established through amdxdna_gem_vmap() + */ +static void amdxdna_gem_vunmap(struct amdxdna_gem_obj *abo) +{ + guard(mutex)(&abo->lock); + + if (abo->mem.kva) { + struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva); + + drm_gem_vunmap(to_gobj(abo), &map); + abo->mem.kva = NULL; + } +} + +/* + * Obtain the address for device to access the BO. + */ +u64 amdxdna_gem_dev_addr(struct amdxdna_gem_obj *abo) +{ + if (abo->type == AMDXDNA_BO_DEV_HEAP) + return abo->dev_addr; + if (abo->type == AMDXDNA_BO_DEV) + return abo->mm_node.start; + return amdxdna_obj_dma_addr(abo); +} + +static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq) +{ + struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier); + struct amdxdna_gem_obj *abo = mapp->abo; + struct amdxdna_dev *xdna; + + xdna = to_xdna_dev(to_gobj(abo)->dev); + XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d", + mapp->range.start, mapp->range.end, abo->type); + + if (!mmu_notifier_range_blockable(range)) + return false; + + down_write(&xdna->notifier_lock); + abo->mem.map_invalid = true; + mapp->invalid = true; + mmu_interval_set_seq(&mapp->notifier, cur_seq); + up_write(&xdna->notifier_lock); + + if (xdna->dev_info->ops->hmm_invalidate) + xdna->dev_info->ops->hmm_invalidate(abo, cur_seq); + + if (range->event == MMU_NOTIFY_UNMAP) { + down_write(&xdna->notifier_lock); + if (!mapp->unmapped) { + queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); + mapp->unmapped = true; + } + up_write(&xdna->notifier_lock); + } + + return true; +} + +static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = { + .invalidate = amdxdna_hmm_invalidate, +}; + +static inline bool compare_range(struct amdxdna_umap *mapp, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + return (!mapp->unmapped && mapp->notifier.mm == mm && + mapp->range.start == start && mapp->range.end == end); +} + +static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, + struct vm_area_struct *vma) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct amdxdna_umap *mapp; + + down_write(&xdna->notifier_lock); + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (!vma || compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) { + if (!mapp->unmapped) { + queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); + mapp->unmapped = true; + } + if (vma) + break; + } + } + up_write(&xdna->notifier_lock); +} + +static void amdxdna_umap_release(struct kref *ref) +{ + struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); + struct amdxdna_gem_obj *abo = mapp->abo; + struct amdxdna_dev *xdna; + + mmu_interval_notifier_remove(&mapp->notifier); + + xdna = to_xdna_dev(to_gobj(mapp->abo)->dev); + down_write(&xdna->notifier_lock); + list_del(&mapp->node); + if (list_empty(&abo->mem.umap_list)) + abo->mem.uva = AMDXDNA_INVALID_ADDR; + up_write(&xdna->notifier_lock); + + kvfree(mapp->range.hmm_pfns); + kfree(mapp); +} + +void amdxdna_umap_put(struct amdxdna_umap *mapp) +{ + kref_put(&mapp->refcnt, amdxdna_umap_release); +} + +static void amdxdna_hmm_unreg_work(struct work_struct *work) +{ + struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap, + hmm_unreg_work); + + amdxdna_umap_put(mapp); +} + +static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, + struct vm_area_struct *vma) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long addr = vma->vm_start; + struct amdxdna_umap *mapp; + unsigned long nr_pages; + int ret; + + /* + * When PASID is off, amdxdna_gem_obj_open() called amdxdna_dma_map_bo() + * and mem.dma_addr is valid; use the DMA address directly and skip HMM. + * Avoid dereferencing abo->client which may be NULL (cleared in close()) + * while internal kernel references are still held. + */ + if (abo->mem.dma_addr != AMDXDNA_INVALID_ADDR) { + /* Need to set uva for heap uva validation */ + abo->mem.uva = addr; + return 0; + } + + down_read(&xdna->notifier_lock); + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (compare_range(mapp, current->mm, addr, addr + len)) { + up_read(&xdna->notifier_lock); + return 0; + } + } + up_read(&xdna->notifier_lock); + + mapp = kzalloc_obj(*mapp); + if (!mapp) + return -ENOMEM; + + nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT; + mapp->range.hmm_pfns = kvzalloc_objs(*mapp->range.hmm_pfns, nr_pages); + if (!mapp->range.hmm_pfns) { + ret = -ENOMEM; + goto free_map; + } + + ret = mmu_interval_notifier_insert_locked(&mapp->notifier, + current->mm, + addr, + len, + &amdxdna_hmm_ops); + if (ret) { + XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret); + goto free_pfns; + } + + mapp->range.notifier = &mapp->notifier; + mapp->range.start = vma->vm_start; + mapp->range.end = vma->vm_end; + mapp->range.default_flags = HMM_PFN_REQ_FAULT; + mapp->abo = abo; + kref_init(&mapp->refcnt); + + INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work); + + down_write(&xdna->notifier_lock); + if (list_empty(&abo->mem.umap_list)) + abo->mem.uva = addr; + list_add_tail(&mapp->node, &abo->mem.umap_list); + up_write(&xdna->notifier_lock); + + return 0; + +free_pfns: + kvfree(mapp->range.hmm_pfns); +free_map: + kfree(mapp); + return ret; +} + +static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj) +{ + struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + + XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, amdxdna_gem_dev_addr(abo)); + if (abo->pinned) + amdxdna_gem_unpin(abo); + + amdxdna_gem_vunmap(abo); + amdxdna_gem_heap_free(abo); + drm_gem_object_release(gobj); + amdxdna_gem_destroy_obj(abo); +} + +static void amdxdna_mark_mapp_invalid(struct amdxdna_gem_obj *abo, + struct vm_area_struct *vma) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + struct amdxdna_umap *mapp; + + down_write(&xdna->notifier_lock); + abo->mem.map_invalid = true; + list_for_each_entry(mapp, &abo->mem.umap_list, node) { + if (compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) { + mapp->invalid = true; + break; + } + } + up_write(&xdna->notifier_lock); +} + +static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo, + struct vm_area_struct *vma) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + unsigned long num_pages = vma_pages(vma); + unsigned long offset = 0; + int ret; + + if (!is_import_bo(abo)) { + ret = drm_gem_shmem_mmap(&abo->base, vma); + if (ret) { + XDNA_ERR(xdna, "Failed shmem mmap %d", ret); + return ret; + } + } else { + vma->vm_private_data = NULL; + vma->vm_ops = NULL; + ret = dma_buf_mmap(abo->dma_buf, vma, 0); + if (ret) { + XDNA_ERR(xdna, "Failed to mmap dma buf %d", ret); + return ret; + } + + /* Drop the reference drm_gem_mmap_obj() acquired.*/ + drm_gem_object_put(to_gobj(abo)); + } + + do { + vm_fault_t fault_ret; + + fault_ret = handle_mm_fault(vma, vma->vm_start + offset, + FAULT_FLAG_WRITE, NULL); + if (fault_ret & VM_FAULT_ERROR) { + XDNA_ERR(xdna, "Fault in page failed"); + amdxdna_mark_mapp_invalid(abo, vma); + break; + } + + offset += PAGE_SIZE; + } while (--num_pages); + + return 0; +} + +static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj, + struct vm_area_struct *vma) +{ + struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + int ret; + + ret = amdxdna_hmm_register(abo, vma); + if (ret) + return ret; + + ret = amdxdna_insert_pages(abo, vma); + if (ret) { + XDNA_ERR(xdna, "Failed insert pages, ret %d", ret); + goto hmm_unreg; + } + + XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx", + drm_vma_node_offset_addr(&gobj->vma_node), abo->type, + vma->vm_start, gobj->size); + return 0; + +hmm_unreg: + amdxdna_hmm_unregister(abo, vma); + return ret; +} + +static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) +{ + struct drm_gem_object *gobj = dma_buf->priv; + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + unsigned long num_pages = vma_pages(vma); + int ret; + + vma->vm_ops = &drm_gem_shmem_vm_ops; + vma->vm_private_data = gobj; + + drm_gem_object_get(gobj); + ret = drm_gem_shmem_mmap(&abo->base, vma); + if (ret) + goto put_obj; + + /* The buffer is based on memory pages. Fix the flag. */ + vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); + ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, + &num_pages); + if (ret) + goto close_vma; + + return 0; + +close_vma: + vma->vm_ops->close(vma); + return ret; +put_obj: + drm_gem_object_put(gobj); + return ret; +} + +static const struct dma_buf_ops amdxdna_dmabuf_ops = { + .attach = drm_gem_map_attach, + .detach = drm_gem_map_detach, + .map_dma_buf = drm_gem_map_dma_buf, + .unmap_dma_buf = drm_gem_unmap_dma_buf, + .release = drm_gem_dmabuf_release, + .mmap = amdxdna_gem_dmabuf_mmap, + .vmap = drm_gem_dmabuf_vmap, + .vunmap = drm_gem_dmabuf_vunmap, +}; + +static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags) +{ + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + if (abo->private_buffer) + return ERR_PTR(-EOPNOTSUPP); + + if (abo->dma_buf) { + get_dma_buf(abo->dma_buf); + return abo->dma_buf; + } + + exp_info.ops = &amdxdna_dmabuf_ops; + exp_info.size = gobj->size; + exp_info.flags = flags; + exp_info.priv = gobj; + exp_info.resv = gobj->resv; + + return drm_gem_dmabuf_export(gobj->dev, &exp_info); +} + +static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo) +{ + dma_buf_unmap_attachment_unlocked(abo->attach, abo->base.sgt, DMA_BIDIRECTIONAL); + dma_buf_detach(abo->dma_buf, abo->attach); + dma_buf_put(abo->dma_buf); + drm_gem_object_release(to_gobj(abo)); + kfree(abo); +} + +static inline bool +amdxdna_gem_skip_bo_usage(struct amdxdna_gem_obj *abo) +{ + /* Already counted as part of HEAP BO */ + if (abo->type == AMDXDNA_BO_DEV) + return true; + + return false; +} + +static void +amdxdna_gem_add_bo_usage(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_client *client = abo->client; + + if (amdxdna_gem_skip_bo_usage(abo)) + return; + + guard(mutex)(&client->mm_lock); + + client->total_bo_usage += abo->mem.size; + if (abo->internal) + client->total_int_bo_usage += abo->mem.size; +} + +static void +amdxdna_gem_del_bo_usage(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_client *client = abo->client; + + if (amdxdna_gem_skip_bo_usage(abo)) + return; + + guard(mutex)(&client->mm_lock); + + client->total_bo_usage -= abo->mem.size; + if (abo->internal) + client->total_int_bo_usage -= abo->mem.size; +} + +static void amdxdna_gem_obj_free(struct drm_gem_object *gobj) +{ + struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + + amdxdna_hmm_unregister(abo, NULL); + flush_workqueue(xdna->notifier_wq); + + if (abo->pinned) + amdxdna_gem_unpin(abo); + + amdxdna_dma_unmap_bo(xdna, abo); + amdxdna_gem_vunmap(abo); + mutex_destroy(&abo->lock); + + if (is_import_bo(abo)) + amdxdna_imported_obj_free(abo); + else + drm_gem_shmem_free(&abo->base); +} + +static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *filp) +{ + struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + int ret; + + guard(mutex)(&abo->lock); + abo->open_ref++; + if (abo->open_ref > 1) + return 0; + + /* Attached to the client when first opened by it. */ + abo->client = filp->driver_priv; + + /* No need to set up dma addr mapping in PASID mode. */ + if (!amdxdna_pasid_on(abo->client)) { + ret = amdxdna_dma_map_bo(xdna, abo); + if (ret) { + abo->open_ref--; + abo->client = NULL; + return ret; + } + } + + amdxdna_gem_add_bo_usage(abo); + return 0; +} + +static void amdxdna_gem_obj_close(struct drm_gem_object *gobj, struct drm_file *filp) +{ + struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); + + guard(mutex)(&abo->lock); + abo->open_ref--; + + if (abo->open_ref == 0) { + amdxdna_gem_del_bo_usage(abo); + /* Detach from the client when last closed by it. */ + abo->client = NULL; + } +} + +static int amdxdna_gem_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map) +{ + struct amdxdna_gem_obj *abo = to_xdna_obj(obj); + int ret; + + iosys_map_clear(map); + + dma_resv_assert_held(obj->resv); + + if (is_import_bo(abo)) + ret = dma_buf_vmap(abo->dma_buf, map); + else + ret = drm_gem_shmem_object_vmap(obj, map); + if (ret) + return ret; + if (!map->vaddr) + return -ENOMEM; + + return 0; +} + +static void amdxdna_gem_obj_vunmap(struct drm_gem_object *obj, struct iosys_map *map) +{ + struct amdxdna_gem_obj *abo = to_xdna_obj(obj); + + dma_resv_assert_held(obj->resv); + + if (is_import_bo(abo)) + dma_buf_vunmap(abo->dma_buf, map); + else + drm_gem_shmem_object_vunmap(obj, map); +} + +static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map) +{ + struct amdxdna_gem_obj *abo = to_xdna_obj(obj); + struct amdxdna_gem_obj *heap; + void *base; + u64 offset; + + /* vmap dev bo which is across more than 1 heap is not allowed */ + if (abo->heap_start_id != abo->heap_end_id) + return -ENOMEM; + + heap = xa_load(&abo->client->dev_heap_xa, abo->heap_start_id); + if (!heap) + return -ENOMEM; + + base = amdxdna_gem_vmap(heap); + if (!base) + return -ENOMEM; + + offset = amdxdna_gem_dev_addr(abo) - amdxdna_gem_dev_addr(heap); + iosys_map_set_vaddr(map, base + offset); + return 0; +} + +static struct dma_buf *amdxdna_gem_dev_obj_export(struct drm_gem_object *gobj, int flags) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = { + .free = amdxdna_gem_dev_obj_free, + .vmap = amdxdna_gem_dev_obj_vmap, + .export = amdxdna_gem_dev_obj_export, +}; + +static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = { + .free = amdxdna_gem_obj_free, + .open = amdxdna_gem_obj_open, + .close = amdxdna_gem_obj_close, + .print_info = drm_gem_shmem_object_print_info, + .pin = drm_gem_shmem_object_pin, + .unpin = drm_gem_shmem_object_unpin, + .get_sg_table = drm_gem_shmem_object_get_sg_table, + .vmap = amdxdna_gem_obj_vmap, + .vunmap = amdxdna_gem_obj_vunmap, + .mmap = amdxdna_gem_obj_mmap, + .vm_ops = &drm_gem_shmem_vm_ops, + .export = amdxdna_gem_prime_export, +}; + +/* For drm_driver->gem_create_object callback */ +struct drm_gem_object * +amdxdna_gem_create_shmem_object_cb(struct drm_device *dev, size_t size) +{ + struct amdxdna_gem_obj *abo; + + abo = amdxdna_gem_create_obj(dev, size); + if (IS_ERR(abo)) + return ERR_CAST(abo); + + to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs; + + return to_gobj(abo); +} + +static struct amdxdna_gem_obj * +amdxdna_gem_create_shmem_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args) +{ + size_t size = args->size; + struct drm_gem_shmem_object *shmem = drm_gem_shmem_create(dev, size); + + if (IS_ERR(shmem)) + return ERR_CAST(shmem); + + shmem->map_wc = false; + return to_xdna_obj(&shmem->base); +} + +static struct amdxdna_gem_obj * +amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_va_tbl va_tbl; + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + struct dma_buf *dma_buf; + + if (copy_from_user(&va_tbl, u64_to_user_ptr(args->vaddr), sizeof(va_tbl))) { + XDNA_DBG(xdna, "Access va table failed"); + return ERR_PTR(-EINVAL); + } + + if (va_tbl.num_entries) { + dma_buf = amdxdna_get_ubuf(dev, va_tbl.num_entries, + u64_to_user_ptr(args->vaddr + sizeof(va_tbl))); + } else { + dma_buf = dma_buf_get(va_tbl.dmabuf_fd); + } + + if (IS_ERR(dma_buf)) + return ERR_CAST(dma_buf); + + gobj = amdxdna_gem_prime_import(dev, dma_buf); + if (IS_ERR(gobj)) { + dma_buf_put(dma_buf); + return ERR_CAST(gobj); + } + + dma_buf_put(dma_buf); + + abo = to_xdna_obj(gobj); + abo->private_buffer = true; + + return abo; +} + +static struct amdxdna_gem_obj * +amdxdna_gem_create_cbuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + size_t size = PAGE_ALIGN(args->size); + struct drm_gem_object *gobj; + struct amdxdna_gem_obj *ret; + struct dma_buf *dma_buf; + u64 align; + + if (!size) { + XDNA_ERR(xdna, "Invalid BO size 0x%llx", args->size); + return ERR_PTR(-EINVAL); + } + + align = (args->type == AMDXDNA_BO_DEV_HEAP) ? xdna->dev_info->dev_mem_size : 0; + dma_buf = amdxdna_get_cbuf(dev, size, align); + if (IS_ERR(dma_buf)) + return ERR_CAST(dma_buf); + + gobj = amdxdna_gem_prime_import(dev, dma_buf); + if (IS_ERR(gobj)) + ret = ERR_CAST(gobj); + else + ret = to_xdna_obj(gobj); + + dma_buf_put(dma_buf); + return ret; +} + +struct drm_gem_object * +amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) +{ + struct dma_buf_attachment *attach; + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + struct sg_table *sgt; + int ret; + + get_dma_buf(dma_buf); + + attach = dma_buf_attach(dma_buf, dev->dev); + if (IS_ERR(attach)) { + ret = PTR_ERR(attach); + goto put_buf; + } + + sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + goto fail_detach; + } + + gobj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); + if (IS_ERR(gobj)) { + ret = PTR_ERR(gobj); + goto fail_unmap; + } + + abo = to_xdna_obj(gobj); + abo->attach = attach; + abo->dma_buf = dma_buf; + abo->type = AMDXDNA_BO_SHARE; + gobj->resv = dma_buf->resv; + + return gobj; + +fail_unmap: + dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); +fail_detach: + dma_buf_detach(dma_buf, attach); +put_buf: + dma_buf_put(dma_buf); + + return ERR_PTR(ret); +} + +static struct amdxdna_gem_obj * +amdxdna_drm_create_share_bo(struct drm_device *dev, + struct amdxdna_drm_create_bo *args, struct drm_file *filp) +{ + struct amdxdna_gem_obj *abo; + + if (args->vaddr) + abo = amdxdna_gem_create_ubuf_object(dev, args); + else if (amdxdna_use_carveout(to_xdna_dev(dev))) + abo = amdxdna_gem_create_cbuf_object(dev, args); + else + abo = amdxdna_gem_create_shmem_object(dev, args); + if (IS_ERR(abo)) + return ERR_CAST(abo); + + if (args->type == AMDXDNA_BO_DEV_HEAP) { + abo->type = AMDXDNA_BO_DEV_HEAP; + abo->internal = true; + } else { + abo->type = AMDXDNA_BO_SHARE; + abo->internal = args->type == AMDXDNA_BO_CMD; + } + + return abo; +} + +static struct amdxdna_gem_obj * +amdxdna_drm_create_dev_heap_bo(struct drm_device *dev, + struct amdxdna_drm_create_bo *args, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_gem_obj *abo; + int ret; + + WARN_ON(!is_power_of_2(xdna->dev_info->dev_mem_size)); + XDNA_DBG(xdna, "Requested dev heap size 0x%llx", args->size); + if (!args->size || !IS_ALIGNED(args->size, xdna->dev_info->dev_mem_size)) { + XDNA_ERR(xdna, "The dev heap size 0x%llx is not multiple of 0x%lx", + args->size, xdna->dev_info->dev_mem_size); + return ERR_PTR(-EINVAL); + } + + /* HEAP BO is a special case of SHARE BO. */ + abo = amdxdna_drm_create_share_bo(dev, args, filp); + if (IS_ERR(abo)) + return ERR_CAST(abo); + + /* Set up heap for this client. */ + mutex_lock(&client->mm_lock); + + if (!xdna->dev_info->ops->hwctx_heap_expand && + client->dev_heap_nid > 0) { + XDNA_ERR(xdna, "Heap expansion is not supported"); + ret = -EOPNOTSUPP; + goto mm_unlock; + } + + if (client->total_heap_size + abo->mem.size > + xdna->dev_info->dev_heap_max_size) { + XDNA_ERR(xdna, "Heap size 0x%lx + 0x%lx exceeds max 0x%lx", + client->total_heap_size, abo->mem.size, + xdna->dev_info->dev_heap_max_size); + ret = -ENOSPC; + goto mm_unlock; + } + + ret = xa_insert(&client->dev_heap_xa, client->dev_heap_nid, abo, GFP_KERNEL); + if (ret) { + XDNA_ERR(xdna, "Add heap failed %d", ret); + goto mm_unlock; + } + + abo->dev_addr = xdna->dev_info->dev_mem_base + client->total_heap_size; + client->total_heap_size += abo->mem.size; + client->dev_heap_nid++; + drm_gem_object_get(to_gobj(abo)); + + mutex_unlock(&client->mm_lock); + + return abo; + +mm_unlock: + mutex_unlock(&client->mm_lock); + drm_gem_object_put(to_gobj(abo)); + return ERR_PTR(ret); +} + +struct amdxdna_gem_obj * +amdxdna_drm_create_dev_bo(struct drm_device *dev, + struct amdxdna_drm_create_bo *args, struct drm_file *filp) +{ + size_t aligned_sz = PAGE_ALIGN(args->size); + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + int ret; + + if (!aligned_sz) { + XDNA_ERR(xdna, "Invalid BO size 0x%llx", args->size); + return ERR_PTR(-EINVAL); + } + + abo = amdxdna_gem_create_obj(dev, aligned_sz); + if (IS_ERR(abo)) + return abo; + gobj = to_gobj(abo); + gobj->funcs = &amdxdna_gem_dev_obj_funcs; + abo->type = AMDXDNA_BO_DEV; + abo->internal = true; + /* + * DEV BOs cannot be alive when client is gone, it's OK to + * always establish the connection. + */ + abo->client = client; + + ret = amdxdna_gem_heap_alloc(abo); + if (ret) { + amdxdna_gem_destroy_obj(abo); + return ERR_PTR(ret); + } + + drm_gem_private_object_init(dev, gobj, aligned_sz); + + return abo; +} + +int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_create_bo *args = data; + struct amdxdna_gem_obj *abo; + int ret; + + if (args->flags) + return -EINVAL; + + XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx", + args->type, args->vaddr, args->size, args->flags); + switch (args->type) { + case AMDXDNA_BO_CMD: + case AMDXDNA_BO_SHARE: + abo = amdxdna_drm_create_share_bo(dev, args, filp); + break; + case AMDXDNA_BO_DEV_HEAP: + abo = amdxdna_drm_create_dev_heap_bo(dev, args, filp); + break; + case AMDXDNA_BO_DEV: + abo = amdxdna_drm_create_dev_bo(dev, args, filp); + if (!IS_ERR(abo)) { + mutex_lock(&xdna->dev_lock); + ret = amdxdna_update_heap(client, NULL); + mutex_unlock(&xdna->dev_lock); + if (ret) + goto put_obj; + } + break; + default: + return -EINVAL; + } + if (IS_ERR(abo)) + return PTR_ERR(abo); + + /* Ready to publish object to userspace and count for BO usage. */ + ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle); + if (ret) { + XDNA_ERR(xdna, "Create handle failed"); + goto put_obj; + } + + XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx", + args->handle, args->type, amdxdna_gem_uva(abo), + amdxdna_gem_dev_addr(abo), abo->mem.size); +put_obj: + /* Dereference object reference. Handle holds it now. */ + drm_gem_object_put(to_gobj(abo)); + return ret; +} + +static int amdxdna_bo_pin(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + int ret; + + if (is_import_bo(abo)) + return 0; + + ret = drm_gem_shmem_pin(&abo->base); + + XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret); + return ret; +} + +static void amdxdna_bo_unpin(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); + + if (is_import_bo(abo)) + return; + + drm_gem_shmem_unpin(&abo->base); + + XDNA_DBG(xdna, "BO type %d", abo->type); +} + +int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo) +{ + struct amdxdna_client *client = abo->client; + struct amdxdna_gem_obj *heap; + unsigned long heap_id, last = ULONG_MAX; + int ret = 0; + + if (abo->type != AMDXDNA_BO_DEV) + return amdxdna_bo_pin(abo); + + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, + abo->heap_start_id, abo->heap_end_id) { + ret = amdxdna_bo_pin(heap); + if (ret) + break; + last = heap_id; + } + + if (ret && last <= abo->heap_end_id) { + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, + abo->heap_start_id, last) + amdxdna_bo_unpin(heap); + } + + return ret; +} + +int amdxdna_gem_pin(struct amdxdna_gem_obj *abo) +{ + int ret; + + mutex_lock(&abo->lock); + ret = amdxdna_gem_pin_nolock(abo); + mutex_unlock(&abo->lock); + + return ret; +} + +void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo) +{ + mutex_lock(&abo->lock); + if (abo->type == AMDXDNA_BO_DEV) { + struct amdxdna_gem_obj *heap; + unsigned long heap_id; + + xa_for_each_range(&abo->client->dev_heap_xa, heap_id, heap, + abo->heap_start_id, abo->heap_end_id) + amdxdna_bo_unpin(heap); + } else { + amdxdna_bo_unpin(abo); + } + + mutex_unlock(&abo->lock); +} + +struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, + u32 bo_hdl, u8 bo_type) +{ + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + + gobj = drm_gem_object_lookup(client->filp, bo_hdl); + if (!gobj) { + XDNA_DBG(client->xdna, "Can not find bo %d", bo_hdl); + return NULL; + } + + abo = to_xdna_obj(gobj); + if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) + return abo; + + drm_gem_object_put(gobj); + return NULL; +} + +int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_drm_get_bo_info *args = data; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + int ret = 0; + + if (args->ext || args->ext_flags || args->pad) + return -EINVAL; + + gobj = drm_gem_object_lookup(filp, args->handle); + if (!gobj) { + XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle); + return -ENOENT; + } + + abo = to_xdna_obj(gobj); + args->vaddr = amdxdna_gem_uva(abo); + args->xdna_addr = amdxdna_gem_dev_addr(abo); + + if (abo->type != AMDXDNA_BO_DEV) + args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node); + else + args->map_offset = AMDXDNA_INVALID_ADDR; + + XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx", + args->handle, args->map_offset, args->vaddr, args->xdna_addr); + + drm_gem_object_put(gobj); + return ret; +} + +static int amdxdna_flush_bo(struct amdxdna_gem_obj *abo, u64 offset, u64 size) +{ + u64 end; + + if (offset >= abo->mem.size) + return -EINVAL; + + if (check_add_overflow(offset, size, &end)) + return -EINVAL; + + size = min(abo->mem.size, end) - offset; + if (is_import_bo(abo)) + drm_clflush_sg(abo->base.sgt); + else if (amdxdna_gem_vmap(abo)) + drm_clflush_virt_range(amdxdna_gem_vmap(abo) + offset, size); + else if (abo->base.pages) + drm_clflush_pages(abo->base.pages, abo->mem.size >> PAGE_SHIFT); + else + return -EINVAL; + + return 0; +} + +/* + * The sync bo ioctl is to make sure the CPU cache is in sync with memory. + * This is required because NPU is not cache coherent device. CPU cache + * flushing/invalidation is expensive so it is best to handle this outside + * of the command submission path. This ioctl allows explicit cache + * flushing/invalidation outside of the critical path. + */ +int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, + void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_sync_bo *args = data; + struct amdxdna_gem_obj *abo; + struct drm_gem_object *gobj; + int ret = 0; + + gobj = drm_gem_object_lookup(filp, args->handle); + if (!gobj) { + XDNA_ERR(xdna, "Lookup GEM object failed"); + return -ENOENT; + } + abo = to_xdna_obj(gobj); + + if (abo->type == AMDXDNA_BO_DEV) { + struct amdxdna_gem_obj *heap; + unsigned long heap_id; + u64 bo_start = amdxdna_gem_dev_addr(abo); + u64 flush_start = bo_start + args->offset; + u64 flush_end = flush_start + args->size; + + xa_for_each_range(&client->dev_heap_xa, heap_id, heap, + abo->heap_start_id, abo->heap_end_id) { + u64 heap_start = amdxdna_gem_dev_addr(heap); + u64 heap_end = heap_start + heap->mem.size; + u64 start = max(flush_start, heap_start); + u64 end = min(flush_end, heap_end); + + if (start >= end) + continue; + + ret = amdxdna_flush_bo(heap, start - heap_start, end - start); + if (ret) { + XDNA_ERR(xdna, "Failed to flush heap %ld ret %d", + heap_id, ret); + goto put_obj; + } + } + } else { + ret = amdxdna_gem_pin(abo); + if (ret) { + XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret); + goto put_obj; + } + + ret = amdxdna_flush_bo(abo, args->offset, args->size); + amdxdna_gem_unpin(abo); + + if (ret) { + drm_WARN(&xdna->ddev, 1, "Can not get flush memory"); + goto put_obj; + } + } + + XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n", + args->handle, args->offset, args->size); + + if (args->direction == SYNC_DIRECT_FROM_DEVICE) + ret = amdxdna_hwctx_sync_debug_bo(client, args->handle); + +put_obj: + drm_gem_object_put(gobj); + return ret; +} + +int amdxdna_drm_get_bo_usage(struct drm_device *dev, struct amdxdna_drm_get_array *args) +{ + size_t min_sz = min(args->element_size, sizeof(struct amdxdna_drm_bo_usage)); + char __user *buf = u64_to_user_ptr(args->buffer); + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_client *tmp_client; + struct amdxdna_drm_bo_usage tmp; + + drm_WARN_ON(dev, !mutex_is_locked(&xdna->dev_lock)); + + if (args->num_element != 1) + return -EINVAL; + + if (copy_from_user(&tmp, buf, min_sz)) + return -EFAULT; + + if (!tmp.pid) + return -EINVAL; + + tmp.total_usage = 0; + tmp.internal_usage = 0; + tmp.heap_usage = 0; + + list_for_each_entry(tmp_client, &xdna->client_list, node) { + if (tmp.pid != tmp_client->pid) + continue; + + mutex_lock(&tmp_client->mm_lock); + tmp.total_usage += tmp_client->total_bo_usage; + tmp.internal_usage += tmp_client->total_int_bo_usage; + tmp.heap_usage += tmp_client->heap_usage; + mutex_unlock(&tmp_client->mm_lock); + } + + if (copy_to_user(buf, &tmp, min_sz)) + return -EFAULT; + + return 0; +} diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h new file mode 100644 index 000000000..1e90e32bf --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_gem.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#ifndef _AMDXDNA_GEM_H_ +#define _AMDXDNA_GEM_H_ + +#include <drm/drm_gem_shmem_helper.h> +#include <linux/hmm.h> +#include <linux/iova.h> +#include "amdxdna_pci_drv.h" + +struct amdxdna_umap { + struct mmu_interval_notifier notifier; + struct hmm_range range; + struct work_struct hmm_unreg_work; + struct amdxdna_gem_obj *abo; + struct list_head node; + struct kref refcnt; + bool invalid; + bool unmapped; +}; + +struct amdxdna_mem { + void *kva; + u64 dma_addr; + size_t size; + struct list_head umap_list; + bool map_invalid; + /* + * Cache the first mmap uva as PASID addr, which can be accessed by driver + * without taking notifier_lock. + */ + u64 uva; +}; + +struct amdxdna_gem_obj { + struct drm_gem_shmem_object base; + struct amdxdna_client *client; + u8 type; + bool pinned; + struct mutex lock; /* Protects: pinned, mem.kva, open_ref */ + struct amdxdna_mem mem; + int open_ref; + + /* Below members are initialized when needed */ + struct drm_mm_node mm_node; /* For AMDXDNA_BO_DEV */ + u32 heap_start_id; + u32 heap_end_id; + u64 dev_addr; /* For heap bo */ + u32 assigned_hwctx; + struct dma_buf *dma_buf; + struct dma_buf_attachment *attach; + + /* True, if BO is managed by XRT, not application */ + bool internal; + /* True, if BO is not exportable */ + bool private_buffer; +}; + +#define to_gobj(obj) (&(obj)->base.base) +#define is_import_bo(obj) ((obj)->attach) + +static inline struct amdxdna_gem_obj *to_xdna_obj(struct drm_gem_object *gobj) +{ + return container_of(gobj, struct amdxdna_gem_obj, base.base); +} + +struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, + u32 bo_hdl, u8 bo_type); +static inline void amdxdna_gem_put_obj(struct amdxdna_gem_obj *abo) +{ + drm_gem_object_put(to_gobj(abo)); +} + +/* + * Obtain the user virtual address for accessing the BO. + * It can be used for device to access the BO when PASID is enabled. + */ +static inline u64 amdxdna_gem_uva(struct amdxdna_gem_obj *abo) +{ + return abo->mem.uva; +} + +void *amdxdna_gem_vmap(struct amdxdna_gem_obj *abo); +u64 amdxdna_gem_dev_addr(struct amdxdna_gem_obj *abo); + +static inline u64 amdxdna_dev_bo_offset(struct amdxdna_gem_obj *abo) +{ + return amdxdna_gem_dev_addr(abo) - to_xdna_dev(to_gobj(abo)->dev)->dev_info->dev_mem_base; +} + +static inline u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *abo) +{ + /* + * amdxdna_gem_obj_open() calls amdxdna_dma_map_bo() only when PASID is + * off, leaving mem.dma_addr at AMDXDNA_INVALID_ADDR when PASID is on. + * Avoid dereferencing abo->client, which is cleared to NULL by + * amdxdna_gem_obj_close() while internal kernel references remain. + */ + return (abo->mem.dma_addr != AMDXDNA_INVALID_ADDR) ? + abo->mem.dma_addr : amdxdna_gem_uva(abo); +} + +void amdxdna_umap_put(struct amdxdna_umap *mapp); + +struct drm_gem_object * +amdxdna_gem_create_shmem_object_cb(struct drm_device *dev, size_t size); +struct drm_gem_object * +amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf); +struct amdxdna_gem_obj * +amdxdna_drm_create_dev_bo(struct drm_device *dev, + struct amdxdna_drm_create_bo *args, struct drm_file *filp); + +int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo); +int amdxdna_gem_pin(struct amdxdna_gem_obj *abo); +void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo); + +int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdxdna_drm_get_bo_usage(struct drm_device *dev, struct amdxdna_drm_get_array *args); + +#endif /* _AMDXDNA_GEM_H_ */ diff --git a/drivers/accel/amdxdna/amdxdna_iommu.c b/drivers/accel/amdxdna/amdxdna_iommu.c new file mode 100644 index 000000000..4f245b969 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_iommu.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_managed.h> +#include <linux/iommu.h> +#include <linux/iova.h> + +#include "amdxdna_gem.h" +#include "amdxdna_pci_drv.h" + +static bool force_iova; +module_param(force_iova, bool, 0600); +MODULE_PARM_DESC(force_iova, "Force use IOVA (Default false)"); + +static struct iova *amdxdna_iommu_alloc_iova(struct amdxdna_dev *xdna, + size_t size, + dma_addr_t *dma_addr, + bool size_aligned) +{ + unsigned long shift, end; + struct iova *iova; + + end = xdna->domain->geometry.aperture_end; + shift = iova_shift(&xdna->iovad); + size = iova_align(&xdna->iovad, size); + + iova = alloc_iova(&xdna->iovad, size >> shift, end >> shift, size_aligned); + if (!iova) + return ERR_PTR(-ENOMEM); + + *dma_addr = iova_dma_addr(&xdna->iovad, iova); + + return iova; +} + +int amdxdna_dma_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo) +{ + unsigned long contig_sz; + struct sg_table *sgt; + dma_addr_t dma_addr; + struct iova *iova; + ssize_t size; + + if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHARE) + return 0; + + sgt = drm_gem_shmem_get_pages_sgt(&abo->base); + if (IS_ERR(sgt)) { + XDNA_ERR(xdna, "Get sgt failed, ret %ld", PTR_ERR(sgt)); + return PTR_ERR(sgt); + } + + if (!sgt->orig_nents) { + XDNA_ERR(xdna, "sgl is zero length"); + return -EOPNOTSUPP; + } + + if (amdxdna_iova_on(xdna)) { + if (!sg_page(sgt->sgl)) { + XDNA_ERR(xdna, "sgl is not page backed"); + return -EOPNOTSUPP; + } + + iova = amdxdna_iommu_alloc_iova(xdna, abo->mem.size, &dma_addr, + (abo->type == AMDXDNA_BO_DEV_HEAP)); + if (IS_ERR(iova)) { + XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova)); + return PTR_ERR(iova); + } + + size = iommu_map_sgtable(xdna->domain, dma_addr, sgt, + IOMMU_READ | IOMMU_WRITE); + if (size < 0) { + XDNA_ERR(xdna, "iommu_map_sgtable failed: %zd", size); + __free_iova(&xdna->iovad, iova); + return size; + } + if (size < abo->mem.size) { + iommu_unmap(xdna->domain, dma_addr, size); + __free_iova(&xdna->iovad, iova); + return -ENXIO; + } + abo->mem.dma_addr = dma_addr; + } else { + /* Device doesn't support scatter/gather list, fail non-contiguous mapping. */ + contig_sz = drm_prime_get_contiguous_size(sgt); + if (contig_sz < abo->mem.size) { + XDNA_ERR(xdna, + "noncontiguous dma addr, contig size:%ld, expected size:%ld", + contig_sz, abo->mem.size); + return -EINVAL; + } + abo->mem.dma_addr = sg_dma_address(sgt->sgl); + } + return 0; +} + +void amdxdna_dma_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo) +{ + size_t size; + + if (abo->mem.dma_addr == AMDXDNA_INVALID_ADDR) + return; + + if (amdxdna_iova_on(xdna)) { + size = iova_align(&xdna->iovad, abo->mem.size); + iommu_unmap(xdna->domain, abo->mem.dma_addr, size); + free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, abo->mem.dma_addr)); + } + abo->mem.dma_addr = AMDXDNA_INVALID_ADDR; +} + +void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr) +{ + struct iova *iova; + void *cpu_addr; + int ret; + + iova = amdxdna_iommu_alloc_iova(xdna, size, dma_addr, true); + if (IS_ERR(iova)) { + XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova)); + return iova; + } + + cpu_addr = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!cpu_addr) { + ret = -ENOMEM; + goto free_iova; + } + + ret = iommu_map(xdna->domain, *dma_addr, virt_to_phys(cpu_addr), + iova_align(&xdna->iovad, size), + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + if (ret) + goto free_cpu_addr; + + return cpu_addr; + +free_cpu_addr: + free_pages((unsigned long)cpu_addr, get_order(size)); +free_iova: + __free_iova(&xdna->iovad, iova); + return ERR_PTR(ret); +} + +void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size, + void *cpu_addr, dma_addr_t dma_addr) +{ + iommu_unmap(xdna->domain, dma_addr, iova_align(&xdna->iovad, size)); + free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, dma_addr)); + free_pages((unsigned long)cpu_addr, get_order(size)); +} + +static void amdxdna_cleanup_force_iova(struct drm_device *dev, void *res) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + + if (xdna->domain) { + iommu_detach_group(xdna->domain, xdna->group); + put_iova_domain(&xdna->iovad); + iova_cache_put(); + iommu_domain_free(xdna->domain); + } + + iommu_group_put(xdna->group); +} + +void amdxdna_iommu_fini(struct amdxdna_dev *xdna) +{ + if (xdna->group && !xdna->domain) + iommu_group_put(xdna->group); +} + +int amdxdna_iommu_init(struct amdxdna_dev *xdna) +{ + unsigned long order; + int ret = 0; + + xdna->group = iommu_group_get(xdna->ddev.dev); + if (!xdna->group || !force_iova) + return 0; + + XDNA_WARN(xdna, "Enabled force_iova mode."); + xdna->domain = iommu_paging_domain_alloc_flags(xdna->ddev.dev, + IOMMU_HWPT_ALLOC_PASID); + if (IS_ERR(xdna->domain)) { + XDNA_ERR(xdna, "Failed to alloc iommu domain"); + ret = PTR_ERR(xdna->domain); + goto put_group; + } + + ret = iova_cache_get(); + if (ret) + goto free_domain; + + order = __ffs(xdna->domain->pgsize_bitmap); + init_iova_domain(&xdna->iovad, 1UL << order, 0); + + ret = iommu_attach_group(xdna->domain, xdna->group); + if (ret) + goto put_iova; + + ret = drmm_add_action(&xdna->ddev, amdxdna_cleanup_force_iova, NULL); + if (ret) + goto detach_group; + + return 0; + +detach_group: + iommu_detach_group(xdna->domain, xdna->group); +put_iova: + put_iova_domain(&xdna->iovad); + iova_cache_put(); +free_domain: + iommu_domain_free(xdna->domain); +put_group: + iommu_group_put(xdna->group); + xdna->group = NULL; + xdna->domain = NULL; + + return ret; +} diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c new file mode 100644 index 000000000..cc8865f4e --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/drm_device.h> +#include <drm/drm_managed.h> +#include <linux/bitfield.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/slab.h> +#include <linux/xarray.h> + +#define CREATE_TRACE_POINTS +#include <trace/events/amdxdna.h> + +#include "amdxdna_mailbox.h" + +#define MB_ERR(chann, fmt, args...) \ +({ \ + typeof(chann) _chann = chann; \ + dev_err((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \ + (_chann)->msix_irq, ##args); \ +}) +#define MB_DBG(chann, fmt, args...) \ +({ \ + typeof(chann) _chann = chann; \ + dev_dbg((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \ + (_chann)->msix_irq, ##args); \ +}) +#define MB_WARN_ONCE(chann, fmt, args...) \ +({ \ + typeof(chann) _chann = chann; \ + dev_warn_once((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \ + (_chann)->msix_irq, ##args); \ +}) + +#define MAGIC_VAL 0x1D000000U +#define MAGIC_VAL_MASK 0xFF000000 +#define MAX_MSG_ID_ENTRIES 256 +#define MSG_RX_TIMER 200 /* milliseconds */ +#define MAILBOX_NAME "xdna_mailbox" + +enum channel_res_type { + CHAN_RES_X2I, + CHAN_RES_I2X, + CHAN_RES_NUM +}; + +struct mailbox { + struct device *dev; + struct xdna_mailbox_res res; +}; + +struct mailbox_channel { + struct mailbox *mb; + struct xdna_mailbox_chann_res res[CHAN_RES_NUM]; + int msix_irq; + u32 iohub_int_addr; + struct xarray chan_xa; + u32 next_msgid; + u32 x2i_tail; + + /* Received msg related fields */ + struct workqueue_struct *work_q; + struct work_struct rx_work; + u32 i2x_head; + bool bad_state; +}; + +#define MSG_BODY_SZ GENMASK(10, 0) +#define MSG_PROTO_VER GENMASK(23, 16) +struct xdna_msg_header { + __u32 total_size; + __u32 sz_ver; + __u32 id; + __u32 opcode; +} __packed; + +static_assert(sizeof(struct xdna_msg_header) == 16); + +struct mailbox_pkg { + struct xdna_msg_header header; + __u32 payload[]; +}; + +/* The protocol version. */ +#define MSG_PROTOCOL_VERSION 0x1 +/* The tombstone value. */ +#define TOMBSTONE 0xDEADFACE + +struct mailbox_msg { + void *handle; + int (*notify_cb)(void *handle, void __iomem *data, size_t size); + size_t pkg_size; /* package size in bytes */ + struct mailbox_pkg pkg; +}; + +static void mailbox_reg_write(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 data) +{ + struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; + void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; + + writel(data, ringbuf_addr); +} + +static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg) +{ + struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; + void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; + + return readl(ringbuf_addr); +} + +static inline void mailbox_irq_acknowledge(struct mailbox_channel *mb_chann) +{ + if (mb_chann->iohub_int_addr) + mailbox_reg_write(mb_chann, mb_chann->iohub_int_addr, 0); +} + +static inline u32 mailbox_irq_status(struct mailbox_channel *mb_chann) +{ + return (mb_chann->iohub_int_addr) ? + mailbox_reg_read(mb_chann, mb_chann->iohub_int_addr) : 0; +} + +static inline void +mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val) +{ + mailbox_reg_write(mb_chann, mb_chann->res[CHAN_RES_I2X].mb_head_ptr_reg, headptr_val); + mb_chann->i2x_head = headptr_val; +} + +static inline void +mailbox_set_tailptr(struct mailbox_channel *mb_chann, u32 tailptr_val) +{ + mailbox_reg_write(mb_chann, mb_chann->res[CHAN_RES_X2I].mb_tail_ptr_reg, tailptr_val); + mb_chann->x2i_tail = tailptr_val; +} + +static inline u32 +mailbox_get_headptr(struct mailbox_channel *mb_chann, enum channel_res_type type) +{ + return mailbox_reg_read(mb_chann, mb_chann->res[type].mb_head_ptr_reg); +} + +static inline u32 +mailbox_get_tailptr(struct mailbox_channel *mb_chann, enum channel_res_type type) +{ + return mailbox_reg_read(mb_chann, mb_chann->res[type].mb_tail_ptr_reg); +} + +static inline u32 +mailbox_get_ringbuf_size(struct mailbox_channel *mb_chann, enum channel_res_type type) +{ + return mb_chann->res[type].rb_size; +} + +static inline int mailbox_validate_msgid(int msg_id) +{ + return (msg_id & MAGIC_VAL_MASK) == MAGIC_VAL; +} + +static int mailbox_acquire_msgid(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) +{ + u32 msg_id; + int ret; + + ret = xa_alloc_cyclic_irq(&mb_chann->chan_xa, &msg_id, mb_msg, + XA_LIMIT(0, MAX_MSG_ID_ENTRIES - 1), + &mb_chann->next_msgid, GFP_NOWAIT); + if (ret < 0) + return ret; + + /* + * Add MAGIC_VAL to the higher bits. + */ + msg_id |= MAGIC_VAL; + return msg_id; +} + +static void mailbox_release_msgid(struct mailbox_channel *mb_chann, int msg_id) +{ + msg_id &= ~MAGIC_VAL_MASK; + xa_erase_irq(&mb_chann->chan_xa, msg_id); +} + +static void mailbox_release_msg(struct mailbox_channel *mb_chann, + struct mailbox_msg *mb_msg) +{ + MB_DBG(mb_chann, "msg_id 0x%x msg opcode 0x%x", + mb_msg->pkg.header.id, mb_msg->pkg.header.opcode); + if (mb_msg->notify_cb) + mb_msg->notify_cb(mb_msg->handle, NULL, 0); + kfree(mb_msg); +} + +static int +mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) +{ + void __iomem *write_addr; + u32 ringbuf_size; + u32 head, tail; + u32 start_addr; + u32 tmp_tail; + int ret; + + head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I); + tail = mb_chann->x2i_tail; + ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32); + start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr; + tmp_tail = tail + mb_msg->pkg_size; + +check_again: + if (tail >= head && tmp_tail > ringbuf_size) { + write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; + writel(TOMBSTONE, write_addr); + + /* tombstone is set. Write from the start of the ringbuf */ + tail = 0; + tmp_tail = tail + mb_msg->pkg_size; + } + + if (tail < head && tmp_tail >= head) { + ret = read_poll_timeout(mailbox_get_headptr, head, + tmp_tail < head || tail >= head, + 1, 100, false, mb_chann, CHAN_RES_X2I); + if (ret) + return ret; + + if (tail >= head) + goto check_again; + } + + write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; + memcpy_toio(write_addr, &mb_msg->pkg, mb_msg->pkg_size); + mailbox_set_tailptr(mb_chann, tail + mb_msg->pkg_size); + + trace_mbox_set_tail(MAILBOX_NAME, mb_chann->msix_irq, + mb_msg->pkg.header.opcode, + mb_msg->pkg.header.id); + + return 0; +} + +static int +mailbox_get_resp(struct mailbox_channel *mb_chann, struct xdna_msg_header *header, + void __iomem *data) +{ + struct mailbox_msg *mb_msg; + int msg_id; + int ret = 0; + + msg_id = header->id; + if (!mailbox_validate_msgid(msg_id)) { + MB_ERR(mb_chann, "Bad message ID 0x%x", msg_id); + return -EINVAL; + } + + msg_id &= ~MAGIC_VAL_MASK; + mb_msg = xa_erase_irq(&mb_chann->chan_xa, msg_id); + if (!mb_msg) { + MB_ERR(mb_chann, "Cannot find msg 0x%x", msg_id); + return -EINVAL; + } + + MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x", + header->opcode, header->total_size, header->id); + if (mb_msg->notify_cb) { + ret = mb_msg->notify_cb(mb_msg->handle, data, header->total_size); + if (unlikely(ret)) + MB_ERR(mb_chann, "Message callback ret %d", ret); + } + + kfree(mb_msg); + return ret; +} + +static int mailbox_get_msg(struct mailbox_channel *mb_chann) +{ + struct xdna_msg_header header; + void __iomem *read_addr; + u32 msg_size, rest; + u32 ringbuf_size; + u32 head, tail; + u32 start_addr; + int ret; + + tail = mailbox_get_tailptr(mb_chann, CHAN_RES_I2X); + head = mb_chann->i2x_head; + ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_I2X); + start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr; + + if (unlikely(tail > ringbuf_size || !IS_ALIGNED(tail, 4))) { + MB_WARN_ONCE(mb_chann, "Invalid tail 0x%x", tail); + return -EINVAL; + } + + /* ringbuf empty */ + if (head == tail) + return -ENOENT; + + if (head == ringbuf_size) + head = 0; + + /* Peek size of the message or TOMBSTONE */ + read_addr = mb_chann->mb->res.ringbuf_base + start_addr + head; + header.total_size = readl(read_addr); + /* size is TOMBSTONE, set next read from 0 */ + if (header.total_size == TOMBSTONE) { + if (head < tail) { + MB_WARN_ONCE(mb_chann, "Tombstone, head 0x%x tail 0x%x", + head, tail); + return -EINVAL; + } + mailbox_set_headptr(mb_chann, 0); + return 0; + } + + if (unlikely(!header.total_size || !IS_ALIGNED(header.total_size, 4))) { + MB_WARN_ONCE(mb_chann, "Invalid total size 0x%x", header.total_size); + return -EINVAL; + } + msg_size = sizeof(header) + header.total_size; + + if (msg_size > ringbuf_size - head || msg_size > tail - head) { + MB_WARN_ONCE(mb_chann, "Invalid message size %d, tail %d, head %d", + msg_size, tail, head); + return -EINVAL; + } + + rest = sizeof(header) - sizeof(u32); + read_addr += sizeof(u32); + memcpy_fromio((u32 *)&header + 1, read_addr, rest); + read_addr += rest; + + ret = mailbox_get_resp(mb_chann, &header, read_addr); + + mailbox_set_headptr(mb_chann, head + msg_size); + /* After update head, it can equal to ringbuf_size. This is expected. */ + trace_mbox_set_head(MAILBOX_NAME, mb_chann->msix_irq, + header.opcode, header.id); + + return ret; +} + +static irqreturn_t mailbox_irq_handler(int irq, void *p) +{ + struct mailbox_channel *mb_chann = p; + + trace_mbox_irq_handle(MAILBOX_NAME, irq); + /* Schedule a rx_work to call the callback functions */ + queue_work(mb_chann->work_q, &mb_chann->rx_work); + + return IRQ_HANDLED; +} + +static void mailbox_rx_worker(struct work_struct *rx_work) +{ + struct mailbox_channel *mb_chann; + int ret; + + mb_chann = container_of(rx_work, struct mailbox_channel, rx_work); + trace_mbox_rx_worker(MAILBOX_NAME, mb_chann->msix_irq); + + if (READ_ONCE(mb_chann->bad_state)) { + MB_ERR(mb_chann, "Channel in bad state, work aborted"); + return; + } + +again: + mailbox_irq_acknowledge(mb_chann); + + while (1) { + /* + * If return is 0, keep consuming next message, until there is + * no messages or an error happened. + */ + ret = mailbox_get_msg(mb_chann); + if (ret == -ENOENT) + break; + + /* Other error means device doesn't look good, disable irq. */ + if (unlikely(ret)) { + MB_ERR(mb_chann, "Unexpected ret %d, disable irq", ret); + WRITE_ONCE(mb_chann->bad_state, true); + return; + } + } + + /* + * The hardware will not generate interrupt if firmware creates a new + * response right after driver clears interrupt register. Check + * the interrupt register to make sure there is not any new response + * before exiting. + */ + if (mailbox_irq_status(mb_chann)) + goto again; +} + +int xdna_mailbox_send_msg(struct mailbox_channel *mb_chann, + const struct xdna_mailbox_msg *msg, u64 tx_timeout) +{ + struct xdna_msg_header *header; + struct mailbox_msg *mb_msg; + size_t pkg_size; + int ret; + + pkg_size = sizeof(*header) + msg->send_size; + if (pkg_size > mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I)) { + MB_ERR(mb_chann, "Message size larger than ringbuf size"); + return -EINVAL; + } + + if (unlikely(!IS_ALIGNED(msg->send_size, 4))) { + MB_ERR(mb_chann, "Message must be 4 bytes align"); + return -EINVAL; + } + + /* The fist word in payload can NOT be TOMBSTONE */ + if (unlikely(((u32 *)msg->send_data)[0] == TOMBSTONE)) { + MB_ERR(mb_chann, "Tomb stone in data"); + return -EINVAL; + } + + if (READ_ONCE(mb_chann->bad_state)) { + MB_ERR(mb_chann, "Channel in bad state"); + return -EPIPE; + } + + mb_msg = kzalloc(sizeof(*mb_msg) + pkg_size, GFP_KERNEL); + if (!mb_msg) + return -ENOMEM; + + mb_msg->handle = msg->handle; + mb_msg->notify_cb = msg->notify_cb; + mb_msg->pkg_size = pkg_size; + + header = &mb_msg->pkg.header; + /* + * Hardware use total_size and size to split huge message. + * We do not support it here. Thus the values are the same. + */ + header->total_size = msg->send_size; + header->sz_ver = FIELD_PREP(MSG_BODY_SZ, msg->send_size) | + FIELD_PREP(MSG_PROTO_VER, MSG_PROTOCOL_VERSION); + header->opcode = msg->opcode; + memcpy(mb_msg->pkg.payload, msg->send_data, msg->send_size); + + ret = mailbox_acquire_msgid(mb_chann, mb_msg); + if (unlikely(ret < 0)) { + MB_ERR(mb_chann, "mailbox_acquire_msgid failed"); + goto msg_id_failed; + } + header->id = ret; + + MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x", + header->opcode, header->total_size, header->id); + + ret = mailbox_send_msg(mb_chann, mb_msg); + if (ret) { + MB_DBG(mb_chann, "Error in mailbox send msg, ret %d", ret); + goto release_id; + } + + return 0; + +release_id: + mailbox_release_msgid(mb_chann, header->id); +msg_id_failed: + kfree(mb_msg); + return ret; +} + +struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb) +{ + struct mailbox_channel *mb_chann; + + mb_chann = kzalloc_obj(*mb_chann); + if (!mb_chann) + return NULL; + + INIT_WORK(&mb_chann->rx_work, mailbox_rx_worker); + mb_chann->work_q = create_singlethread_workqueue(MAILBOX_NAME); + if (!mb_chann->work_q) { + MB_ERR(mb_chann, "Create workqueue failed"); + goto free_chann; + } + mb_chann->mb = mb; + + return mb_chann; + +free_chann: + kfree(mb_chann); + return NULL; +} + +void xdna_mailbox_free_channel(struct mailbox_channel *mb_chann) +{ + if (!mb_chann) + return; + + destroy_workqueue(mb_chann->work_q); + kfree(mb_chann); +} + +int +xdna_mailbox_start_channel(struct mailbox_channel *mb_chann, + const struct xdna_mailbox_chann_res *x2i, + const struct xdna_mailbox_chann_res *i2x, + u32 iohub_int_addr, + int mb_irq) +{ + int ret; + + if (!is_power_of_2(x2i->rb_size) || !is_power_of_2(i2x->rb_size)) { + pr_err("Ring buf size must be power of 2\n"); + return -EINVAL; + } + + mb_chann->msix_irq = mb_irq; + mb_chann->iohub_int_addr = iohub_int_addr; + memcpy(&mb_chann->res[CHAN_RES_X2I], x2i, sizeof(*x2i)); + memcpy(&mb_chann->res[CHAN_RES_I2X], i2x, sizeof(*i2x)); + + xa_init_flags(&mb_chann->chan_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); + mb_chann->x2i_tail = mailbox_get_tailptr(mb_chann, CHAN_RES_X2I); + mb_chann->i2x_head = mailbox_get_headptr(mb_chann, CHAN_RES_I2X); + + /* Everything look good. Time to enable irq handler */ + ret = request_irq(mb_irq, mailbox_irq_handler, 0, MAILBOX_NAME, mb_chann); + if (ret) { + MB_ERR(mb_chann, "Failed to request irq %d ret %d", mb_irq, ret); + return ret; + } + + mb_chann->bad_state = false; + mailbox_irq_acknowledge(mb_chann); + + MB_DBG(mb_chann, "Mailbox channel started (irq: %d)", mb_chann->msix_irq); + return 0; +} + +void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann) +{ + struct mailbox_msg *mb_msg; + unsigned long msg_id; + + if (!mb_chann) + return; + + /* Disable an irq and wait. This might sleep. */ + free_irq(mb_chann->msix_irq, mb_chann); + + /* Cancel RX work and wait for it to finish */ + drain_workqueue(mb_chann->work_q); + + /* We can clean up and release resources */ + xa_for_each_start(&mb_chann->chan_xa, msg_id, mb_msg, mb_chann->next_msgid) + mailbox_release_msg(mb_chann, mb_msg); + xa_for_each_range(&mb_chann->chan_xa, msg_id, mb_msg, 0, mb_chann->next_msgid - 1) + mailbox_release_msg(mb_chann, mb_msg); + xa_destroy(&mb_chann->chan_xa); + + MB_DBG(mb_chann, "Mailbox channel stopped, irq: %d", mb_chann->msix_irq); +} + +struct mailbox *xdnam_mailbox_create(struct drm_device *ddev, + const struct xdna_mailbox_res *res) +{ + struct mailbox *mb; + + mb = drmm_kzalloc(ddev, sizeof(*mb), GFP_KERNEL); + if (!mb) + return NULL; + mb->dev = ddev->dev; + + /* mailbox and ring buf base and size information */ + memcpy(&mb->res, res, sizeof(*res)); + + return mb; +} diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.h b/drivers/accel/amdxdna/amdxdna_mailbox.h new file mode 100644 index 000000000..290840430 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_mailbox.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022-2026, Advanced Micro Devices, Inc. + */ + +#ifndef _AIE_MAILBOX_H_ +#define _AIE_MAILBOX_H_ + +struct mailbox; +struct mailbox_channel; + +/* + * xdna_mailbox_msg - message struct + * + * @opcode: opcode for firmware + * @handle: handle used for the notify callback + * @notify_cb: callback function to notify the sender when there is response + * @send_data: pointing to sending data + * @send_size: size of the sending data + * + * The mailbox will split the sending data in to multiple firmware message if + * the size of the data is too big. This is transparent to the sender. The + * sender will receive one notification. + */ +struct xdna_mailbox_msg { + u32 opcode; + void *handle; + int (*notify_cb)(void *handle, void __iomem *data, size_t size); + u8 *send_data; + size_t send_size; +}; + +/* + * xdna_mailbox_res - mailbox hardware resource + * + * @ringbuf_base: ring buffer base address + * @ringbuf_size: ring buffer size + * @mbox_base: mailbox base address + * @mbox_size: mailbox size + */ +struct xdna_mailbox_res { + void __iomem *ringbuf_base; + size_t ringbuf_size; + void __iomem *mbox_base; + size_t mbox_size; + const char *name; +}; + +/* + * xdna_mailbox_chann_res - resources + * + * @rb_start_addr: ring buffer start address + * @rb_size: ring buffer size + * @mb_head_ptr_reg: mailbox head pointer register + * @mb_tail_ptr_reg: mailbox tail pointer register + */ +struct xdna_mailbox_chann_res { + u32 rb_start_addr; + u32 rb_size; + u32 mb_head_ptr_reg; + u32 mb_tail_ptr_reg; +}; + +/* + * xdna_mailbox_create() -- create mailbox subsystem and initialize + * + * @ddev: device pointer + * @res: SRAM and mailbox resources + * + * Return: If success, return a handle of mailbox subsystem. + * Otherwise, return NULL pointer. + */ +struct mailbox *xdnam_mailbox_create(struct drm_device *ddev, + const struct xdna_mailbox_res *res); + +/* + * xdna_mailbox_alloc_channel() -- alloc a mailbox channel + * + * @mb: mailbox handle + */ +struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb); + +/* + * xdna_mailbox_start_channel() -- start a mailbox channel instance + * + * @mb_chann: the handle return from xdna_mailbox_alloc_channel() + * @x2i: host to firmware mailbox resources + * @i2x: firmware to host mailbox resources + * @xdna_mailbox_intr_reg: register addr of MSI-X interrupt + * @mb_irq: Linux IRQ number associated with mailbox MSI-X interrupt vector index + * + * Return: If success, return a handle of mailbox channel. Otherwise, return NULL. + */ +int +xdna_mailbox_start_channel(struct mailbox_channel *mb_chann, + const struct xdna_mailbox_chann_res *x2i, + const struct xdna_mailbox_chann_res *i2x, + u32 xdna_mailbox_intr_reg, + int mb_irq); + +/* + * xdna_mailbox_free_channel() -- free mailbox channel + * + * @mailbox_chann: the handle return from xdna_mailbox_create_channel() + */ +void xdna_mailbox_free_channel(struct mailbox_channel *mailbox_chann); + +/* + * xdna_mailbox_stop_channel() -- stop mailbox channel + * + * @mailbox_chann: the handle return from xdna_mailbox_create_channel() + */ +void xdna_mailbox_stop_channel(struct mailbox_channel *mailbox_chann); + +/* + * xdna_mailbox_send_msg() -- Send a message + * + * @mailbox_chann: Mailbox channel handle + * @msg: message struct for message information + * @tx_timeout: the timeout value for sending the message in ms. + * + * Return: If success return 0, otherwise, return error code + */ +int xdna_mailbox_send_msg(struct mailbox_channel *mailbox_chann, + const struct xdna_mailbox_msg *msg, u64 tx_timeout); + +#endif /* _AIE_MAILBOX_ */ diff --git a/drivers/accel/amdxdna/amdxdna_mailbox_helper.c b/drivers/accel/amdxdna/amdxdna_mailbox_helper.c new file mode 100644 index 000000000..6d0c24513 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_mailbox_helper.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_print.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/gpu_scheduler.h> +#include <linux/completion.h> + +#include "amdxdna_gem.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_mailbox_helper.h" +#include "amdxdna_pci_drv.h" + +int xdna_msg_cb(void *handle, void __iomem *data, size_t size) +{ + struct xdna_notify *cb_arg = handle; + int ret; + + if (unlikely(!data)) + goto out; + + if (unlikely(cb_arg->size != size)) { + cb_arg->error = -EINVAL; + goto out; + } + + memcpy_fromio(cb_arg->data, data, cb_arg->size); + print_hex_dump_debug("resp data: ", DUMP_PREFIX_OFFSET, + 16, 4, cb_arg->data, cb_arg->size, true); +out: + ret = cb_arg->error; + complete(&cb_arg->comp); + return ret; +} + +int xdna_send_msg_wait(struct amdxdna_dev *xdna, struct mailbox_channel *chann, + struct xdna_mailbox_msg *msg) +{ + struct xdna_notify *hdl = msg->handle; + int ret; + + ret = xdna_mailbox_send_msg(chann, msg, TX_TIMEOUT); + if (ret) { + XDNA_ERR(xdna, "Send message failed, ret %d", ret); + return ret; + } + + ret = wait_for_completion_timeout(&hdl->comp, + msecs_to_jiffies(RX_TIMEOUT)); + if (!ret) { + XDNA_ERR(xdna, "Wait for completion timeout"); + return -ETIME; + } + + return hdl->error; +} diff --git a/drivers/accel/amdxdna/amdxdna_mailbox_helper.h b/drivers/accel/amdxdna/amdxdna_mailbox_helper.h new file mode 100644 index 000000000..556c712ca --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_mailbox_helper.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef _AMDXDNA_MAILBOX_HELPER_H +#define _AMDXDNA_MAILBOX_HELPER_H + +#define TX_TIMEOUT 2000 /* milliseconds */ +#define RX_TIMEOUT 5000 /* milliseconds */ + +struct amdxdna_dev; + +struct xdna_notify { + struct completion comp; + u32 *data; + size_t size; + int error; + u32 *status; +}; + +#define DECLARE_XDNA_MSG_COMMON(name, op, s) \ + struct name##_req req = { 0 }; \ + struct name##_resp resp = { .status = s }; \ + struct xdna_notify hdl = { \ + .error = 0, \ + .data = (u32 *)&resp, \ + .size = sizeof(resp), \ + .comp = COMPLETION_INITIALIZER_ONSTACK(hdl.comp), \ + .status = (u32 *)&resp.status, \ + }; \ + struct xdna_mailbox_msg msg = { \ + .send_data = (u8 *)&req, \ + .send_size = sizeof(req), \ + .handle = &hdl, \ + .opcode = op, \ + .notify_cb = xdna_msg_cb, \ + } + +int xdna_msg_cb(void *handle, void __iomem *data, size_t size); +int xdna_send_msg_wait(struct amdxdna_dev *xdna, struct mailbox_channel *chann, + struct xdna_mailbox_msg *msg); + +#endif /* _AMDXDNA_MAILBOX_HELPER_H */ diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c new file mode 100644 index 000000000..bb339e641 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_accel.h> +#include <drm/drm_drv.h> +#include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_ioctl.h> +#include <drm/drm_managed.h> +#include <drm/gpu_scheduler.h> +#include <linux/iommu.h> +#include <linux/pci.h> + +#include "amdxdna_cbuf.h" +#include "amdxdna_ctx.h" +#include "amdxdna_debugfs.h" +#include "amdxdna_gem.h" +#include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" + +MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin"); +MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin"); +MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin"); +MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin"); +MODULE_FIRMWARE("amdnpu/1502_00/npu_7.sbin"); +MODULE_FIRMWARE("amdnpu/17f0_10/npu_7.sbin"); +MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin"); + +/* + * 0.0: Initial version + * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY + * 0.2: Support getting last error hardware error + * 0.3: Support firmware debug buffer + * 0.4: Support getting resource information + * 0.5: Support getting telemetry data + * 0.6: Support preemption + * 0.7: Support getting power and utilization data + * 0.8: Support BO usage query + * 0.9: Add new device type AMDXDNA_DEV_TYPE_PF + * 0.10: Support AIE4 UMQ + */ +#define AMDXDNA_DRIVER_MAJOR 0 +#define AMDXDNA_DRIVER_MINOR 10 + +/* + * Bind the driver base on (vendor_id, device_id) pair and later use the + * (device_id, rev_id) pair as a key to select the devices. The devices with + * same device_id have very similar interface to host driver. + */ +static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f2) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f3) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0B) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0C) }, + {0} +}; + +MODULE_DEVICE_TABLE(pci, pci_ids); + +static const struct amdxdna_device_id amdxdna_ids[] = { + { 0x1502, 0x0, &dev_npu1_info }, + { 0x17f0, 0x10, &dev_npu4_info }, + { 0x17f0, 0x11, &dev_npu5_info }, + { 0x17f0, 0x20, &dev_npu6_info }, + { 0x17f2, 0x10, &dev_npu3_pf_info }, + { 0x17f3, 0x10, &dev_npu3_vf_info }, + { 0x1B0B, 0x10, &dev_npu3_pf_info }, + { 0x1B0C, 0x10, &dev_npu3_vf_info }, + {0} +}; + +static int amdxdna_sva_init(struct amdxdna_client *client) +{ + struct amdxdna_dev *xdna = client->xdna; + + client->sva = iommu_sva_bind_device(xdna->ddev.dev, client->mm); + if (IS_ERR(client->sva)) { + XDNA_ERR(xdna, "SVA bind device failed, ret %ld", PTR_ERR(client->sva)); + return PTR_ERR(client->sva); + } + + client->pasid = iommu_sva_get_pasid(client->sva); + if (client->pasid == IOMMU_PASID_INVALID) { + iommu_sva_unbind_device(client->sva); + client->sva = NULL; + XDNA_ERR(xdna, "SVA get pasid failed"); + return -ENODEV; + } + + return 0; +} + +static void amdxdna_sva_fini(struct amdxdna_client *client) +{ + if (IS_ERR_OR_NULL(client->sva)) + return; + + iommu_sva_unbind_device(client->sva); + client->sva = NULL; + client->pasid = IOMMU_PASID_INVALID; +} + +static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) +{ + struct amdxdna_dev *xdna = to_xdna_dev(ddev); + struct amdxdna_client *client; + int ret; + + client = kzalloc_obj(*client); + if (!client) + return -ENOMEM; + + ret = init_srcu_struct(&client->hwctx_srcu); + if (ret) + goto free_client; + + client->pid = pid_nr(rcu_access_pointer(filp->pid)); + client->xdna = xdna; + client->pasid = IOMMU_PASID_INVALID; + client->mm = current->mm; + + if (!amdxdna_iova_on(xdna)) { + /* No need to fail open since user may use pa + carveout later. */ + if (amdxdna_sva_init(client)) { + XDNA_WARN(xdna, "PASID not available for pid %d", client->pid); + if (!amdxdna_use_carveout(xdna)) { + XDNA_ERR(xdna, "PASID unavailable and carveout not configured"); + ret = -EINVAL; + goto cleanup_srcu; + } + } + } + mmgrab(client->mm); + xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); + xa_init_flags(&client->dev_heap_xa, XA_FLAGS_ALLOC); + drm_mm_init(&client->dev_heap_mm, xdna->dev_info->dev_mem_base, + xdna->dev_info->dev_heap_max_size); + mutex_init(&client->mm_lock); + + mutex_lock(&xdna->client_lock); + mutex_lock(&xdna->dev_lock); + list_add_tail(&client->node, &xdna->client_list); + mutex_unlock(&xdna->dev_lock); + mutex_unlock(&xdna->client_lock); + + filp->driver_priv = client; + client->filp = filp; + + XDNA_DBG(xdna, "pid %d opened", client->pid); + return 0; + +cleanup_srcu: + cleanup_srcu_struct(&client->hwctx_srcu); +free_client: + kfree(client); + return ret; +} + +static void amdxdna_client_cleanup(struct amdxdna_client *client) +{ + struct amdxdna_gem_obj *heap; + unsigned long heap_id; + + list_del(&client->node); + amdxdna_hwctx_remove_all(client); + xa_destroy(&client->hwctx_xa); + cleanup_srcu_struct(&client->hwctx_srcu); + + xa_for_each(&client->dev_heap_xa, heap_id, heap) + drm_gem_object_put(to_gobj(heap)); + xa_destroy(&client->dev_heap_xa); + drm_mm_takedown(&client->dev_heap_mm); + + mutex_destroy(&client->mm_lock); + mmdrop(client->mm); + amdxdna_sva_fini(client); + kfree(client); +} + +static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(ddev); + + XDNA_DBG(xdna, "closing pid %d", client->pid); + + mutex_lock(&xdna->client_lock); + mutex_lock(&xdna->dev_lock); + amdxdna_client_cleanup(client); + mutex_unlock(&xdna->dev_lock); + mutex_unlock(&xdna->client_lock); +} + +static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_get_info *args = data; + int ret; + + if (!xdna->dev_info->ops->get_aie_info) + return -EOPNOTSUPP; + + XDNA_DBG(xdna, "Request parameter %u", args->param); + mutex_lock(&xdna->dev_lock); + ret = xdna->dev_info->ops->get_aie_info(client, args); + mutex_unlock(&xdna->dev_lock); + return ret; +} + +static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_get_array *args = data; + + if (!xdna->dev_info->ops->get_array) + return -EOPNOTSUPP; + + if (args->pad || !args->num_element || !args->element_size) + return -EINVAL; + + guard(mutex)(&xdna->dev_lock); + return xdna->dev_info->ops->get_array(client, args); +} + +static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + struct amdxdna_dev *xdna = to_xdna_dev(dev); + struct amdxdna_drm_set_state *args = data; + int ret; + + if (!xdna->dev_info->ops->set_aie_state) + return -EOPNOTSUPP; + + XDNA_DBG(xdna, "Request parameter %u", args->param); + mutex_lock(&xdna->dev_lock); + ret = xdna->dev_info->ops->set_aie_state(client, args); + mutex_unlock(&xdna->dev_lock); + + return ret; +} + +static int amdxdna_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *drm_filp = filp->private_data; + struct amdxdna_client *client = drm_filp->driver_priv; + struct amdxdna_dev *xdna = client->xdna; + + if (likely(vma->vm_pgoff >= DRM_FILE_PAGE_OFFSET_START)) + return drm_gem_mmap(filp, vma); + + if (!xdna->dev_info->ops->mmap) + return -EOPNOTSUPP; + + return xdna->dev_info->ops->mmap(client, vma); +} + +static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { + /* Context */ + DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0), + /* BO */ + DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0), + /* Execution */ + DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_WAIT_CMD, amdxdna_drm_wait_cmd_ioctl, 0), + /* AIE hardware */ + DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0), + DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY), +}; + +static void amdxdna_show_fdinfo(struct drm_printer *p, struct drm_file *filp) +{ + struct amdxdna_client *client = filp->driver_priv; + size_t heap_usage, external_usage, internal_usage; + char *drv_name = filp->minor->dev->driver->name; + + mutex_lock(&client->mm_lock); + + heap_usage = client->heap_usage; + internal_usage = client->total_int_bo_usage; + external_usage = client->total_bo_usage - internal_usage; + + mutex_unlock(&client->mm_lock); + + /* + * Note for driver specific BO memory usage stat. + * Total memory in use = amdxdna-internal-alloc + amdxdna-external-alloc, which + * includes both imported and created BOs. To avoid double counts, it includes + * HEAP BO, but not DEV BO. DEV BO is counted by amdxdna-heap-alloc. + */ + drm_fdinfo_print_size(p, drv_name, "heap", "alloc", heap_usage); + drm_fdinfo_print_size(p, drv_name, "internal", "alloc", internal_usage); + drm_fdinfo_print_size(p, drv_name, "external", "alloc", external_usage); + /* + * Note for DRM standard BO memory stat. + * drm-total-memory counts both DEV BO and HEAP BO. The DEV BO size is double counted. + * drm-shared-memory counts BO shared with other processes/devices. + */ + drm_show_memory_stats(p, filp); +} + +static const struct file_operations amdxdna_fops = { + .owner = THIS_MODULE, + .open = accel_open, + .release = drm_release, + .unlocked_ioctl = drm_ioctl, + .compat_ioctl = drm_compat_ioctl, + .poll = drm_poll, + .read = drm_read, + .llseek = noop_llseek, + .mmap = amdxdna_drm_gem_mmap, + .show_fdinfo = drm_show_fdinfo, + .fop_flags = FOP_UNSIGNED_OFFSET, +}; + +const struct drm_driver amdxdna_drm_drv = { + .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL | + DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, + .fops = &amdxdna_fops, + .name = "amdxdna_accel_driver", + .desc = "AMD XDNA DRM implementation", + .major = AMDXDNA_DRIVER_MAJOR, + .minor = AMDXDNA_DRIVER_MINOR, + .open = amdxdna_drm_open, + .postclose = amdxdna_drm_close, + .ioctls = amdxdna_drm_ioctls, + .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls), + .show_fdinfo = amdxdna_show_fdinfo, + .gem_create_object = amdxdna_gem_create_shmem_object_cb, + .gem_prime_import = amdxdna_gem_prime_import, +}; + +static const struct amdxdna_dev_info * +amdxdna_get_dev_info(struct pci_dev *pdev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) { + if (pdev->device == amdxdna_ids[i].device && + pdev->revision == amdxdna_ids[i].revision) + return amdxdna_ids[i].dev_info; + } + return NULL; +} + +static void amdxdna_xdna_drm_release(struct drm_device *drm, void *res) +{ + struct amdxdna_dev *xdna = res; + + amdxdna_carveout_fini(xdna); +} + +static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct amdxdna_dev *xdna; + struct drm_device *ddev; + int ret; + + xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev); + if (IS_ERR(xdna)) + return PTR_ERR(xdna); + ddev = &xdna->ddev; + + xdna->dev_info = amdxdna_get_dev_info(pdev); + if (!xdna->dev_info) + return -ENODEV; + + ret = drmm_mutex_init(ddev, &xdna->client_lock); + if (ret) + return ret; + + ret = drmm_mutex_init(ddev, &xdna->dev_lock); + if (ret) + return ret; + + init_rwsem(&xdna->notifier_lock); + INIT_LIST_HEAD(&xdna->client_list); + pci_set_drvdata(pdev, xdna); + + ret = drmm_add_action(ddev, amdxdna_xdna_drm_release, xdna); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_LOCKDEP)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&xdna->notifier_lock); + fs_reclaim_release(GFP_KERNEL); + } + + ret = amdxdna_iommu_init(xdna); + if (ret) + return ret; + + xdna->notifier_wq = drmm_alloc_ordered_workqueue(ddev, "notifier_wq", WQ_MEM_RECLAIM); + if (IS_ERR(xdna->notifier_wq)) { + ret = PTR_ERR(xdna->notifier_wq); + goto iommu_fini; + } + + mutex_lock(&xdna->dev_lock); + ret = xdna->dev_info->ops->init(xdna); + mutex_unlock(&xdna->dev_lock); + if (ret) { + XDNA_ERR(xdna, "Hardware init failed, ret %d", ret); + goto iommu_fini; + } + + ret = amdxdna_sysfs_init(xdna); + if (ret) { + XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret); + goto failed_dev_fini; + } + + ret = drm_dev_register(ddev, 0); + if (ret) { + XDNA_ERR(xdna, "DRM register failed, ret %d", ret); + goto failed_sysfs_fini; + } + + amdxdna_debugfs_init(xdna); + return 0; + +failed_sysfs_fini: + amdxdna_sysfs_fini(xdna); +failed_dev_fini: + mutex_lock(&xdna->dev_lock); + xdna->dev_info->ops->fini(xdna); + mutex_unlock(&xdna->dev_lock); +iommu_fini: + amdxdna_iommu_fini(xdna); + return ret; +} + +static void amdxdna_remove(struct pci_dev *pdev) +{ + struct amdxdna_dev *xdna = pci_get_drvdata(pdev); + struct amdxdna_client *client; + + drm_dev_unplug(&xdna->ddev); + amdxdna_sysfs_fini(xdna); + + mutex_lock(&xdna->client_lock); + mutex_lock(&xdna->dev_lock); + list_for_each_entry(client, &xdna->client_list, node) { + amdxdna_hwctx_remove_all(client); + amdxdna_sva_fini(client); + } + + xdna->dev_info->ops->fini(xdna); + mutex_unlock(&xdna->dev_lock); + mutex_unlock(&xdna->client_lock); + + amdxdna_iommu_fini(xdna); +} + +static const struct dev_pm_ops amdxdna_pm_ops = { + SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume) + RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL) +}; + +static int amdxdna_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + struct amdxdna_dev *xdna = pci_get_drvdata(pdev); + + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->sriov_configure) + return xdna->dev_info->ops->sriov_configure(xdna, num_vfs); + + return -ENOENT; +} + +static struct pci_driver amdxdna_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = pci_ids, + .probe = amdxdna_probe, + .remove = amdxdna_remove, + .driver.pm = &amdxdna_pm_ops, + .sriov_configure = amdxdna_sriov_configure, +}; + +module_pci_driver(amdxdna_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("AMD_PMF"); +MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); +MODULE_DESCRIPTION("amdxdna driver"); diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h new file mode 100644 index 000000000..a997d27a5 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. + */ + +#ifndef _AMDXDNA_PCI_DRV_H_ +#define _AMDXDNA_PCI_DRV_H_ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_mm.h> +#include <drm/drm_print.h> +#include <linux/iommu.h> +#include <linux/iova.h> +#include <linux/workqueue.h> +#include <linux/xarray.h> + +#define XDNA_INFO(xdna, fmt, args...) drm_info(&(xdna)->ddev, fmt, ##args) +#define XDNA_WARN(xdna, fmt, args...) drm_warn(&(xdna)->ddev, "%s: "fmt, __func__, ##args) +#define XDNA_ERR(xdna, fmt, args...) drm_err(&(xdna)->ddev, "%s: "fmt, __func__, ##args) +#define XDNA_DBG(xdna, fmt, args...) drm_dbg(&(xdna)->ddev, fmt, ##args) +#define XDNA_INFO_ONCE(xdna, fmt, args...) drm_info_once(&(xdna)->ddev, fmt, ##args) + +#define XDNA_MBZ_DBG(xdna, ptr, sz) \ + ({ \ + int __i; \ + int __ret = 0; \ + u8 *__ptr = (u8 *)(ptr); \ + for (__i = 0; __i < (sz); __i++) { \ + if (__ptr[__i]) { \ + XDNA_DBG(xdna, "MBZ check failed"); \ + __ret = -EINVAL; \ + break; \ + } \ + } \ + __ret; \ + }) + +#define to_xdna_dev(drm_dev) \ + ((struct amdxdna_dev *)container_of(drm_dev, struct amdxdna_dev, ddev)) + +extern const struct drm_driver amdxdna_drm_drv; + +struct amdxdna_client; +struct amdxdna_dev; +struct amdxdna_drm_get_info; +struct amdxdna_drm_set_state; +struct amdxdna_gem_obj; +struct amdxdna_hwctx; +struct amdxdna_sched_job; + +/* + * struct amdxdna_dev_ops - Device hardware operation callbacks + */ +struct amdxdna_dev_ops { + int (*init)(struct amdxdna_dev *xdna); + void (*fini)(struct amdxdna_dev *xdna); + int (*resume)(struct amdxdna_dev *xdna); + int (*suspend)(struct amdxdna_dev *xdna); + int (*sriov_configure)(struct amdxdna_dev *xdna, int num_vfs); + int (*mmap)(struct amdxdna_client *client, struct vm_area_struct *vma); + int (*hwctx_init)(struct amdxdna_hwctx *hwctx); + void (*hwctx_fini)(struct amdxdna_hwctx *hwctx); + int (*hwctx_config)(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size); + int (*hwctx_sync_debug_bo)(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl); + int (*hwctx_heap_expand)(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap); + void (*hmm_invalidate)(struct amdxdna_gem_obj *abo, unsigned long cur_seq); + int (*cmd_submit)(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq); + int (*cmd_wait)(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout); + int (*get_aie_info)(struct amdxdna_client *client, struct amdxdna_drm_get_info *args); + int (*set_aie_state)(struct amdxdna_client *client, struct amdxdna_drm_set_state *args); + int (*get_array)(struct amdxdna_client *client, struct amdxdna_drm_get_array *args); + int (*get_dev_revision)(struct amdxdna_dev *xdna, u32 *rev); +}; + +struct amdxdna_fw_feature_tbl { + u64 features; + u32 major; + u32 max_minor; + u32 min_minor; +}; + +/* + * struct amdxdna_dev_info - Device hardware information + * Record device static information, like reg, mbox, PSP, SMU bar index + */ +struct amdxdna_dev_info { + int reg_bar; + int mbox_bar; + int sram_bar; + int psp_bar; + int smu_bar; + int doorbell_bar; + int device_type; + int first_col; + u32 dev_mem_buf_shift; + u64 dev_mem_base; + size_t dev_mem_size; + const char *default_vbnv; + const struct amdxdna_rev_vbnv *rev_vbnv_tbl; + size_t dev_heap_max_size; + const struct amdxdna_dev_priv *dev_priv; + const struct amdxdna_fw_feature_tbl *fw_feature_tbl; + const struct amdxdna_dev_ops *ops; +}; + +struct amdxdna_fw_ver { + u32 major; + u32 minor; + u32 sub; + u32 build; +}; + +struct amdxdna_carveout; + +struct amdxdna_dev { + struct drm_device ddev; + struct amdxdna_dev_hdl *dev_handle; + const struct amdxdna_dev_info *dev_info; + void *xrs_hdl; + + struct mutex dev_lock; /* per device lock */ + struct list_head client_list; + struct mutex client_lock; /* client_list */ + struct amdxdna_fw_ver fw_ver; + struct rw_semaphore notifier_lock; /* for mmu notifier*/ + struct workqueue_struct *notifier_wq; + + struct iommu_group *group; + struct iommu_domain *domain; + struct iova_domain iovad; + /* Accurate board name queried from firmware, or default_vbnv as fallback */ + const char *vbnv; + + struct amdxdna_carveout *carveout; +}; + +/* + * struct amdxdna_device_id - PCI device info + */ +struct amdxdna_device_id { + unsigned short device; + u8 revision; + const struct amdxdna_dev_info *dev_info; +}; + +/* + * struct amdxdna_client - amdxdna client + * A per fd data structure for managing context and other user process stuffs. + */ +struct amdxdna_client { + struct list_head node; + pid_t pid; + struct srcu_struct hwctx_srcu; + struct xarray hwctx_xa; + u32 next_hwctxid; + struct amdxdna_dev *xdna; + struct drm_file *filp; + + struct mutex mm_lock; /* protect memory related */ + struct xarray dev_heap_xa; + struct drm_mm dev_heap_mm; + u32 dev_heap_nid; + size_t total_heap_size; + + struct iommu_sva *sva; + int pasid; + struct mm_struct *mm; + + size_t heap_usage; + size_t total_bo_usage; + size_t total_int_bo_usage; +}; + +#define amdxdna_for_each_hwctx(client, hwctx_id, entry) \ + xa_for_each(&(client)->hwctx_xa, hwctx_id, entry) + +/* Add device info below */ +extern const struct amdxdna_dev_info dev_npu1_info; +extern const struct amdxdna_dev_info dev_npu3_pf_info; +extern const struct amdxdna_dev_info dev_npu3_vf_info; +extern const struct amdxdna_dev_info dev_npu4_info; +extern const struct amdxdna_dev_info dev_npu5_info; +extern const struct amdxdna_dev_info dev_npu6_info; + +int amdxdna_sysfs_init(struct amdxdna_dev *xdna); +void amdxdna_sysfs_fini(struct amdxdna_dev *xdna); + +int amdxdna_iommu_init(struct amdxdna_dev *xdna); +void amdxdna_iommu_fini(struct amdxdna_dev *xdna); +void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr); +void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size, + void *cpu_addr, dma_addr_t dma_addr); +int amdxdna_dma_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo); +void amdxdna_dma_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo); + +static inline bool amdxdna_iova_on(struct amdxdna_dev *xdna) +{ + return !!xdna->domain; +} + +static inline bool amdxdna_pasid_on(struct amdxdna_client *client) +{ + return client->pasid != IOMMU_PASID_INVALID; +} +#endif /* _AMDXDNA_PCI_DRV_H_ */ diff --git a/drivers/accel/amdxdna/amdxdna_pm.c b/drivers/accel/amdxdna/amdxdna_pm.c new file mode 100644 index 000000000..b1fafddd7 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_pm.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_drv.h> +#include <linux/pm_runtime.h> + +#include "amdxdna_pm.h" + +#define AMDXDNA_AUTOSUSPEND_DELAY 5000 /* milliseconds */ + +int amdxdna_pm_suspend(struct device *dev) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); + int ret = -EOPNOTSUPP; + + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->suspend) + ret = xdna->dev_info->ops->suspend(xdna); + + XDNA_DBG(xdna, "Suspend done ret %d", ret); + return ret; +} + +int amdxdna_pm_resume(struct device *dev) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); + int ret = -EOPNOTSUPP; + + guard(mutex)(&xdna->dev_lock); + if (xdna->dev_info->ops->resume) + ret = xdna->dev_info->ops->resume(xdna); + + XDNA_DBG(xdna, "Resume done ret %d", ret); + return ret; +} + +int amdxdna_pm_resume_get(struct amdxdna_dev *xdna) +{ + struct device *dev = xdna->ddev.dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret) { + XDNA_ERR(xdna, "Resume failed: %d", ret); + pm_runtime_set_suspended(dev); + } + + return ret; +} + +void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna) +{ + struct device *dev = xdna->ddev.dev; + + pm_runtime_put_autosuspend(dev); +} + +void amdxdna_pm_init(struct amdxdna_dev *xdna) +{ + struct device *dev = xdna->ddev.dev; + + pm_runtime_set_active(dev); + pm_runtime_set_autosuspend_delay(dev, AMDXDNA_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(dev); + pm_runtime_allow(dev); + pm_runtime_put_autosuspend(dev); +} + +void amdxdna_pm_fini(struct amdxdna_dev *xdna) +{ + struct device *dev = xdna->ddev.dev; + + pm_runtime_get_noresume(dev); + pm_runtime_forbid(dev); +} diff --git a/drivers/accel/amdxdna/amdxdna_pm.h b/drivers/accel/amdxdna/amdxdna_pm.h new file mode 100644 index 000000000..3d26b973e --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_pm.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025, Advanced Micro Devices, Inc. + */ + +#ifndef _AMDXDNA_PM_H_ +#define _AMDXDNA_PM_H_ + +#include "amdxdna_pci_drv.h" + +int amdxdna_pm_suspend(struct device *dev); +int amdxdna_pm_resume(struct device *dev); +int amdxdna_pm_resume_get(struct amdxdna_dev *xdna); +void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna); +void amdxdna_pm_init(struct amdxdna_dev *xdna); +void amdxdna_pm_fini(struct amdxdna_dev *xdna); + +static inline int amdxdna_pm_resume_get_locked(struct amdxdna_dev *xdna) +{ + int ret; + + mutex_unlock(&xdna->dev_lock); + ret = amdxdna_pm_resume_get(xdna); + mutex_lock(&xdna->dev_lock); + + return ret; +} + +#endif /* _AMDXDNA_PM_H_ */ diff --git a/drivers/accel/amdxdna/amdxdna_sysfs.c b/drivers/accel/amdxdna/amdxdna_sysfs.c new file mode 100644 index 000000000..d9e359ee8 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_sysfs.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_print.h> +#include <drm/gpu_scheduler.h> +#include <linux/types.h> + +#include "amdxdna_gem.h" +#include "amdxdna_pci_drv.h" + +static ssize_t vbnv_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct amdxdna_dev *xdna = dev_get_drvdata(dev); + + if (!xdna->vbnv) + return sprintf(buf, "\n"); + + return sprintf(buf, "%s\n", xdna->vbnv); +} +static DEVICE_ATTR_RO(vbnv); + +static ssize_t device_type_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct amdxdna_dev *xdna = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", xdna->dev_info->device_type); +} +static DEVICE_ATTR_RO(device_type); + +static ssize_t fw_version_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct amdxdna_dev *xdna = dev_get_drvdata(dev); + + return sprintf(buf, "%d.%d.%d.%d\n", xdna->fw_ver.major, + xdna->fw_ver.minor, xdna->fw_ver.sub, + xdna->fw_ver.build); +} +static DEVICE_ATTR_RO(fw_version); + +static struct attribute *amdxdna_attrs[] = { + &dev_attr_device_type.attr, + &dev_attr_vbnv.attr, + &dev_attr_fw_version.attr, + NULL, +}; + +static struct attribute_group amdxdna_attr_group = { + .attrs = amdxdna_attrs, +}; + +int amdxdna_sysfs_init(struct amdxdna_dev *xdna) +{ + int ret; + + ret = sysfs_create_group(&xdna->ddev.dev->kobj, &amdxdna_attr_group); + if (ret) + XDNA_ERR(xdna, "Create attr group failed"); + + return ret; +} + +void amdxdna_sysfs_fini(struct amdxdna_dev *xdna) +{ + sysfs_remove_group(&xdna->ddev.dev->kobj, &amdxdna_attr_group); +} diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c new file mode 100644 index 000000000..0e0cd69cd --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/drm_print.h> +#include <linux/dma-buf.h> +#include <linux/overflow.h> +#include <linux/pagemap.h> +#include <linux/vmalloc.h> + +#include "amdxdna_pci_drv.h" +#include "amdxdna_ubuf.h" + +struct amdxdna_ubuf_priv { + struct page **pages; + u64 nr_pages; + struct mm_struct *mm; +}; + +static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, + enum dma_data_direction direction) +{ + struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv; + struct sg_table *sg; + int ret; + + sg = kzalloc_obj(*sg); + if (!sg) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, + ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); + if (ret) + goto err_free_sg; + + ret = dma_map_sgtable(attach->dev, sg, direction, 0); + if (ret) + goto err_free_table; + + return sg; + +err_free_table: + sg_free_table(sg); +err_free_sg: + kfree(sg); + return ERR_PTR(ret); +} + +static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, + struct sg_table *sg, + enum dma_data_direction direction) +{ + dma_unmap_sgtable(attach->dev, sg, direction, 0); + sg_free_table(sg); + kfree(sg); +} + +static void amdxdna_ubuf_release(struct dma_buf *dbuf) +{ + struct amdxdna_ubuf_priv *ubuf = dbuf->priv; + + unpin_user_pages(ubuf->pages, ubuf->nr_pages); + kvfree(ubuf->pages); + atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm); + mmdrop(ubuf->mm); + kfree(ubuf); +} + +static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = { + .map_dma_buf = amdxdna_ubuf_map, + .unmap_dma_buf = amdxdna_ubuf_unmap, + .release = amdxdna_ubuf_release, +}; + +struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, + u32 num_entries, void __user *va_entries) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + unsigned long lock_limit, new_pinned; + struct amdxdna_drm_va_entry *va_ent; + struct amdxdna_ubuf_priv *ubuf; + u32 npages, start = 0; + struct dma_buf *dbuf; + int i, ret; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + if (!can_do_mlock()) + return ERR_PTR(-EPERM); + + ubuf = kzalloc_obj(*ubuf); + if (!ubuf) + return ERR_PTR(-ENOMEM); + + ubuf->mm = current->mm; + mmgrab(ubuf->mm); + + va_ent = kvzalloc_objs(*va_ent, num_entries); + if (!va_ent) { + ret = -ENOMEM; + goto free_ubuf; + } + + if (copy_from_user(va_ent, va_entries, sizeof(*va_ent) * num_entries)) { + XDNA_DBG(xdna, "Access va entries failed"); + ret = -EINVAL; + goto free_ent; + } + + for (i = 0, exp_info.size = 0; i < num_entries; i++) { + if (!IS_ALIGNED(va_ent[i].vaddr, PAGE_SIZE) || + !IS_ALIGNED(va_ent[i].len, PAGE_SIZE)) { + XDNA_ERR(xdna, "Invalid address or len %llx, %llx", + va_ent[i].vaddr, va_ent[i].len); + ret = -EINVAL; + goto free_ent; + } + + if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) { + ret = -EINVAL; + goto free_ent; + } + } + + ubuf->nr_pages = exp_info.size >> PAGE_SHIFT; + lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + new_pinned = atomic64_add_return(ubuf->nr_pages, &ubuf->mm->pinned_vm); + if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) { + XDNA_DBG(xdna, "New pin %ld, limit %ld, cap %d", + new_pinned, lock_limit, capable(CAP_IPC_LOCK)); + ret = -ENOMEM; + goto sub_pin_cnt; + } + + ubuf->pages = kvmalloc_objs(*ubuf->pages, ubuf->nr_pages); + if (!ubuf->pages) { + ret = -ENOMEM; + goto sub_pin_cnt; + } + + for (i = 0; i < num_entries; i++) { + npages = va_ent[i].len >> PAGE_SHIFT; + + ret = pin_user_pages_fast(va_ent[i].vaddr, npages, + FOLL_WRITE | FOLL_LONGTERM, + &ubuf->pages[start]); + if (ret >= 0) { + start += ret; + if (ret != npages) { + XDNA_ERR(xdna, "Partially pinned pages %d/%u", ret, npages); + ret = -ENOMEM; + goto destroy_pages; + } + } else { + XDNA_ERR(xdna, "Failed to pin pages ret %d", ret); + goto destroy_pages; + } + } + + exp_info.ops = &amdxdna_ubuf_dmabuf_ops; + exp_info.priv = ubuf; + exp_info.flags = O_RDWR | O_CLOEXEC; + + dbuf = dma_buf_export(&exp_info); + if (IS_ERR(dbuf)) { + ret = PTR_ERR(dbuf); + goto destroy_pages; + } + kvfree(va_ent); + + return dbuf; + +destroy_pages: + if (start) + unpin_user_pages(ubuf->pages, start); + kvfree(ubuf->pages); +sub_pin_cnt: + atomic64_sub(ubuf->nr_pages, &ubuf->mm->pinned_vm); +free_ent: + kvfree(va_ent); +free_ubuf: + mmdrop(ubuf->mm); + kfree(ubuf); + return ERR_PTR(ret); +} diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.h b/drivers/accel/amdxdna/amdxdna_ubuf.h new file mode 100644 index 000000000..8900a6dc4 --- /dev/null +++ b/drivers/accel/amdxdna/amdxdna_ubuf.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025, Advanced Micro Devices, Inc. + */ +#ifndef _AMDXDNA_UBUF_H_ +#define _AMDXDNA_UBUF_H_ + +#include <drm/drm_device.h> +#include <linux/dma-buf.h> + +struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, + u32 num_entries, void __user *va_entries); + +#endif /* _AMDXDNA_UBUF_H_ */ diff --git a/drivers/accel/amdxdna/npu1_regs.c b/drivers/accel/amdxdna/npu1_regs.c new file mode 100644 index 000000000..ca7796740 --- /dev/null +++ b/drivers/accel/amdxdna/npu1_regs.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/gpu_scheduler.h> +#include <linux/bits.h> +#include <linux/sizes.h> + +#include "aie2_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +/* Address definition from NPU1 docs */ +#define MPNPU_PWAITMODE 0x3010034 +#define MPNPU_PUB_SEC_INTR 0x3010090 +#define MPNPU_PUB_PWRMGMT_INTR 0x3010094 +#define MPNPU_PUB_SCRATCH2 0x30100A0 +#define MPNPU_PUB_SCRATCH3 0x30100A4 +#define MPNPU_PUB_SCRATCH4 0x30100A8 +#define MPNPU_PUB_SCRATCH5 0x30100AC +#define MPNPU_PUB_SCRATCH6 0x30100B0 +#define MPNPU_PUB_SCRATCH7 0x30100B4 +#define MPNPU_PUB_SCRATCH9 0x30100BC + +#define MPNPU_SRAM_X2I_MAILBOX_0 0x30A0000 +#define MPNPU_SRAM_X2I_MAILBOX_1 0x30A2000 +#define MPNPU_SRAM_I2X_MAILBOX_15 0x30BF000 + +#define MPNPU_APERTURE0_BASE 0x3000000 +#define MPNPU_APERTURE1_BASE 0x3080000 +#define MPNPU_APERTURE2_BASE 0x30C0000 + +/* PCIe BAR Index for NPU1 */ +#define NPU1_REG_BAR_INDEX 0 +#define NPU1_MBOX_BAR_INDEX 4 +#define NPU1_PSP_BAR_INDEX 0 +#define NPU1_SMU_BAR_INDEX 0 +#define NPU1_SRAM_BAR_INDEX 2 +/* Associated BARs and Apertures */ +#define NPU1_REG_BAR_BASE MPNPU_APERTURE0_BASE +#define NPU1_MBOX_BAR_BASE MPNPU_APERTURE2_BASE +#define NPU1_PSP_BAR_BASE MPNPU_APERTURE0_BASE +#define NPU1_SMU_BAR_BASE MPNPU_APERTURE0_BASE +#define NPU1_SRAM_BAR_BASE MPNPU_APERTURE1_BASE + +const struct rt_config npu1_default_rt_cfg[] = { + { 2, 1, AIE2_RT_CFG_INIT }, /* PDI APP LOAD MODE */ + { 4, 1, AIE2_RT_CFG_INIT }, /* Debug BO */ + { 1, 1, AIE2_RT_CFG_CLK_GATING }, /* Clock gating on */ + { 0 }, +}; + +const struct dpm_clk_freq npu1_dpm_clk_table[] = { + {400, 800}, + {600, 1024}, + {600, 1024}, + {600, 1024}, + {600, 1024}, + {720, 1309}, + {720, 1309}, + {847, 1600}, + { 0 } +}; + +static const struct amdxdna_fw_feature_tbl npu1_fw_feature_table[] = { + { .major = 5, .min_minor = 7 }, + { .features = BIT_U64(AIE2_NPU_COMMAND), .major = 5, .min_minor = 8 }, + { 0 } +}; + +static int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) +{ + u32 npuclk, hclk; + int ret; + + npuclk = ndev->priv->dpm_clk_tbl[dpm_level].npuclk; + hclk = ndev->priv->dpm_clk_tbl[dpm_level].hclk; + ret = aie_smu_set_clocks(ndev->aie.smu_hdl, &npuclk, &hclk); + if (ret) + return ret; + + ndev->npuclk_freq = npuclk; + ndev->hclk_freq = hclk; + ndev->max_tops = 2 * ndev->total_col; + ndev->curr_tops = ndev->max_tops * hclk / 1028; + + XDNA_DBG(ndev->aie.xdna, "MP-NPU clock %d, H clock %d\n", + ndev->npuclk_freq, ndev->hclk_freq); + return 0; +} + +static const struct amdxdna_dev_priv npu1_dev_priv = { + .fw_path = "amdnpu/1502_00/", + .rt_config = npu1_default_rt_cfg, + .dpm_clk_tbl = npu1_dpm_clk_table, + .col_align = COL_ALIGN_NONE, + .col_opc = 2048, + .mbox_dev_addr = NPU1_MBOX_BAR_BASE, + .mbox_size = 0, /* Use BAR size */ + .sram_dev_addr = NPU1_SRAM_BAR_BASE, + .hwctx_limit = 6, + .sram_offs = { + DEFINE_BAR_OFFSET(MBOX_CHANN_OFF, NPU1_SRAM, MPNPU_SRAM_X2I_MAILBOX_0), + DEFINE_BAR_OFFSET(FW_ALIVE_OFF, NPU1_SRAM, MPNPU_SRAM_I2X_MAILBOX_15), + }, + .psp_regs_off = { + DEFINE_BAR_OFFSET(PSP_CMD_REG, NPU1_PSP, MPNPU_PUB_SCRATCH2), + DEFINE_BAR_OFFSET(PSP_ARG0_REG, NPU1_PSP, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_ARG1_REG, NPU1_PSP, MPNPU_PUB_SCRATCH4), + DEFINE_BAR_OFFSET(PSP_ARG2_REG, NPU1_PSP, MPNPU_PUB_SCRATCH9), + DEFINE_BAR_OFFSET(PSP_INTR_REG, NPU1_PSP, MPNPU_PUB_SEC_INTR), + DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU1_PSP, MPNPU_PUB_SCRATCH2), + DEFINE_BAR_OFFSET(PSP_RESP_REG, NPU1_PSP, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU1_PSP, MPNPU_PWAITMODE), + }, + .smu_regs_off = { + DEFINE_BAR_OFFSET(SMU_CMD_REG, NPU1_SMU, MPNPU_PUB_SCRATCH5), + DEFINE_BAR_OFFSET(SMU_ARG_REG, NPU1_SMU, MPNPU_PUB_SCRATCH7), + DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU1_SMU, MPNPU_PUB_PWRMGMT_INTR), + DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU1_SMU, MPNPU_PUB_SCRATCH6), + DEFINE_BAR_OFFSET(SMU_OUT_REG, NPU1_SMU, MPNPU_PUB_SCRATCH7), + }, + .hw_ops = &(const struct aie2_hw_ops) { + .set_dpm = npu1_set_dpm, + }, +}; + +const struct amdxdna_dev_info dev_npu1_info = { + .reg_bar = NPU1_REG_BAR_INDEX, + .mbox_bar = NPU1_MBOX_BAR_INDEX, + .sram_bar = NPU1_SRAM_BAR_INDEX, + .psp_bar = NPU1_PSP_BAR_INDEX, + .smu_bar = NPU1_SMU_BAR_INDEX, + .first_col = 1, + .dev_mem_buf_shift = 15, /* 32 KiB aligned */ + .dev_mem_base = AIE2_DEVM_BASE, + .dev_mem_size = AIE2_DEVM_SIZE, + .default_vbnv = "RyzenAI-npu1", + .dev_heap_max_size = AIE2_DEVM_SIZE, + .device_type = AMDXDNA_DEV_TYPE_KMQ, + .dev_priv = &npu1_dev_priv, + .fw_feature_tbl = npu1_fw_feature_table, + .ops = &aie2_ops, +}; diff --git a/drivers/accel/amdxdna/npu3_regs.c b/drivers/accel/amdxdna/npu3_regs.c new file mode 100644 index 000000000..d76b2e99c --- /dev/null +++ b/drivers/accel/amdxdna/npu3_regs.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> + +#include "aie4_pci.h" +#include "amdxdna_pci_drv.h" + +#define NPU3_MBOX_BAR 0 + +#define NPU3_MBOX_BUFFER_BAR 2 +#define NPU3_MBOX_INFO_OFF 0x0 + +#define NPU3_DOORBELL_BAR 2 +#define NPU3_DOORBELL_OFF 0x0 + +/* PCIe BAR Index for NPU3 */ +#define NPU3_REG_BAR_INDEX 0 +#define NPU3_PSP_BAR_INDEX 4 +#define NPU3_SMU_BAR_INDEX 5 + +#define MMNPU_APERTURE3_BASE 0x3810000 +#define MMNPU_APERTURE4_BASE 0x3B10000 + +#define NPU3_PSP_BAR_BASE MMNPU_APERTURE3_BASE +#define NPU3_SMU_BAR_BASE MMNPU_APERTURE4_BASE + +#define MPASP_C2PMSG_123_ALT_1 0x3810AEC +#define MPASP_C2PMSG_156_ALT_1 0x3810B70 +#define MPASP_C2PMSG_157_ALT_1 0x3810B74 +#define MPASP_C2PMSG_73_ALT_1 0x3810A24 + +#define MP1_C2PMSG_59_ALT_1 0x3B109EC +#define MP1_C2PMSG_61_ALT_1 0x3B109F4 +#define MP1_C2PMSG_60_ALT_1 0x3B109F0 + +static const struct amdxdna_fw_feature_tbl npu3_fw_feature_table[] = { + { .major = 5, .min_minor = 10 }, + { 0 } +}; + +static const struct amdxdna_dev_priv npu3_dev_priv = { + .npufw_path = "npu.dev.sbin", + .certfw_path = "cert.dev.sbin", + .mbox_bar = NPU3_MBOX_BAR, + .mbox_rbuf_bar = NPU3_MBOX_BUFFER_BAR, + .mbox_info_off = NPU3_MBOX_INFO_OFF, + .doorbell_off = NPU3_DOORBELL_OFF, + .psp_regs_off = { + DEFINE_BAR_OFFSET(PSP_CMD_REG, NPU3_PSP, MPASP_C2PMSG_123_ALT_1), + DEFINE_BAR_OFFSET(PSP_ARG0_REG, NPU3_PSP, MPASP_C2PMSG_156_ALT_1), + DEFINE_BAR_OFFSET(PSP_ARG1_REG, NPU3_PSP, MPASP_C2PMSG_157_ALT_1), + DEFINE_BAR_OFFSET(PSP_ARG2_REG, NPU3_PSP, MPASP_C2PMSG_123_ALT_1), + DEFINE_BAR_OFFSET(PSP_INTR_REG, NPU3_PSP, MPASP_C2PMSG_73_ALT_1), + DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU3_PSP, MPASP_C2PMSG_123_ALT_1), + DEFINE_BAR_OFFSET(PSP_RESP_REG, NPU3_PSP, MPASP_C2PMSG_156_ALT_1), + /* npu3 doesn't use 8th pwaitmode register */ + }, + .smu_regs_off = { + DEFINE_BAR_OFFSET(SMU_CMD_REG, NPU3_SMU, MP1_C2PMSG_59_ALT_1), + DEFINE_BAR_OFFSET(SMU_ARG_REG, NPU3_SMU, MP1_C2PMSG_61_ALT_1), + DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU3_SMU, MMNPU_APERTURE4_BASE), + DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU3_SMU, MP1_C2PMSG_60_ALT_1), + DEFINE_BAR_OFFSET(SMU_OUT_REG, NPU3_SMU, MP1_C2PMSG_61_ALT_1), + }, +}; + +static const struct amdxdna_dev_priv npu3_dev_vf_priv = { + /* vf device does not load firmware */ + .mbox_bar = NPU3_MBOX_BAR, + .mbox_rbuf_bar = NPU3_MBOX_BUFFER_BAR, + .mbox_info_off = NPU3_MBOX_INFO_OFF, + /* vf device does not have smu and psp */ +}; + +const struct amdxdna_dev_info dev_npu3_pf_info = { + .mbox_bar = NPU3_MBOX_BAR, + .sram_bar = NPU3_MBOX_BUFFER_BAR, + .psp_bar = NPU3_PSP_BAR_INDEX, + .smu_bar = NPU3_SMU_BAR_INDEX, + .default_vbnv = "RyzenAI-npu3-pf", + .device_type = AMDXDNA_DEV_TYPE_PF, + .dev_priv = &npu3_dev_priv, + .fw_feature_tbl = npu3_fw_feature_table, + .ops = &aie4_pf_ops, +}; + +const struct amdxdna_dev_info dev_npu3_vf_info = { + .mbox_bar = NPU3_MBOX_BAR, + .sram_bar = NPU3_MBOX_BUFFER_BAR, + .doorbell_bar = NPU3_DOORBELL_BAR, + .default_vbnv = "RyzenAI-npu3-vf", + .device_type = AMDXDNA_DEV_TYPE_UMQ, + .dev_priv = &npu3_dev_vf_priv, + .fw_feature_tbl = npu3_fw_feature_table, + .ops = &aie4_vf_ops, +}; diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c new file mode 100644 index 000000000..15a161384 --- /dev/null +++ b/drivers/accel/amdxdna/npu4_regs.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/gpu_scheduler.h> +#include <linux/amd-pmf-io.h> +#include <linux/bits.h> +#include <linux/sizes.h> + +#include "aie2_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */ +#define MPNPU_PWAITMODE 0x301003C +#define MPNPU_PUB_SEC_INTR 0x3010060 +#define MPNPU_PUB_PWRMGMT_INTR 0x3010064 +#define MPNPU_PUB_SCRATCH0 0x301006C +#define MPNPU_PUB_SCRATCH1 0x3010070 +#define MPNPU_PUB_SCRATCH2 0x3010074 +#define MPNPU_PUB_SCRATCH3 0x3010078 +#define MPNPU_PUB_SCRATCH4 0x301007C +#define MPNPU_PUB_SCRATCH5 0x3010080 +#define MPNPU_PUB_SCRATCH6 0x3010084 +#define MPNPU_PUB_SCRATCH7 0x3010088 +#define MPNPU_PUB_SCRATCH8 0x301008C +#define MPNPU_PUB_SCRATCH9 0x3010090 +#define MPNPU_PUB_SCRATCH10 0x3010094 +#define MPNPU_PUB_SCRATCH11 0x3010098 +#define MPNPU_PUB_SCRATCH12 0x301009C +#define MPNPU_PUB_SCRATCH13 0x30100A0 +#define MPNPU_PUB_SCRATCH14 0x30100A4 +#define MPNPU_PUB_SCRATCH15 0x30100A8 +#define MP0_C2PMSG_73 0x3810A24 +#define MP0_C2PMSG_123 0x3810AEC + +#define MP1_C2PMSG_0 0x3B10900 +#define MP1_C2PMSG_60 0x3B109F0 +#define MP1_C2PMSG_61 0x3B109F4 + +#define MPNPU_SRAM_X2I_MAILBOX_0 0x3600000 +#define MPNPU_SRAM_X2I_MAILBOX_15 0x361E000 +#define MPNPU_SRAM_X2I_MAILBOX_31 0x363E000 +#define MPNPU_SRAM_I2X_MAILBOX_31 0x363F000 + +#define MMNPU_APERTURE0_BASE 0x3000000 +#define MMNPU_APERTURE1_BASE 0x3600000 +#define MMNPU_APERTURE3_BASE 0x3810000 +#define MMNPU_APERTURE4_BASE 0x3B10000 + +/* PCIe BAR Index for NPU4 */ +#define NPU4_REG_BAR_INDEX 0 +#define NPU4_MBOX_BAR_INDEX 0 +#define NPU4_PSP_BAR_INDEX 4 +#define NPU4_SMU_BAR_INDEX 5 +#define NPU4_SRAM_BAR_INDEX 2 +/* Associated BARs and Apertures */ +#define NPU4_REG_BAR_BASE MMNPU_APERTURE0_BASE +#define NPU4_MBOX_BAR_BASE MMNPU_APERTURE0_BASE +#define NPU4_PSP_BAR_BASE MMNPU_APERTURE3_BASE +#define NPU4_SMU_BAR_BASE MMNPU_APERTURE4_BASE +#define NPU4_SRAM_BAR_BASE MMNPU_APERTURE1_BASE + +#define NPU4_DPM_TOPS(ndev, hclk) (4096 * (ndev)->total_col * (hclk) / 1000000) + +const struct rt_config npu4_default_rt_cfg[] = { + { 5, 1, AIE2_RT_CFG_INIT }, /* PDI APP LOAD MODE */ + { 10, 1, AIE2_RT_CFG_INIT }, /* DEBUG BUF */ + { 14, 0, AIE2_RT_CFG_INIT, BIT_U64(AIE2_PREEMPT) }, /* Frame boundary preemption */ + { 1, 1, AIE2_RT_CFG_CLK_GATING }, /* Clock gating on */ + { 2, 1, AIE2_RT_CFG_CLK_GATING }, /* Clock gating on */ + { 3, 1, AIE2_RT_CFG_CLK_GATING }, /* Clock gating on */ + { 4, 1, AIE2_RT_CFG_CLK_GATING }, /* Clock gating on */ + { 13, 0, AIE2_RT_CFG_FORCE_PREEMPT }, + { 14, 0, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT }, + { 0 }, +}; + +const struct dpm_clk_freq npu4_dpm_clk_table[] = { + {396, 792}, + {600, 1056}, + {792, 1152}, + {975, 1267}, + {975, 1267}, + {1056, 1408}, + {1152, 1584}, + {1267, 1800}, + { 0 } +}; + +const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[] = { + { .major = 6, .min_minor = 12 }, + { .features = BIT_U64(AIE2_PREEMPT), .major = 6, .min_minor = 12 }, + { .features = BIT_U64(AIE2_TEMPORAL_ONLY), .major = 6, .min_minor = 12 }, + { .features = BIT_U64(AIE2_NPU_COMMAND), .major = 6, .min_minor = 15 }, + { .features = BIT_U64(AIE2_UPDATE_PROPERTY), .major = 6, .min_minor = 15 }, + { .features = BIT_U64(AIE2_APP_HEALTH), .major = 6, .min_minor = 18 }, + { .features = BIT_U64(AIE2_ADD_HOST_BUFFER), .major = 6, .min_minor = 18 }, + { .features = BIT_U64(AIE2_GET_DEV_REVISION), .major = 6, .min_minor = 24 }, + { .features = AIE2_ALL_FEATURES, .major = 7 }, + { 0 } +}; + +static int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) +{ + int ret; + + ret = aie_smu_set_dpm(ndev->aie.smu_hdl, dpm_level); + if (ret) + return ret; + + ndev->npuclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].npuclk; + ndev->hclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].hclk; + ndev->max_tops = NPU4_DPM_TOPS(ndev, ndev->priv->dpm_clk_tbl[ndev->max_dpm_level].hclk); + ndev->curr_tops = NPU4_DPM_TOPS(ndev, ndev->hclk_freq); + + XDNA_DBG(ndev->aie.xdna, "MP-NPU clock %d, H clock %d\n", + ndev->npuclk_freq, ndev->hclk_freq); + + return 0; +} + +static int npu4_update_counters(struct amdxdna_dev_hdl *ndev) +{ + struct amd_pmf_npu_metrics npu_metrics; + int ret; + + ret = AIE2_GET_PMF_NPU_METRICS(&npu_metrics); + if (ret) + return ret; + + ndev->npuclk_freq = npu_metrics.mpnpuclk_freq; + ndev->hclk_freq = npu_metrics.npuclk_freq; + ndev->curr_tops = NPU4_DPM_TOPS(ndev, ndev->hclk_freq); + + return 0; +} + +const struct aie2_hw_ops npu4_hw_ops = { + .set_dpm = npu4_set_dpm, + .update_counters = npu4_update_counters, +}; + +const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[] = { + { AIE2_DEV_REVISION_STXA, "NPU Strix" }, + { AIE2_DEV_REVISION_STXB, "NPU Strix" }, + { AIE2_DEV_REVISION_KRK1, "NPU Krackan 1" }, + { AIE2_DEV_REVISION_KRK2, "NPU Krackan 2" }, + { AIE2_DEV_REVISION_HALO, "NPU Strix Halo" }, + { AIE2_DEV_REVISION_GPT1, "NPU Gorgon Point 1" }, + { AIE2_DEV_REVISION_GPT2, "NPU Gorgon Point 2" }, + { AIE2_DEV_REVISION_GPT3, "NPU Gorgon Point 3" }, + { 0 } +}; + +static const struct amdxdna_dev_priv npu4_dev_priv = { + .fw_path = "amdnpu/17f0_10/", + .rt_config = npu4_default_rt_cfg, + .dpm_clk_tbl = npu4_dpm_clk_table, + .col_align = COL_ALIGN_NATURE, + .col_opc = 4096, + .mbox_dev_addr = NPU4_MBOX_BAR_BASE, + .mbox_size = 0, /* Use BAR size */ + .sram_dev_addr = NPU4_SRAM_BAR_BASE, + .hwctx_limit = 16, + .sram_offs = { + DEFINE_BAR_OFFSET(MBOX_CHANN_OFF, NPU4_SRAM, MPNPU_SRAM_X2I_MAILBOX_0), + DEFINE_BAR_OFFSET(FW_ALIVE_OFF, NPU4_SRAM, MPNPU_SRAM_X2I_MAILBOX_15), + }, + .psp_regs_off = { + DEFINE_BAR_OFFSET(PSP_CMD_REG, NPU4_PSP, MP0_C2PMSG_123), + DEFINE_BAR_OFFSET(PSP_ARG0_REG, NPU4_REG, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_ARG1_REG, NPU4_REG, MPNPU_PUB_SCRATCH4), + DEFINE_BAR_OFFSET(PSP_ARG2_REG, NPU4_REG, MPNPU_PUB_SCRATCH9), + DEFINE_BAR_OFFSET(PSP_INTR_REG, NPU4_PSP, MP0_C2PMSG_73), + DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU4_PSP, MP0_C2PMSG_123), + DEFINE_BAR_OFFSET(PSP_RESP_REG, NPU4_REG, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU4_REG, MPNPU_PWAITMODE), + }, + .smu_regs_off = { + DEFINE_BAR_OFFSET(SMU_CMD_REG, NPU4_SMU, MP1_C2PMSG_0), + DEFINE_BAR_OFFSET(SMU_ARG_REG, NPU4_SMU, MP1_C2PMSG_60), + DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU4_SMU, MMNPU_APERTURE4_BASE), + DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU4_SMU, MP1_C2PMSG_61), + DEFINE_BAR_OFFSET(SMU_OUT_REG, NPU4_SMU, MP1_C2PMSG_60), + }, + .hw_ops = &npu4_hw_ops +}; + +const struct amdxdna_dev_info dev_npu4_info = { + .reg_bar = NPU4_REG_BAR_INDEX, + .mbox_bar = NPU4_MBOX_BAR_INDEX, + .sram_bar = NPU4_SRAM_BAR_INDEX, + .psp_bar = NPU4_PSP_BAR_INDEX, + .smu_bar = NPU4_SMU_BAR_INDEX, + .first_col = 0, + .dev_mem_buf_shift = 15, /* 32 KiB aligned */ + .dev_mem_base = AIE2_DEVM_BASE, + .dev_mem_size = AIE2_DEVM_SIZE, + .default_vbnv = "RyzenAI-npu4", + .dev_heap_max_size = AIE2_DEVM_MAX_SIZE, + .device_type = AMDXDNA_DEV_TYPE_KMQ, + .rev_vbnv_tbl = npu4_rev_vbnv_tbl, + .dev_priv = &npu4_dev_priv, + .fw_feature_tbl = npu4_fw_feature_table, + .ops = &aie2_ops, /* NPU4 can share NPU1's callback */ +}; diff --git a/drivers/accel/amdxdna/npu5_regs.c b/drivers/accel/amdxdna/npu5_regs.c new file mode 100644 index 000000000..306b359d0 --- /dev/null +++ b/drivers/accel/amdxdna/npu5_regs.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/gpu_scheduler.h> +#include <linux/sizes.h> + +#include "aie2_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */ +#define MPNPU_PWAITMODE 0x301003C +#define MPNPU_PUB_SEC_INTR 0x3010060 +#define MPNPU_PUB_PWRMGMT_INTR 0x3010064 +#define MPNPU_PUB_SCRATCH0 0x301006C +#define MPNPU_PUB_SCRATCH1 0x3010070 +#define MPNPU_PUB_SCRATCH2 0x3010074 +#define MPNPU_PUB_SCRATCH3 0x3010078 +#define MPNPU_PUB_SCRATCH4 0x301007C +#define MPNPU_PUB_SCRATCH5 0x3010080 +#define MPNPU_PUB_SCRATCH6 0x3010084 +#define MPNPU_PUB_SCRATCH7 0x3010088 +#define MPNPU_PUB_SCRATCH8 0x301008C +#define MPNPU_PUB_SCRATCH9 0x3010090 +#define MPNPU_PUB_SCRATCH10 0x3010094 +#define MPNPU_PUB_SCRATCH11 0x3010098 +#define MPNPU_PUB_SCRATCH12 0x301009C +#define MPNPU_PUB_SCRATCH13 0x30100A0 +#define MPNPU_PUB_SCRATCH14 0x30100A4 +#define MPNPU_PUB_SCRATCH15 0x30100A8 +#define MP0_C2PMSG_73 0x3810A24 +#define MP0_C2PMSG_123 0x3810AEC + +#define MP1_C2PMSG_0 0x3B10900 +#define MP1_C2PMSG_60 0x3B109F0 +#define MP1_C2PMSG_61 0x3B109F4 + +#define MPNPU_SRAM_X2I_MAILBOX_0 0x3600000 +#define MPNPU_SRAM_X2I_MAILBOX_15 0x361E000 +#define MPNPU_SRAM_X2I_MAILBOX_31 0x363E000 +#define MPNPU_SRAM_I2X_MAILBOX_31 0x363F000 + +#define MMNPU_APERTURE0_BASE 0x3000000 +#define MMNPU_APERTURE1_BASE 0x3600000 +#define MMNPU_APERTURE3_BASE 0x3810000 +#define MMNPU_APERTURE4_BASE 0x3B10000 + +/* PCIe BAR Index for NPU5 */ +#define NPU5_REG_BAR_INDEX 0 +#define NPU5_MBOX_BAR_INDEX 0 +#define NPU5_PSP_BAR_INDEX 4 +#define NPU5_SMU_BAR_INDEX 5 +#define NPU5_SRAM_BAR_INDEX 2 +/* Associated BARs and Apertures */ +#define NPU5_REG_BAR_BASE MMNPU_APERTURE0_BASE +#define NPU5_MBOX_BAR_BASE MMNPU_APERTURE0_BASE +#define NPU5_PSP_BAR_BASE MMNPU_APERTURE3_BASE +#define NPU5_SMU_BAR_BASE MMNPU_APERTURE4_BASE +#define NPU5_SRAM_BAR_BASE MMNPU_APERTURE1_BASE + +static const struct amdxdna_dev_priv npu5_dev_priv = { + .fw_path = "amdnpu/17f0_11/", + .rt_config = npu4_default_rt_cfg, + .dpm_clk_tbl = npu4_dpm_clk_table, + .col_align = COL_ALIGN_NATURE, + .col_opc = 4096, + .mbox_dev_addr = NPU5_MBOX_BAR_BASE, + .mbox_size = 0, /* Use BAR size */ + .sram_dev_addr = NPU5_SRAM_BAR_BASE, + .hwctx_limit = 16, + .sram_offs = { + DEFINE_BAR_OFFSET(MBOX_CHANN_OFF, NPU5_SRAM, MPNPU_SRAM_X2I_MAILBOX_0), + DEFINE_BAR_OFFSET(FW_ALIVE_OFF, NPU5_SRAM, MPNPU_SRAM_X2I_MAILBOX_15), + }, + .psp_regs_off = { + DEFINE_BAR_OFFSET(PSP_CMD_REG, NPU5_PSP, MP0_C2PMSG_123), + DEFINE_BAR_OFFSET(PSP_ARG0_REG, NPU5_REG, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_ARG1_REG, NPU5_REG, MPNPU_PUB_SCRATCH4), + DEFINE_BAR_OFFSET(PSP_ARG2_REG, NPU5_REG, MPNPU_PUB_SCRATCH9), + DEFINE_BAR_OFFSET(PSP_INTR_REG, NPU5_PSP, MP0_C2PMSG_73), + DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU5_PSP, MP0_C2PMSG_123), + DEFINE_BAR_OFFSET(PSP_RESP_REG, NPU5_REG, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU5_REG, MPNPU_PWAITMODE), + }, + .smu_regs_off = { + DEFINE_BAR_OFFSET(SMU_CMD_REG, NPU5_SMU, MP1_C2PMSG_0), + DEFINE_BAR_OFFSET(SMU_ARG_REG, NPU5_SMU, MP1_C2PMSG_60), + DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU5_SMU, MMNPU_APERTURE4_BASE), + DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU5_SMU, MP1_C2PMSG_61), + DEFINE_BAR_OFFSET(SMU_OUT_REG, NPU5_SMU, MP1_C2PMSG_60), + }, + .hw_ops = &npu4_hw_ops +}; + +const struct amdxdna_dev_info dev_npu5_info = { + .reg_bar = NPU5_REG_BAR_INDEX, + .mbox_bar = NPU5_MBOX_BAR_INDEX, + .sram_bar = NPU5_SRAM_BAR_INDEX, + .psp_bar = NPU5_PSP_BAR_INDEX, + .smu_bar = NPU5_SMU_BAR_INDEX, + .first_col = 0, + .dev_mem_buf_shift = 15, /* 32 KiB aligned */ + .dev_mem_base = AIE2_DEVM_BASE, + .dev_mem_size = AIE2_DEVM_SIZE, + .default_vbnv = "RyzenAI-npu5", + .dev_heap_max_size = AIE2_DEVM_MAX_SIZE, + .device_type = AMDXDNA_DEV_TYPE_KMQ, + .rev_vbnv_tbl = npu4_rev_vbnv_tbl, + .dev_priv = &npu5_dev_priv, + .fw_feature_tbl = npu4_fw_feature_table, + .ops = &aie2_ops, +}; diff --git a/drivers/accel/amdxdna/npu6_regs.c b/drivers/accel/amdxdna/npu6_regs.c new file mode 100644 index 000000000..e68637d2a --- /dev/null +++ b/drivers/accel/amdxdna/npu6_regs.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024, Advanced Micro Devices, Inc. + */ + +#include <drm/amdxdna_accel.h> +#include <drm/drm_device.h> +#include <drm/gpu_scheduler.h> +#include <linux/sizes.h> + +#include "aie2_pci.h" +#include "amdxdna_mailbox.h" +#include "amdxdna_pci_drv.h" + +/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */ +#define MPNPU_PWAITMODE 0x301003C +#define MPNPU_PUB_SEC_INTR 0x3010060 +#define MPNPU_PUB_PWRMGMT_INTR 0x3010064 +#define MPNPU_PUB_SCRATCH0 0x301006C +#define MPNPU_PUB_SCRATCH1 0x3010070 +#define MPNPU_PUB_SCRATCH2 0x3010074 +#define MPNPU_PUB_SCRATCH3 0x3010078 +#define MPNPU_PUB_SCRATCH4 0x301007C +#define MPNPU_PUB_SCRATCH5 0x3010080 +#define MPNPU_PUB_SCRATCH6 0x3010084 +#define MPNPU_PUB_SCRATCH7 0x3010088 +#define MPNPU_PUB_SCRATCH8 0x301008C +#define MPNPU_PUB_SCRATCH9 0x3010090 +#define MPNPU_PUB_SCRATCH10 0x3010094 +#define MPNPU_PUB_SCRATCH11 0x3010098 +#define MPNPU_PUB_SCRATCH12 0x301009C +#define MPNPU_PUB_SCRATCH13 0x30100A0 +#define MPNPU_PUB_SCRATCH14 0x30100A4 +#define MPNPU_PUB_SCRATCH15 0x30100A8 +#define MP0_C2PMSG_73 0x3810A24 +#define MP0_C2PMSG_123 0x3810AEC + +#define MP1_C2PMSG_0 0x3B10900 +#define MP1_C2PMSG_60 0x3B109F0 +#define MP1_C2PMSG_61 0x3B109F4 + +#define MPNPU_SRAM_X2I_MAILBOX_0 0x3600000 +#define MPNPU_SRAM_X2I_MAILBOX_15 0x361E000 +#define MPNPU_SRAM_X2I_MAILBOX_31 0x363E000 +#define MPNPU_SRAM_I2X_MAILBOX_31 0x363F000 + +#define MMNPU_APERTURE0_BASE 0x3000000 +#define MMNPU_APERTURE1_BASE 0x3600000 +#define MMNPU_APERTURE3_BASE 0x3810000 +#define MMNPU_APERTURE4_BASE 0x3B10000 + +/* PCIe BAR Index for NPU6 */ +#define NPU6_REG_BAR_INDEX 0 +#define NPU6_MBOX_BAR_INDEX 0 +#define NPU6_PSP_BAR_INDEX 4 +#define NPU6_SMU_BAR_INDEX 5 +#define NPU6_SRAM_BAR_INDEX 2 +/* Associated BARs and Apertures */ +#define NPU6_REG_BAR_BASE MMNPU_APERTURE0_BASE +#define NPU6_MBOX_BAR_BASE MMNPU_APERTURE0_BASE +#define NPU6_PSP_BAR_BASE MMNPU_APERTURE3_BASE +#define NPU6_SMU_BAR_BASE MMNPU_APERTURE4_BASE +#define NPU6_SRAM_BAR_BASE MMNPU_APERTURE1_BASE + +static const struct amdxdna_dev_priv npu6_dev_priv = { + .fw_path = "amdnpu/17f0_10/", + .rt_config = npu4_default_rt_cfg, + .dpm_clk_tbl = npu4_dpm_clk_table, + .col_align = COL_ALIGN_NATURE, + .col_opc = 4096, + .mbox_dev_addr = NPU6_MBOX_BAR_BASE, + .mbox_size = 0, /* Use BAR size */ + .sram_dev_addr = NPU6_SRAM_BAR_BASE, + .hwctx_limit = 16, + .sram_offs = { + DEFINE_BAR_OFFSET(MBOX_CHANN_OFF, NPU6_SRAM, MPNPU_SRAM_X2I_MAILBOX_0), + DEFINE_BAR_OFFSET(FW_ALIVE_OFF, NPU6_SRAM, MPNPU_SRAM_X2I_MAILBOX_15), + }, + .psp_regs_off = { + DEFINE_BAR_OFFSET(PSP_CMD_REG, NPU6_PSP, MP0_C2PMSG_123), + DEFINE_BAR_OFFSET(PSP_ARG0_REG, NPU6_REG, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_ARG1_REG, NPU6_REG, MPNPU_PUB_SCRATCH4), + DEFINE_BAR_OFFSET(PSP_ARG2_REG, NPU6_REG, MPNPU_PUB_SCRATCH9), + DEFINE_BAR_OFFSET(PSP_INTR_REG, NPU6_PSP, MP0_C2PMSG_73), + DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU6_PSP, MP0_C2PMSG_123), + DEFINE_BAR_OFFSET(PSP_RESP_REG, NPU6_REG, MPNPU_PUB_SCRATCH3), + DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU6_REG, MPNPU_PWAITMODE), + }, + .smu_regs_off = { + DEFINE_BAR_OFFSET(SMU_CMD_REG, NPU6_SMU, MP1_C2PMSG_0), + DEFINE_BAR_OFFSET(SMU_ARG_REG, NPU6_SMU, MP1_C2PMSG_60), + DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU6_SMU, MMNPU_APERTURE4_BASE), + DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU6_SMU, MP1_C2PMSG_61), + DEFINE_BAR_OFFSET(SMU_OUT_REG, NPU6_SMU, MP1_C2PMSG_60), + }, + .hw_ops = &npu4_hw_ops + +}; + +const struct amdxdna_dev_info dev_npu6_info = { + .reg_bar = NPU6_REG_BAR_INDEX, + .mbox_bar = NPU6_MBOX_BAR_INDEX, + .sram_bar = NPU6_SRAM_BAR_INDEX, + .psp_bar = NPU6_PSP_BAR_INDEX, + .smu_bar = NPU6_SMU_BAR_INDEX, + .first_col = 0, + .dev_mem_buf_shift = 15, /* 32 KiB aligned */ + .dev_mem_base = AIE2_DEVM_BASE, + .dev_mem_size = AIE2_DEVM_SIZE, + .default_vbnv = "RyzenAI-npu6", + .dev_heap_max_size = AIE2_DEVM_MAX_SIZE, + .device_type = AMDXDNA_DEV_TYPE_KMQ, + .rev_vbnv_tbl = npu4_rev_vbnv_tbl, + .dev_priv = &npu6_dev_priv, + .fw_feature_tbl = npu4_fw_feature_table, + .ops = &aie2_ops, +}; |