From 5febf432df1cfa5b25d99b54c32103fafbdd0eb9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 16 Aug 2026 15:32:33 +0200 Subject: kbuild: rust: keep Rust objects out of Clang LTO with inline helpers Under `CONFIG_LTO_CLANG` + `CONFIG_RUST_INLINE_HELPERS`, one may hit `objtool` errors such as: vmlinux.o: error: objtool: _R..._3Gsp4boot+0xd6a: can't find jump dest instruction at .text._R..._3Gsp4boot+0x1dfd The reason is that in such builds, the Clang invocation that compiles the combined Rust plus helpers bitcode emits LLVM bitcode (again) -- the final code generation happens in the linker's LTO step, which the `-mllvm` trap options passed to Clang do not reach. This, in turn, means that unreachable traps are missing, and the impossible paths do not merely fallthrough to the next symbol, but past the end of their own section, since LTO builds place each function in its own section. Thus filter `CC_FLAGS_LTO` out of the Clang invocation, so that it always emits machine code directly, with the traps in place. Assisted-by: LLM Cc: Gary Guo Cc: Boqun Feng Cc: Alice Ryhl Cc: Matthew Maurer Cc: Josh Poimboeuf Cc: Peter Zijlstra Cc: stable@vger.kernel.org Fixes: 3a2486cc1da5 ("kbuild: rust: provide an option to inline C helpers into Rust") Acked-by: Gary Guo Link: https://patch.msgid.link/20260816133233.197500-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- drivers/accel/ivpu/Kconfig | 27 + drivers/accel/ivpu/Makefile | 30 + drivers/accel/ivpu/ivpu_coredump.c | 39 + drivers/accel/ivpu/ivpu_coredump.h | 25 + drivers/accel/ivpu/ivpu_debugfs.c | 524 +++++++++ drivers/accel/ivpu/ivpu_debugfs.h | 17 + drivers/accel/ivpu/ivpu_drv.c | 893 +++++++++++++++ drivers/accel/ivpu/ivpu_drv.h | 333 ++++++ drivers/accel/ivpu/ivpu_fw.c | 724 ++++++++++++ drivers/accel/ivpu/ivpu_fw.h | 70 ++ drivers/accel/ivpu/ivpu_fw_log.c | 184 +++ drivers/accel/ivpu/ivpu_fw_log.h | 31 + drivers/accel/ivpu/ivpu_gem.c | 579 ++++++++++ drivers/accel/ivpu/ivpu_gem.h | 117 ++ drivers/accel/ivpu/ivpu_gem_userptr.c | 213 ++++ drivers/accel/ivpu/ivpu_hw.c | 422 +++++++ drivers/accel/ivpu/ivpu_hw.h | 148 +++ drivers/accel/ivpu/ivpu_hw_37xx_reg.h | 178 +++ drivers/accel/ivpu/ivpu_hw_40xx_reg.h | 195 ++++ drivers/accel/ivpu/ivpu_hw_btrs.c | 971 ++++++++++++++++ drivers/accel/ivpu/ivpu_hw_btrs.h | 54 + drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h | 112 ++ drivers/accel/ivpu/ivpu_hw_btrs_mtl_reg.h | 83 ++ drivers/accel/ivpu/ivpu_hw_ip.c | 1226 ++++++++++++++++++++ drivers/accel/ivpu/ivpu_hw_ip.h | 35 + drivers/accel/ivpu/ivpu_hw_reg_io.h | 133 +++ drivers/accel/ivpu/ivpu_ipc.c | 597 ++++++++++ drivers/accel/ivpu/ivpu_ipc.h | 113 ++ drivers/accel/ivpu/ivpu_job.c | 1215 ++++++++++++++++++++ drivers/accel/ivpu/ivpu_job.h | 93 ++ drivers/accel/ivpu/ivpu_jsm_msg.c | 586 ++++++++++ drivers/accel/ivpu/ivpu_jsm_msg.h | 50 + drivers/accel/ivpu/ivpu_mmu.c | 1030 +++++++++++++++++ drivers/accel/ivpu/ivpu_mmu.h | 53 + drivers/accel/ivpu/ivpu_mmu_context.c | 654 +++++++++++ drivers/accel/ivpu/ivpu_mmu_context.h | 52 + drivers/accel/ivpu/ivpu_ms.c | 349 ++++++ drivers/accel/ivpu/ivpu_ms.h | 36 + drivers/accel/ivpu/ivpu_pm.c | 523 +++++++++ drivers/accel/ivpu/ivpu_pm.h | 50 + drivers/accel/ivpu/ivpu_sysfs.c | 299 +++++ drivers/accel/ivpu/ivpu_sysfs.h | 13 + drivers/accel/ivpu/ivpu_trace.h | 73 ++ drivers/accel/ivpu/ivpu_trace_points.c | 9 + drivers/accel/ivpu/vpu_boot_api.h | 450 ++++++++ drivers/accel/ivpu/vpu_jsm_api.h | 1758 +++++++++++++++++++++++++++++ 46 files changed, 15366 insertions(+) create mode 100644 drivers/accel/ivpu/Kconfig create mode 100644 drivers/accel/ivpu/Makefile create mode 100644 drivers/accel/ivpu/ivpu_coredump.c create mode 100644 drivers/accel/ivpu/ivpu_coredump.h create mode 100644 drivers/accel/ivpu/ivpu_debugfs.c create mode 100644 drivers/accel/ivpu/ivpu_debugfs.h create mode 100644 drivers/accel/ivpu/ivpu_drv.c create mode 100644 drivers/accel/ivpu/ivpu_drv.h create mode 100644 drivers/accel/ivpu/ivpu_fw.c create mode 100644 drivers/accel/ivpu/ivpu_fw.h create mode 100644 drivers/accel/ivpu/ivpu_fw_log.c create mode 100644 drivers/accel/ivpu/ivpu_fw_log.h create mode 100644 drivers/accel/ivpu/ivpu_gem.c create mode 100644 drivers/accel/ivpu/ivpu_gem.h create mode 100644 drivers/accel/ivpu/ivpu_gem_userptr.c create mode 100644 drivers/accel/ivpu/ivpu_hw.c create mode 100644 drivers/accel/ivpu/ivpu_hw.h create mode 100644 drivers/accel/ivpu/ivpu_hw_37xx_reg.h create mode 100644 drivers/accel/ivpu/ivpu_hw_40xx_reg.h create mode 100644 drivers/accel/ivpu/ivpu_hw_btrs.c create mode 100644 drivers/accel/ivpu/ivpu_hw_btrs.h create mode 100644 drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h create mode 100644 drivers/accel/ivpu/ivpu_hw_btrs_mtl_reg.h create mode 100644 drivers/accel/ivpu/ivpu_hw_ip.c create mode 100644 drivers/accel/ivpu/ivpu_hw_ip.h create mode 100644 drivers/accel/ivpu/ivpu_hw_reg_io.h create mode 100644 drivers/accel/ivpu/ivpu_ipc.c create mode 100644 drivers/accel/ivpu/ivpu_ipc.h create mode 100644 drivers/accel/ivpu/ivpu_job.c create mode 100644 drivers/accel/ivpu/ivpu_job.h create mode 100644 drivers/accel/ivpu/ivpu_jsm_msg.c create mode 100644 drivers/accel/ivpu/ivpu_jsm_msg.h create mode 100644 drivers/accel/ivpu/ivpu_mmu.c create mode 100644 drivers/accel/ivpu/ivpu_mmu.h create mode 100644 drivers/accel/ivpu/ivpu_mmu_context.c create mode 100644 drivers/accel/ivpu/ivpu_mmu_context.h create mode 100644 drivers/accel/ivpu/ivpu_ms.c create mode 100644 drivers/accel/ivpu/ivpu_ms.h create mode 100644 drivers/accel/ivpu/ivpu_pm.c create mode 100644 drivers/accel/ivpu/ivpu_pm.h create mode 100644 drivers/accel/ivpu/ivpu_sysfs.c create mode 100644 drivers/accel/ivpu/ivpu_sysfs.h create mode 100644 drivers/accel/ivpu/ivpu_trace.h create mode 100644 drivers/accel/ivpu/ivpu_trace_points.c create mode 100644 drivers/accel/ivpu/vpu_boot_api.h create mode 100644 drivers/accel/ivpu/vpu_jsm_api.h (limited to 'drivers/accel/ivpu') diff --git a/drivers/accel/ivpu/Kconfig b/drivers/accel/ivpu/Kconfig new file mode 100644 index 000000000..9e055b5ce --- /dev/null +++ b/drivers/accel/ivpu/Kconfig @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config DRM_ACCEL_IVPU + tristate "Intel NPU (Neural Processing Unit)" + depends on DRM_ACCEL + depends on X86_64 && !UML + depends on PCI && PCI_MSI + select FW_LOADER + select DRM_GEM_SHMEM_HELPER + select GENERIC_ALLOCATOR + select WANT_DEV_COREDUMP + help + Choose this option if you have a system with an 14th generation + Intel CPU (Meteor Lake) or newer. Intel NPU (formerly called Intel VPU) + is a CPU-integrated inference accelerator for Computer Vision + and Deep Learning applications. + + If "M" is selected, the module will be called intel_vpu. + +config DRM_ACCEL_IVPU_DEBUG + bool "Intel NPU debug mode" + depends on DRM_ACCEL_IVPU + help + Choose this option to enable additional + debug features for the Intel NPU driver: + - Always print debug messages regardless of dyndbg config, + - Enable unsafe module params. diff --git a/drivers/accel/ivpu/Makefile b/drivers/accel/ivpu/Makefile new file mode 100644 index 000000000..dbf76b8a5 --- /dev/null +++ b/drivers/accel/ivpu/Makefile @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2023-2024 Intel Corporation + +intel_vpu-y := \ + ivpu_drv.o \ + ivpu_fw.o \ + ivpu_fw_log.o \ + ivpu_gem.o \ + ivpu_gem_userptr.o \ + ivpu_hw.o \ + ivpu_hw_btrs.o \ + ivpu_hw_ip.o \ + ivpu_ipc.o \ + ivpu_job.o \ + ivpu_jsm_msg.o \ + ivpu_mmu.o \ + ivpu_mmu_context.o \ + ivpu_ms.o \ + ivpu_pm.o \ + ivpu_sysfs.o \ + ivpu_trace_points.o + +intel_vpu-$(CONFIG_DEBUG_FS) += ivpu_debugfs.o +intel_vpu-$(CONFIG_DEV_COREDUMP) += ivpu_coredump.o + +obj-$(CONFIG_DRM_ACCEL_IVPU) += intel_vpu.o + +subdir-ccflags-$(CONFIG_DRM_ACCEL_IVPU_DEBUG) += -DDEBUG + +CFLAGS_ivpu_trace_points.o = -I$(src) diff --git a/drivers/accel/ivpu/ivpu_coredump.c b/drivers/accel/ivpu/ivpu_coredump.c new file mode 100644 index 000000000..16ad0c308 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_coredump.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#include +#include + +#include "ivpu_coredump.h" +#include "ivpu_fw.h" +#include "ivpu_gem.h" +#include "vpu_boot_api.h" + +#define CRASH_DUMP_HEADER "Intel NPU crash dump" +#define CRASH_DUMP_HEADERS_SIZE SZ_4K + +void ivpu_dev_coredump(struct ivpu_device *vdev) +{ + struct drm_print_iterator pi = {}; + struct drm_printer p; + size_t coredump_size; + char *coredump; + + coredump_size = CRASH_DUMP_HEADERS_SIZE + FW_VERSION_HEADER_SIZE + + ivpu_bo_size(vdev->fw->mem_log_crit) + ivpu_bo_size(vdev->fw->mem_log_verb); + coredump = vmalloc(coredump_size); + if (!coredump) + return; + + pi.data = coredump; + pi.remain = coredump_size; + p = drm_coredump_printer(&pi); + + drm_printf(&p, "%s\n", CRASH_DUMP_HEADER); + drm_printf(&p, "FW version: %s\n", vdev->fw->version); + ivpu_fw_log_print(vdev, false, &p); + + dev_coredumpv(vdev->drm.dev, coredump, pi.offset, GFP_KERNEL); +} diff --git a/drivers/accel/ivpu/ivpu_coredump.h b/drivers/accel/ivpu/ivpu_coredump.h new file mode 100644 index 000000000..8efb09d02 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_coredump.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __IVPU_COREDUMP_H__ +#define __IVPU_COREDUMP_H__ + +#include + +#include "ivpu_drv.h" +#include "ivpu_fw_log.h" + +#ifdef CONFIG_DEV_COREDUMP +void ivpu_dev_coredump(struct ivpu_device *vdev); +#else +static inline void ivpu_dev_coredump(struct ivpu_device *vdev) +{ + struct drm_printer p = drm_info_printer(vdev->drm.dev); + + ivpu_fw_log_print(vdev, false, &p); +} +#endif + +#endif /* __IVPU_COREDUMP_H__ */ diff --git a/drivers/accel/ivpu/ivpu_debugfs.c b/drivers/accel/ivpu/ivpu_debugfs.c new file mode 100644 index 000000000..dc20bc73c --- /dev/null +++ b/drivers/accel/ivpu/ivpu_debugfs.c @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include +#include + +#include +#include +#include + +#include + +#include "ivpu_debugfs.h" +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_fw_log.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_pm.h" +#include "vpu_boot_api.h" + +static inline struct ivpu_device *seq_to_ivpu(struct seq_file *s) +{ + struct drm_debugfs_entry *entry = s->private; + + return to_ivpu_device(entry->dev); +} + +static int bo_list_show(struct seq_file *s, void *v) +{ + struct drm_printer p = drm_seq_file_printer(s); + struct ivpu_device *vdev = seq_to_ivpu(s); + + ivpu_bo_list(&vdev->drm, &p); + + return 0; +} + +static int fw_name_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%s\n", vdev->fw->name); + return 0; +} + +static int fw_version_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%s\n", vdev->fw->version); + return 0; +} + +static int fw_trace_capability_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + u64 trace_hw_component_mask; + u32 trace_destination_mask; + int ret; + + ret = ivpu_jsm_trace_get_capability(vdev, &trace_destination_mask, + &trace_hw_component_mask); + if (!ret) { + seq_printf(s, + "trace_destination_mask: %#18x\n" + "trace_hw_component_mask: %#18llx\n", + trace_destination_mask, trace_hw_component_mask); + } + return 0; +} + +static int fw_trace_config_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + /** + * WA: VPU_JSM_MSG_TRACE_GET_CONFIG command is not working yet, + * so we use values from vdev->fw instead of calling ivpu_jsm_trace_get_config() + */ + u32 trace_level = vdev->fw->trace_level; + u32 trace_destination_mask = vdev->fw->trace_destination_mask; + u64 trace_hw_component_mask = vdev->fw->trace_hw_component_mask; + + seq_printf(s, + "trace_level: %#18x\n" + "trace_destination_mask: %#18x\n" + "trace_hw_component_mask: %#18llx\n", + trace_level, trace_destination_mask, trace_hw_component_mask); + + return 0; +} + +static int last_bootmode_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%s\n", (vdev->fw->last_boot_mode == VPU_BOOT_TYPE_WARMBOOT) ? + "warm boot" : "cold boot"); + + return 0; +} + +static int reset_counter_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%d\n", atomic_read(&vdev->pm->reset_counter)); + return 0; +} + +static int reset_pending_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%d\n", atomic_read(&vdev->pm->reset_pending)); + return 0; +} + +static int firewall_irq_counter_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%d\n", atomic_read(&vdev->hw->firewall_irq_counter)); + return 0; +} + +static int engine_reset_counter_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = seq_to_ivpu(s); + + seq_printf(s, "%d\n", atomic_read(&vdev->pm->engine_reset_counter)); + return 0; +} + +static const struct drm_debugfs_info vdev_debugfs_list[] = { + {"bo_list", bo_list_show, 0}, + {"fw_name", fw_name_show, 0}, + {"fw_version", fw_version_show, 0}, + {"fw_trace_capability", fw_trace_capability_show, 0}, + {"fw_trace_config", fw_trace_config_show, 0}, + {"last_bootmode", last_bootmode_show, 0}, + {"reset_counter", reset_counter_show, 0}, + {"reset_pending", reset_pending_show, 0}, + {"firewall_irq_counter", firewall_irq_counter_show, 0}, + {"engine_reset_counter", engine_reset_counter_show, 0}, +}; + +static int dvfs_mode_get(void *data, u64 *dvfs_mode) +{ + struct ivpu_device *vdev = (struct ivpu_device *)data; + + *dvfs_mode = vdev->fw->dvfs_mode; + return 0; +} + +static int dvfs_mode_set(void *data, u64 dvfs_mode) +{ + struct ivpu_device *vdev = (struct ivpu_device *)data; + + vdev->fw->dvfs_mode = (u32)dvfs_mode; + return pci_try_reset_function(to_pci_dev(vdev->drm.dev)); +} + +DEFINE_DEBUGFS_ATTRIBUTE(dvfs_mode_fops, dvfs_mode_get, dvfs_mode_set, "%llu\n"); + +static ssize_t +fw_dyndbg_fops_write(struct file *file, const char __user *user_buf, size_t size, loff_t *pos) +{ + struct ivpu_device *vdev = file->private_data; + char buffer[VPU_DYNDBG_CMD_MAX_LEN] = {}; + int ret; + + if (size >= VPU_DYNDBG_CMD_MAX_LEN) + return -EINVAL; + + ret = strncpy_from_user(buffer, user_buf, size); + if (ret < 0) + return ret; + + ivpu_jsm_dyndbg_control(vdev, buffer, size); + return size; +} + +static const struct file_operations fw_dyndbg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fw_dyndbg_fops_write, +}; + +static int fw_log_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = s->private; + struct drm_printer p = drm_seq_file_printer(s); + + ivpu_fw_log_print(vdev, true, &p); + return 0; +} + +static int fw_log_fops_open(struct inode *inode, struct file *file) +{ + return single_open(file, fw_log_show, inode->i_private); +} + +static ssize_t +fw_log_fops_write(struct file *file, const char __user *user_buf, size_t size, loff_t *pos) +{ + struct seq_file *s = file->private_data; + struct ivpu_device *vdev = s->private; + + if (!size) + return -EINVAL; + + ivpu_fw_log_mark_read(vdev); + return size; +} + +static const struct file_operations fw_log_fops = { + .owner = THIS_MODULE, + .open = fw_log_fops_open, + .write = fw_log_fops_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t +fw_profiling_freq_fops_write(struct file *file, const char __user *user_buf, + size_t size, loff_t *pos) +{ + struct ivpu_device *vdev = file->private_data; + bool enable; + int ret; + + ret = kstrtobool_from_user(user_buf, size, &enable); + if (ret < 0) + return ret; + + ivpu_hw_profiling_freq_drive(vdev, enable); + + ret = pci_try_reset_function(to_pci_dev(vdev->drm.dev)); + if (ret) + return ret; + + return size; +} + +static const struct file_operations fw_profiling_freq_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fw_profiling_freq_fops_write, +}; + +static ssize_t +fw_trace_destination_mask_fops_write(struct file *file, const char __user *user_buf, + size_t size, loff_t *pos) +{ + struct ivpu_device *vdev = file->private_data; + struct ivpu_fw_info *fw = vdev->fw; + u32 trace_destination_mask; + int ret; + + ret = kstrtou32_from_user(user_buf, size, 0, &trace_destination_mask); + if (ret < 0) + return ret; + + fw->trace_destination_mask = trace_destination_mask; + + ivpu_jsm_trace_set_config(vdev, fw->trace_level, trace_destination_mask, + fw->trace_hw_component_mask); + + return size; +} + +static const struct file_operations fw_trace_destination_mask_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fw_trace_destination_mask_fops_write, +}; + +static ssize_t +fw_trace_hw_comp_mask_fops_write(struct file *file, const char __user *user_buf, + size_t size, loff_t *pos) +{ + struct ivpu_device *vdev = file->private_data; + struct ivpu_fw_info *fw = vdev->fw; + u64 trace_hw_component_mask; + int ret; + + ret = kstrtou64_from_user(user_buf, size, 0, &trace_hw_component_mask); + if (ret < 0) + return ret; + + fw->trace_hw_component_mask = trace_hw_component_mask; + + ivpu_jsm_trace_set_config(vdev, fw->trace_level, fw->trace_destination_mask, + trace_hw_component_mask); + + return size; +} + +static const struct file_operations fw_trace_hw_comp_mask_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fw_trace_hw_comp_mask_fops_write, +}; + +static ssize_t +fw_trace_level_fops_write(struct file *file, const char __user *user_buf, size_t size, loff_t *pos) +{ + struct ivpu_device *vdev = file->private_data; + struct ivpu_fw_info *fw = vdev->fw; + u32 trace_level; + int ret; + + ret = kstrtou32_from_user(user_buf, size, 0, &trace_level); + if (ret < 0) + return ret; + + fw->trace_level = trace_level; + + ivpu_jsm_trace_set_config(vdev, trace_level, fw->trace_destination_mask, + fw->trace_hw_component_mask); + + return size; +} + +static const struct file_operations fw_trace_level_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fw_trace_level_fops_write, +}; + +static ssize_t +ivpu_force_recovery_fn(struct file *file, const char __user *user_buf, size_t size, loff_t *pos) +{ + struct ivpu_device *vdev = file->private_data; + int ret; + + if (!size) + return -EINVAL; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + ivpu_pm_trigger_recovery(vdev, "debugfs"); + flush_work(&vdev->pm->recovery_work); + ivpu_rpm_put(vdev); + return size; +} + +static const struct file_operations ivpu_force_recovery_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ivpu_force_recovery_fn, +}; + +static int ivpu_reset_engine_fn(void *data, u64 val) +{ + struct ivpu_device *vdev = (struct ivpu_device *)data; + struct vpu_jsm_msg resp; + + return ivpu_jsm_reset_engine(vdev, (u32)val, &resp); +} + +DEFINE_DEBUGFS_ATTRIBUTE(ivpu_reset_engine_fops, NULL, ivpu_reset_engine_fn, "0x%02llx\n"); + +static int ivpu_resume_engine_fn(void *data, u64 val) +{ + struct ivpu_device *vdev = (struct ivpu_device *)data; + + return ivpu_jsm_hws_resume_engine(vdev, (u32)val); +} + +DEFINE_DEBUGFS_ATTRIBUTE(ivpu_resume_engine_fops, NULL, ivpu_resume_engine_fn, "0x%02llx\n"); + +static int dct_active_get(void *data, u64 *active_percent) +{ + struct ivpu_device *vdev = data; + + *active_percent = vdev->pm->dct_active_percent; + + return 0; +} + +static int dct_active_set(void *data, u64 active_percent) +{ + struct ivpu_device *vdev = data; + int ret; + + if (active_percent > 100) + return -EINVAL; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + if (active_percent) + ret = ivpu_pm_dct_enable(vdev, active_percent); + else + ret = ivpu_pm_dct_disable(vdev); + + ivpu_rpm_put(vdev); + + return ret; +} + +DEFINE_DEBUGFS_ATTRIBUTE(ivpu_dct_fops, dct_active_get, dct_active_set, "%llu\n"); + +static void print_priority_band(struct seq_file *s, struct ivpu_hw_info *hw, + int band, const char *name) +{ + seq_printf(s, "%-9s: grace_period %9u process_grace_period %9u process_quantum %9u\n", + name, + hw->hws.grace_period[band], + hw->hws.process_grace_period[band], + hw->hws.process_quantum[band]); +} + +static int priority_bands_show(struct seq_file *s, void *v) +{ + struct ivpu_device *vdev = s->private; + struct ivpu_hw_info *hw = vdev->hw; + + print_priority_band(s, hw, VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE, "Idle"); + print_priority_band(s, hw, VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL, "Normal"); + print_priority_band(s, hw, VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS, "Focus"); + print_priority_band(s, hw, VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME, "Realtime"); + + return 0; +} + +static int priority_bands_fops_open(struct inode *inode, struct file *file) +{ + return single_open(file, priority_bands_show, inode->i_private); +} + +static ssize_t +priority_bands_fops_write(struct file *file, const char __user *user_buf, size_t size, loff_t *pos) +{ + struct seq_file *s = file->private_data; + struct ivpu_device *vdev = s->private; + char buf[64]; + u32 grace_period; + u32 process_grace_period; + u32 process_quantum; + u32 band; + int ret; + + if (*pos != 0 || size >= sizeof(buf)) + return -EINVAL; + + ret = simple_write_to_buffer(buf, sizeof(buf) - 1, pos, user_buf, size); + if (ret < 0) + return ret; + + buf[ret] = '\0'; + ret = sscanf(buf, "%u %u %u %u", &band, &grace_period, &process_grace_period, + &process_quantum); + if (ret != 4) + return -EINVAL; + + if (band >= VPU_JOB_SCHEDULING_PRIORITY_BAND_COUNT) + return -EINVAL; + + vdev->hw->hws.grace_period[band] = grace_period; + vdev->hw->hws.process_grace_period[band] = process_grace_period; + vdev->hw->hws.process_quantum[band] = process_quantum; + + return size; +} + +static const struct file_operations ivpu_hws_priority_bands_fops = { + .owner = THIS_MODULE, + .open = priority_bands_fops_open, + .write = priority_bands_fops_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void ivpu_debugfs_init(struct ivpu_device *vdev) +{ + struct dentry *debugfs_root = vdev->drm.debugfs_root; + + drm_debugfs_add_files(&vdev->drm, vdev_debugfs_list, ARRAY_SIZE(vdev_debugfs_list)); + + debugfs_create_file("force_recovery", 0200, debugfs_root, vdev, + &ivpu_force_recovery_fops); + + debugfs_create_file("dvfs_mode", 0644, debugfs_root, vdev, + &dvfs_mode_fops); + + debugfs_create_file("fw_dyndbg", 0200, debugfs_root, vdev, + &fw_dyndbg_fops); + debugfs_create_file("fw_log", 0644, debugfs_root, vdev, + &fw_log_fops); + debugfs_create_file("fw_trace_destination_mask", 0200, debugfs_root, vdev, + &fw_trace_destination_mask_fops); + debugfs_create_file("fw_trace_hw_comp_mask", 0200, debugfs_root, vdev, + &fw_trace_hw_comp_mask_fops); + debugfs_create_file("fw_trace_level", 0200, debugfs_root, vdev, + &fw_trace_level_fops); + debugfs_create_file("hws_priority_bands", 0200, debugfs_root, vdev, + &ivpu_hws_priority_bands_fops); + + debugfs_create_file("reset_engine", 0200, debugfs_root, vdev, + &ivpu_reset_engine_fops); + debugfs_create_file("resume_engine", 0200, debugfs_root, vdev, + &ivpu_resume_engine_fops); + + if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX) { + debugfs_create_file("fw_profiling_freq_drive", 0200, + debugfs_root, vdev, &fw_profiling_freq_fops); + debugfs_create_file("dct", 0644, debugfs_root, vdev, &ivpu_dct_fops); + } + +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("fail_hw", debugfs_root, &ivpu_hw_failure); +#endif +} diff --git a/drivers/accel/ivpu/ivpu_debugfs.h b/drivers/accel/ivpu/ivpu_debugfs.h new file mode 100644 index 000000000..49ae9ea78 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_debugfs.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_DEBUGFS_H__ +#define __IVPU_DEBUGFS_H__ + +struct ivpu_device; + +#if defined(CONFIG_DEBUG_FS) +void ivpu_debugfs_init(struct ivpu_device *vdev); +#else +static inline void ivpu_debugfs_init(struct ivpu_device *vdev) { } +#endif + +#endif /* __IVPU_DEBUGFS_H__ */ diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c new file mode 100644 index 000000000..35e506074 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_drv.c @@ -0,0 +1,893 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "ivpu_coredump.h" +#include "ivpu_debugfs.h" +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_fw_log.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_ipc.h" +#include "ivpu_job.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_mmu.h" +#include "ivpu_mmu_context.h" +#include "ivpu_ms.h" +#include "ivpu_pm.h" +#include "ivpu_sysfs.h" +#include "vpu_boot_api.h" + +#ifndef DRIVER_VERSION_STR +#define DRIVER_VERSION_STR "1.0.0 " UTS_RELEASE +#endif + +int ivpu_dbg_mask; +module_param_named(dbg_mask, ivpu_dbg_mask, int, 0644); +MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros."); + +int ivpu_test_mode; +#if IS_ENABLED(CONFIG_DRM_ACCEL_IVPU_DEBUG) +module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644); +MODULE_PARM_DESC(test_mode, "Test mode mask. See IVPU_TEST_MODE_* macros."); +#endif + +u8 ivpu_pll_min_ratio; +module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644); +MODULE_PARM_DESC(pll_min_ratio, "Minimum PLL ratio used to set NPU frequency"); + +u8 ivpu_pll_max_ratio = U8_MAX; +module_param_named(pll_max_ratio, ivpu_pll_max_ratio, byte, 0644); +MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set NPU frequency"); + +int ivpu_sched_mode = IVPU_SCHED_MODE_AUTO; +module_param_named(sched_mode, ivpu_sched_mode, int, 0444); +MODULE_PARM_DESC(sched_mode, "Scheduler mode: -1 - Use default scheduler, 0 - Use OS scheduler (supported on 27XX - 50XX), 1 - Use HW scheduler"); + +bool ivpu_disable_mmu_cont_pages; +module_param_named(disable_mmu_cont_pages, ivpu_disable_mmu_cont_pages, bool, 0444); +MODULE_PARM_DESC(disable_mmu_cont_pages, "Disable MMU contiguous pages optimization"); + +bool ivpu_force_snoop; +module_param_named(force_snoop, ivpu_force_snoop, bool, 0444); +MODULE_PARM_DESC(force_snoop, "Force snooping for NPU host memory access"); + +static struct ivpu_user_limits *ivpu_user_limits_alloc(struct ivpu_device *vdev, uid_t uid) +{ + struct ivpu_user_limits *limits; + + limits = kzalloc_obj(*limits); + if (!limits) + return ERR_PTR(-ENOMEM); + + kref_init(&limits->ref); + atomic_set(&limits->db_count, 0); + limits->vdev = vdev; + limits->uid = uid; + + /* Allow root user to allocate all contexts */ + if (uid == 0) { + limits->max_ctx_count = ivpu_get_context_count(vdev); + limits->max_db_count = ivpu_get_doorbell_count(vdev); + } else { + limits->max_ctx_count = ivpu_get_context_count(vdev) / 2; + limits->max_db_count = ivpu_get_doorbell_count(vdev) / 2; + } + + hash_add(vdev->user_limits, &limits->hash_node, uid); + + return limits; +} + +static struct ivpu_user_limits *ivpu_user_limits_get(struct ivpu_device *vdev) +{ + struct ivpu_user_limits *limits; + uid_t uid = current_uid().val; + + guard(mutex)(&vdev->user_limits_lock); + + hash_for_each_possible(vdev->user_limits, limits, hash_node, uid) { + if (limits->uid == uid) { + if (kref_read(&limits->ref) >= limits->max_ctx_count) { + ivpu_dbg(vdev, IOCTL, "User %u exceeded max ctx count %u\n", uid, + limits->max_ctx_count); + return ERR_PTR(-EMFILE); + } + + kref_get(&limits->ref); + return limits; + } + } + + return ivpu_user_limits_alloc(vdev, uid); +} + +static void ivpu_user_limits_release(struct kref *ref) +{ + struct ivpu_user_limits *limits = container_of(ref, struct ivpu_user_limits, ref); + struct ivpu_device *vdev = limits->vdev; + + lockdep_assert_held(&vdev->user_limits_lock); + drm_WARN_ON(&vdev->drm, atomic_read(&limits->db_count)); + hash_del(&limits->hash_node); + kfree(limits); +} + +static void ivpu_user_limits_put(struct ivpu_device *vdev, struct ivpu_user_limits *limits) +{ + guard(mutex)(&vdev->user_limits_lock); + kref_put(&limits->ref, ivpu_user_limits_release); +} + +struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv) +{ + struct ivpu_device *vdev = file_priv->vdev; + + kref_get(&file_priv->ref); + + ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n", + file_priv->ctx.id, kref_read(&file_priv->ref)); + + return file_priv; +} + +static void file_priv_unbind(struct ivpu_device *vdev, struct ivpu_file_priv *file_priv) +{ + mutex_lock(&file_priv->lock); + if (file_priv->bound) { + ivpu_dbg(vdev, FILE, "file_priv unbind: ctx %u\n", file_priv->ctx.id); + + ivpu_cmdq_release_all_locked(file_priv); + ivpu_bo_unbind_all_bos_from_context(vdev, &file_priv->ctx); + ivpu_mmu_context_fini(vdev, &file_priv->ctx); + file_priv->bound = false; + drm_WARN_ON(&vdev->drm, !xa_erase_irq(&vdev->context_xa, file_priv->ctx.id)); + } + mutex_unlock(&file_priv->lock); +} + +static void file_priv_release(struct kref *ref) +{ + struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref); + struct ivpu_device *vdev = file_priv->vdev; + + ivpu_dbg(vdev, FILE, "file_priv release: ctx %u bound %d\n", + file_priv->ctx.id, (bool)file_priv->bound); + + pm_runtime_get_sync(vdev->drm.dev); + mutex_lock(&vdev->context_list_lock); + file_priv_unbind(vdev, file_priv); + drm_WARN_ON(&vdev->drm, !xa_empty(&file_priv->cmdq_xa)); + xa_destroy(&file_priv->cmdq_xa); + mutex_unlock(&vdev->context_list_lock); + pm_runtime_put_autosuspend(vdev->drm.dev); + + ivpu_user_limits_put(vdev, file_priv->user_limits); + mutex_destroy(&file_priv->ms_lock); + mutex_destroy(&file_priv->lock); + kfree(file_priv); +} + +void ivpu_file_priv_put(struct ivpu_file_priv **link) +{ + struct ivpu_file_priv *file_priv = *link; + struct ivpu_device *vdev = file_priv->vdev; + + ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n", + file_priv->ctx.id, kref_read(&file_priv->ref)); + + *link = NULL; + kref_put(&file_priv->ref, file_priv_release); +} + +bool ivpu_is_capable(struct ivpu_device *vdev, u32 capability) +{ + switch (capability) { + case DRM_IVPU_CAP_METRIC_STREAMER: + return true; + case DRM_IVPU_CAP_DMA_MEMORY_RANGE: + return true; + case DRM_IVPU_CAP_BO_CREATE_FROM_USERPTR: + return true; + case DRM_IVPU_CAP_MANAGE_CMDQ: + return vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW; + default: + return false; + } +} + +static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); + struct drm_ivpu_param *args = data; + int ret = 0; + int idx; + + if (!drm_dev_enter(dev, &idx)) + return -ENODEV; + + switch (args->param) { + case DRM_IVPU_PARAM_DEVICE_ID: + args->value = pdev->device; + break; + case DRM_IVPU_PARAM_DEVICE_REVISION: + args->value = pdev->revision; + break; + case DRM_IVPU_PARAM_PLATFORM_TYPE: + args->value = vdev->platform; + break; + case DRM_IVPU_PARAM_CORE_CLOCK_RATE: + args->value = ivpu_hw_btrs_pll_ratio_to_hz(vdev, vdev->hw->pll.max_ratio); + break; + case DRM_IVPU_PARAM_NUM_CONTEXTS: + args->value = file_priv->user_limits->max_ctx_count; + break; + case DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS: + args->value = vdev->hw->ranges.user.start; + break; + case DRM_IVPU_PARAM_CONTEXT_ID: + args->value = file_priv->ctx.id; + break; + case DRM_IVPU_PARAM_FW_API_VERSION: + if (args->index < VPU_FW_API_VER_NUM) { + struct vpu_firmware_header *fw_hdr; + + fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data; + args->value = fw_hdr->api_version[args->index]; + } else { + ret = -EINVAL; + } + break; + case DRM_IVPU_PARAM_ENGINE_HEARTBEAT: + ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value); + break; + case DRM_IVPU_PARAM_UNIQUE_INFERENCE_ID: + args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter); + break; + case DRM_IVPU_PARAM_TILE_CONFIG: + args->value = vdev->hw->tile_fuse; + break; + case DRM_IVPU_PARAM_SKU: + args->value = vdev->hw->sku; + break; + case DRM_IVPU_PARAM_CAPABILITIES: + args->value = ivpu_is_capable(vdev, args->index); + break; + case DRM_IVPU_PARAM_PREEMPT_BUFFER_SIZE: + args->value = ivpu_fw_preempt_buf_size(vdev); + break; + default: + ret = -EINVAL; + break; + } + + drm_dev_exit(idx); + return ret; +} + +static int ivpu_set_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_ivpu_param *args = data; + int ret = 0; + + switch (args->param) { + default: + ret = -EINVAL; + } + + return ret; +} + +static int ivpu_open(struct drm_device *dev, struct drm_file *file) +{ + struct ivpu_device *vdev = to_ivpu_device(dev); + struct ivpu_file_priv *file_priv; + struct ivpu_user_limits *limits; + u32 ctx_id; + int idx, ret; + + if (!drm_dev_enter(dev, &idx)) + return -ENODEV; + + limits = ivpu_user_limits_get(vdev); + if (IS_ERR(limits)) { + ret = PTR_ERR(limits); + goto err_dev_exit; + } + + file_priv = kzalloc_obj(*file_priv); + if (!file_priv) { + ret = -ENOMEM; + goto err_user_limits_put; + } + + INIT_LIST_HEAD(&file_priv->ms_instance_list); + + file_priv->vdev = vdev; + file_priv->bound = true; + file_priv->user_limits = limits; + kref_init(&file_priv->ref); + mutex_init(&file_priv->lock); + mutex_init(&file_priv->ms_lock); + + mutex_lock(&vdev->context_list_lock); + + ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, file_priv, + vdev->context_xa_limit, GFP_KERNEL); + if (ret) { + ivpu_err(vdev, "Failed to allocate context id: %d\n", ret); + goto err_unlock; + } + + ivpu_mmu_context_init(vdev, &file_priv->ctx, ctx_id); + + file_priv->job_limit.min = FIELD_PREP(IVPU_JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1)); + file_priv->job_limit.max = file_priv->job_limit.min | IVPU_JOB_ID_JOB_MASK; + + xa_init_flags(&file_priv->cmdq_xa, XA_FLAGS_ALLOC1); + file_priv->cmdq_limit.min = IVPU_CMDQ_MIN_ID; + file_priv->cmdq_limit.max = IVPU_CMDQ_MAX_ID; + + mutex_unlock(&vdev->context_list_lock); + drm_dev_exit(idx); + + file->driver_priv = file_priv; + + ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n", + ctx_id, current->comm, task_pid_nr(current)); + + return 0; + +err_unlock: + mutex_unlock(&vdev->context_list_lock); + mutex_destroy(&file_priv->ms_lock); + mutex_destroy(&file_priv->lock); + kfree(file_priv); +err_user_limits_put: + ivpu_user_limits_put(vdev, limits); +err_dev_exit: + drm_dev_exit(idx); + return ret; +} + +static void ivpu_postclose(struct drm_device *dev, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = to_ivpu_device(dev); + + ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n", + file_priv->ctx.id, current->comm, task_pid_nr(current)); + + ivpu_ms_cleanup(file_priv); + ivpu_file_priv_put(&file_priv); +} + +static const struct drm_ioctl_desc ivpu_drm_ioctls[] = { + DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_SUBMIT, ivpu_submit_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_BO_WAIT, ivpu_bo_wait_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_START, ivpu_ms_start_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_GET_DATA, ivpu_ms_get_data_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_STOP, ivpu_ms_stop_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_GET_INFO, ivpu_ms_get_info_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_CMDQ_CREATE, ivpu_cmdq_create_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_CMDQ_DESTROY, ivpu_cmdq_destroy_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_CMDQ_SUBMIT, ivpu_cmdq_submit_ioctl, 0), + DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE_FROM_USERPTR, ivpu_bo_create_from_userptr_ioctl, 0), +}; + +static int ivpu_wait_for_ready(struct ivpu_device *vdev) +{ + struct ivpu_ipc_consumer cons; + struct ivpu_ipc_hdr ipc_hdr; + unsigned long timeout; + int ret; + + if (ivpu_test_mode & IVPU_TEST_MODE_FW_TEST) + return 0; + + ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG, NULL); + + timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot); + while (1) { + ivpu_ipc_irq_handler(vdev); + ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0); + if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout)) + break; + + cond_resched(); + } + + ivpu_ipc_consumer_del(vdev, &cons); + + if (!ret && ipc_hdr.data_addr != IVPU_IPC_BOOT_MSG_DATA_ADDR) { + ivpu_err(vdev, "Invalid NPU ready message: 0x%x\n", ipc_hdr.data_addr); + return -EIO; + } + + if (!ret) + ivpu_dbg(vdev, PM, "NPU ready message received successfully\n"); + + return ret; +} + +static int ivpu_hw_sched_init(struct ivpu_device *vdev) +{ + int ret = 0; + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { + ret = ivpu_jsm_hws_setup_priority_bands(vdev); + if (ret) { + ivpu_err(vdev, "Failed to enable hw scheduler: %d", ret); + return ret; + } + } + + return ret; +} + +/** + * ivpu_boot() - Start VPU firmware + * @vdev: VPU device + * + * This function is paired with ivpu_shutdown() but it doesn't power up the + * VPU because power up has to be called very early in ivpu_probe(). + */ +int ivpu_boot(struct ivpu_device *vdev) +{ + int ret; + + drm_WARN_ON(&vdev->drm, atomic_read(&vdev->job_timeout_counter)); + drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa)); + + ivpu_fw_boot_params_setup(vdev, ivpu_bo_vaddr(vdev->fw->mem_bp)); + vdev->fw->last_boot_mode = vdev->fw->next_boot_mode; + + ret = ivpu_hw_boot_fw(vdev); + if (ret) { + ivpu_err(vdev, "Failed to start the firmware: %d\n", ret); + return ret; + } + + ret = ivpu_wait_for_ready(vdev); + if (ret) { + ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret); + goto err_diagnose_failure; + } + ivpu_hw_irq_clear(vdev); + enable_irq(vdev->irq); + ivpu_hw_irq_enable(vdev); + ivpu_ipc_enable(vdev); + + if (!ivpu_fw_is_warm_boot(vdev)) { + ret = ivpu_pm_dct_init(vdev); + if (ret) + goto err_disable_ipc; + + ret = ivpu_hw_sched_init(vdev); + if (ret) + goto err_disable_ipc; + + ret = ivpu_hw_btrs_cfg_freq_init(vdev); + if (ret) + goto err_disable_ipc; + } + + return 0; + +err_disable_ipc: + ivpu_ipc_disable(vdev); + ivpu_hw_irq_disable(vdev); + disable_irq(vdev->irq); +err_diagnose_failure: + ivpu_hw_diagnose_failure(vdev); + ivpu_mmu_evtq_dump(vdev); + ivpu_dev_coredump(vdev); + return ret; +} + +void ivpu_prepare_for_reset(struct ivpu_device *vdev) +{ + ivpu_hw_irq_disable(vdev); + disable_irq(vdev->irq); + flush_work(&vdev->irq_ipc_work); + flush_work(&vdev->irq_dct_work); + flush_work(&vdev->context_abort_work); + ivpu_ipc_disable(vdev); + ivpu_mmu_disable(vdev); +} + +int ivpu_shutdown(struct ivpu_device *vdev) +{ + int ret; + + /* Save PCI state before powering down as it sometimes gets corrupted if NPU hangs */ + pci_save_state(to_pci_dev(vdev->drm.dev)); + + ret = ivpu_hw_power_down(vdev); + if (ret) + ivpu_warn(vdev, "Failed to power down HW: %d\n", ret); + + pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot); + + return ret; +} + +static const struct file_operations ivpu_fops = { + .owner = THIS_MODULE, + DRM_ACCEL_FOPS, +#ifdef CONFIG_PROC_FS + .show_fdinfo = drm_show_fdinfo, +#endif +}; + +static int ivpu_gem_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file_priv, + u32 handle, u32 flags, int *prime_fd) +{ + struct drm_gem_object *obj; + + obj = drm_gem_object_lookup(file_priv, handle); + if (!obj) + return -ENOENT; + + if (drm_gem_is_imported(obj)) { + /* Do not allow re-exporting */ + drm_gem_object_put(obj); + return -EOPNOTSUPP; + } + + drm_gem_object_put(obj); + + return drm_gem_prime_handle_to_fd(dev, file_priv, handle, flags, prime_fd); +} + +static const struct drm_driver driver = { + .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, + + .open = ivpu_open, + .postclose = ivpu_postclose, + + .gem_create_object = ivpu_gem_create_object, + .gem_prime_import = ivpu_gem_prime_import, + .prime_handle_to_fd = ivpu_gem_prime_handle_to_fd, + + .ioctls = ivpu_drm_ioctls, + .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls), + .fops = &ivpu_fops, +#ifdef CONFIG_PROC_FS + .show_fdinfo = drm_show_memory_stats, +#endif + + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + + .major = 1, +}; + +static int ivpu_irq_init(struct ivpu_device *vdev) +{ + struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); + int ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX); + if (ret < 0) { + ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret); + return ret; + } + + INIT_WORK(&vdev->irq_ipc_work, ivpu_ipc_irq_work_fn); + INIT_WORK(&vdev->irq_dct_work, ivpu_pm_irq_dct_work_fn); + INIT_WORK(&vdev->context_abort_work, ivpu_context_abort_work_fn); + + ivpu_irq_handlers_init(vdev); + + vdev->irq = pci_irq_vector(pdev, 0); + + ret = devm_request_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler, + IRQF_NO_AUTOEN, DRIVER_NAME, vdev); + if (ret) + ivpu_err(vdev, "Failed to request an IRQ %d\n", ret); + + return ret; +} + +static int ivpu_pci_init(struct ivpu_device *vdev) +{ + struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); + struct resource *bar0 = &pdev->resource[0]; + struct resource *bar4 = &pdev->resource[4]; + int ret; + + ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0); + vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0); + if (IS_ERR(vdev->regv)) { + ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv); + return PTR_ERR(vdev->regv); + } + + ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4); + vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4); + if (IS_ERR(vdev->regb)) { + ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb); + return PTR_ERR(vdev->regb); + } + + ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(vdev->hw->dma_bits)); + if (ret) { + ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret); + return ret; + } + dma_set_max_seg_size(vdev->drm.dev, UINT_MAX); + + /* Clear any pending errors */ + pcie_capability_clear_word(pdev, PCI_EXP_DEVSTA, 0x3f); + + /* NPU does not require 10m D3hot delay */ + pdev->d3hot_delay = 0; + + ret = pcim_enable_device(pdev); + if (ret) { + ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret); + return ret; + } + + pci_set_master(pdev); + + return 0; +} + +static int ivpu_dev_init(struct ivpu_device *vdev) +{ + int ret; + + vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL); + if (!vdev->hw) + return -ENOMEM; + + vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL); + if (!vdev->mmu) + return -ENOMEM; + + vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL); + if (!vdev->fw) + return -ENOMEM; + + vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL); + if (!vdev->ipc) + return -ENOMEM; + + vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL); + if (!vdev->pm) + return -ENOMEM; + + if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX) + vdev->hw->dma_bits = 48; + else + vdev->hw->dma_bits = 38; + + vdev->platform = IVPU_PLATFORM_INVALID; + vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID; + vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID; + atomic64_set(&vdev->unique_id_counter, 0); + atomic_set(&vdev->job_timeout_counter, 0); + atomic_set(&vdev->faults_detected, 0); + xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); + xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1); + xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1); + INIT_LIST_HEAD(&vdev->bo_list); + hash_init(vdev->user_limits); + + vdev->db_limit.min = IVPU_MIN_DB; + vdev->db_limit.max = IVPU_MAX_DB; + + ret = drmm_mutex_init(&vdev->drm, &vdev->context_list_lock); + if (ret) + goto err_xa_destroy; + + ret = drmm_mutex_init(&vdev->drm, &vdev->user_limits_lock); + if (ret) + goto err_xa_destroy; + + ret = drmm_mutex_init(&vdev->drm, &vdev->submitted_jobs_lock); + if (ret) + goto err_xa_destroy; + + ret = drmm_mutex_init(&vdev->drm, &vdev->bo_list_lock); + if (ret) + goto err_xa_destroy; + + ret = ivpu_pci_init(vdev); + if (ret) + goto err_xa_destroy; + + ret = ivpu_irq_init(vdev); + if (ret) + goto err_xa_destroy; + + /* Init basic HW info based on buttress registers which are accessible before power up */ + ret = ivpu_hw_init(vdev); + if (ret) + goto err_xa_destroy; + + /* Power up early so the rest of init code can access VPU registers */ + ret = ivpu_hw_power_up(vdev); + if (ret) + goto err_shutdown; + + ivpu_mmu_global_context_init(vdev); + + ret = ivpu_mmu_init(vdev); + if (ret) + goto err_mmu_gctx_fini; + + ret = ivpu_mmu_reserved_context_init(vdev); + if (ret) + goto err_mmu_gctx_fini; + + ret = ivpu_fw_init(vdev); + if (ret) + goto err_mmu_rctx_fini; + + ret = ivpu_ipc_init(vdev); + if (ret) + goto err_fw_fini; + + ivpu_pm_init(vdev); + + ret = ivpu_boot(vdev); + if (ret) + goto err_ipc_fini; + + ivpu_job_done_consumer_init(vdev); + ivpu_pm_enable(vdev); + + return 0; + +err_ipc_fini: + ivpu_ipc_fini(vdev); +err_fw_fini: + ivpu_fw_fini(vdev); +err_mmu_rctx_fini: + ivpu_mmu_reserved_context_fini(vdev); +err_mmu_gctx_fini: + ivpu_mmu_global_context_fini(vdev); +err_shutdown: + ivpu_shutdown(vdev); +err_xa_destroy: + xa_destroy(&vdev->db_xa); + xa_destroy(&vdev->submitted_jobs_xa); + xa_destroy(&vdev->context_xa); + return ret; +} + +static void ivpu_bo_unbind_all_user_contexts(struct ivpu_device *vdev) +{ + struct ivpu_file_priv *file_priv; + unsigned long ctx_id; + + mutex_lock(&vdev->context_list_lock); + + xa_for_each(&vdev->context_xa, ctx_id, file_priv) + file_priv_unbind(vdev, file_priv); + + mutex_unlock(&vdev->context_list_lock); +} + +static void ivpu_dev_fini(struct ivpu_device *vdev) +{ + ivpu_jobs_abort_all(vdev); + ivpu_pm_disable_recovery(vdev); + ivpu_pm_disable(vdev); + ivpu_prepare_for_reset(vdev); + ivpu_shutdown(vdev); + + ivpu_ms_cleanup_all(vdev); + ivpu_job_done_consumer_fini(vdev); + ivpu_bo_unbind_all_user_contexts(vdev); + + ivpu_ipc_fini(vdev); + ivpu_fw_fini(vdev); + ivpu_mmu_reserved_context_fini(vdev); + ivpu_mmu_global_context_fini(vdev); + + drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->db_xa)); + xa_destroy(&vdev->db_xa); + drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa)); + xa_destroy(&vdev->submitted_jobs_xa); + drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa)); + xa_destroy(&vdev->context_xa); +} + +static struct pci_device_id ivpu_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_MTL) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_ARL) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_LNL) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PTL_P) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_WCL) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_NVL) }, + {} +}; +MODULE_DEVICE_TABLE(pci, ivpu_pci_ids); + +static int ivpu_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct ivpu_device *vdev; + int ret; + + vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm); + if (IS_ERR(vdev)) + return PTR_ERR(vdev); + + pci_set_drvdata(pdev, vdev); + + ret = ivpu_dev_init(vdev); + if (ret) + return ret; + + ivpu_debugfs_init(vdev); + ivpu_sysfs_init(vdev); + + ret = drm_dev_register(&vdev->drm, 0); + if (ret) { + dev_err(&pdev->dev, "Failed to register DRM device: %d\n", ret); + ivpu_dev_fini(vdev); + } + + return ret; +} + +static void ivpu_remove(struct pci_dev *pdev) +{ + struct ivpu_device *vdev = pci_get_drvdata(pdev); + + drm_dev_unplug(&vdev->drm); + ivpu_dev_fini(vdev); +} + +static const struct dev_pm_ops ivpu_drv_pci_pm = { + SET_SYSTEM_SLEEP_PM_OPS(ivpu_pm_suspend_cb, ivpu_pm_resume_cb) + SET_RUNTIME_PM_OPS(ivpu_pm_runtime_suspend_cb, ivpu_pm_runtime_resume_cb, NULL) +}; + +static const struct pci_error_handlers ivpu_drv_pci_err = { + .reset_prepare = ivpu_pm_reset_prepare_cb, + .reset_done = ivpu_pm_reset_done_cb, +}; + +static struct pci_driver ivpu_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = ivpu_pci_ids, + .probe = ivpu_probe, + .remove = ivpu_remove, + .driver = { + .pm = &ivpu_drv_pci_pm, + }, + .err_handler = &ivpu_drv_pci_err, +}; + +module_pci_driver(ivpu_pci_driver); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); +MODULE_VERSION(DRIVER_VERSION_STR); diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h new file mode 100644 index 000000000..9eefbbb7b --- /dev/null +++ b/drivers/accel/ivpu/ivpu_drv.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#ifndef __IVPU_DRV_H__ +#define __IVPU_DRV_H__ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ivpu_mmu_context.h" +#include "ivpu_ipc.h" + +#define DRIVER_NAME "intel_vpu" +#define DRIVER_DESC "Driver for Intel NPU (Neural Processing Unit)" + +#define PCI_DEVICE_ID_MTL 0x7d1d +#define PCI_DEVICE_ID_ARL 0xad1d +#define PCI_DEVICE_ID_LNL 0x643e +#define PCI_DEVICE_ID_PTL_P 0xb03e +#define PCI_DEVICE_ID_WCL 0xfd3e +#define PCI_DEVICE_ID_NVL 0xd71d + +#define IVPU_HW_IP_37XX 37 +#define IVPU_HW_IP_40XX 40 +#define IVPU_HW_IP_50XX 50 +#define IVPU_HW_IP_60XX 60 + +#define IVPU_HW_IP_REV_LNL_B0 4 +#define IVPU_HW_IP_REV_NVL_A0 0 + +#define IVPU_HW_BTRS_MTL 1 +#define IVPU_HW_BTRS_LNL 2 + +#define IVPU_GLOBAL_CONTEXT_MMU_SSID 0 +/* SSID 1 is used by the VPU to represent reserved context */ +#define IVPU_RESERVED_CONTEXT_MMU_SSID 1 +#define IVPU_USER_CONTEXT_MIN_SSID 2 +#define IVPU_USER_CONTEXT_MAX_SSID (IVPU_USER_CONTEXT_MIN_SSID + 128) + +#define IVPU_MIN_DB 1 +#define IVPU_MAX_DB 255 + +#define IVPU_JOB_ID_JOB_MASK GENMASK(7, 0) +#define IVPU_JOB_ID_CONTEXT_MASK GENMASK(31, 8) + +#define IVPU_CMDQ_MIN_ID 1 +#define IVPU_CMDQ_MAX_ID 255 + +#define IVPU_PLATFORM_SILICON 0 +#define IVPU_PLATFORM_SIMICS 2 +#define IVPU_PLATFORM_FPGA 3 +#define IVPU_PLATFORM_HSLE 4 +#define IVPU_PLATFORM_INVALID 8 + +#define IVPU_SCHED_MODE_AUTO -1 + +#define IVPU_DBG_REG BIT(0) +#define IVPU_DBG_IRQ BIT(1) +#define IVPU_DBG_MMU BIT(2) +#define IVPU_DBG_FILE BIT(3) +#define IVPU_DBG_MISC BIT(4) +#define IVPU_DBG_FW_BOOT BIT(5) +#define IVPU_DBG_PM BIT(6) +#define IVPU_DBG_IPC BIT(7) +#define IVPU_DBG_BO BIT(8) +#define IVPU_DBG_JOB BIT(9) +#define IVPU_DBG_JSM BIT(10) +#define IVPU_DBG_KREF BIT(11) +#define IVPU_DBG_RPM BIT(12) +#define IVPU_DBG_MMU_MAP BIT(13) +#define IVPU_DBG_IOCTL BIT(14) + +#define ivpu_err(vdev, fmt, ...) \ + drm_err(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__) + +#define ivpu_err_ratelimited(vdev, fmt, ...) \ + drm_err_ratelimited(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__) + +#define ivpu_warn(vdev, fmt, ...) \ + drm_warn(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__) + +#define ivpu_warn_ratelimited(vdev, fmt, ...) \ + drm_err_ratelimited(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__) + +#define ivpu_info(vdev, fmt, ...) drm_info(&(vdev)->drm, fmt, ##__VA_ARGS__) + +#define ivpu_dbg(vdev, type, fmt, args...) do { \ + if (unlikely(IVPU_DBG_##type & ivpu_dbg_mask)) \ + dev_dbg((vdev)->drm.dev, "[%s] " fmt, #type, ##args); \ +} while (0) + +#define IVPU_WA(wa_name) (vdev->wa.wa_name) + +#define IVPU_PRINT_WA(wa_name) do { \ + if (IVPU_WA(wa_name)) \ + ivpu_dbg(vdev, MISC, "Using WA: " #wa_name "\n"); \ +} while (0) + +struct ivpu_wa_table { + bool punit_disabled; + bool clear_runtime_mem; + bool interrupt_clear_with_0; + bool disable_clock_relinquish; + bool wp0_during_power_up; + bool disable_d0i2; +}; + +struct ivpu_hw_info; +struct ivpu_mmu_info; +struct ivpu_fw_info; +struct ivpu_ipc_info; +struct ivpu_pm_info; + +struct ivpu_user_limits { + struct hlist_node hash_node; + struct ivpu_device *vdev; + struct kref ref; + u32 max_ctx_count; + u32 max_db_count; + u32 uid; + atomic_t db_count; +}; + +struct ivpu_device { + struct drm_device drm; + void __iomem *regb; + void __iomem *regv; + u32 platform; + u32 irq; + + struct ivpu_wa_table wa; + struct ivpu_hw_info *hw; + struct ivpu_mmu_info *mmu; + struct ivpu_fw_info *fw; + struct ivpu_ipc_info *ipc; + struct ivpu_pm_info *pm; + + struct ivpu_mmu_context gctx; + struct ivpu_mmu_context rctx; + struct mutex context_list_lock; /* Protects user context addition/removal */ + struct xarray context_xa; + struct xa_limit context_xa_limit; + DECLARE_HASHTABLE(user_limits, 8); + struct mutex user_limits_lock; /* Protects user_limits */ + + struct xarray db_xa; + struct xa_limit db_limit; + u32 db_next; + + struct work_struct irq_ipc_work; + struct work_struct irq_dct_work; + struct work_struct context_abort_work; + + struct mutex bo_list_lock; /* Protects bo_list */ + struct list_head bo_list; + + struct mutex submitted_jobs_lock; /* Protects submitted_jobs */ + struct xarray submitted_jobs_xa; + struct ivpu_ipc_consumer job_done_consumer; + atomic_t job_timeout_counter; + atomic_t faults_detected; + + atomic64_t unique_id_counter; + + ktime_t busy_start_ts; + ktime_t busy_time; + + struct { + int boot; + int jsm; + int tdr; + int inference; + int autosuspend; + int d0i3_entry_msg; + int state_dump_msg; + } timeout; +}; + +/* + * file_priv has its own refcount (ref) that allows user space to close the fd + * without blocking even if VPU is still processing some jobs. + */ +struct ivpu_file_priv { + struct kref ref; + struct ivpu_device *vdev; + struct mutex lock; /* Protects cmdq */ + struct xarray cmdq_xa; + struct ivpu_mmu_context ctx; + struct mutex ms_lock; /* Protects ms_instance_list, ms_info_bo */ + struct list_head ms_instance_list; + struct ivpu_bo *ms_info_bo; + struct xa_limit job_limit; + struct ivpu_user_limits *user_limits; + u32 job_id_next; + struct xa_limit cmdq_limit; + u32 cmdq_id_next; + bool has_mmu_faults; + bool bound; + bool aborted; +}; + +extern int ivpu_dbg_mask; +extern u8 ivpu_pll_min_ratio; +extern u8 ivpu_pll_max_ratio; +extern int ivpu_sched_mode; +extern bool ivpu_disable_mmu_cont_pages; +extern bool ivpu_force_snoop; + +#define IVPU_TEST_MODE_FW_TEST BIT(0) +#define IVPU_TEST_MODE_NULL_HW BIT(1) +#define IVPU_TEST_MODE_NULL_SUBMISSION BIT(2) +#define IVPU_TEST_MODE_MIP_DISABLE BIT(6) +#define IVPU_TEST_MODE_DISABLE_TIMEOUTS BIT(8) +#define IVPU_TEST_MODE_TURBO_ENABLE BIT(9) +#define IVPU_TEST_MODE_TURBO_DISABLE BIT(10) +#define IVPU_TEST_MODE_CLK_RELINQ_DISABLE BIT(11) +#define IVPU_TEST_MODE_CLK_RELINQ_ENABLE BIT(12) +#define IVPU_TEST_MODE_D0I2_DISABLE BIT(13) +extern int ivpu_test_mode; + +struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv); +void ivpu_file_priv_put(struct ivpu_file_priv **link); + +int ivpu_boot(struct ivpu_device *vdev); +int ivpu_shutdown(struct ivpu_device *vdev); +void ivpu_prepare_for_reset(struct ivpu_device *vdev); +bool ivpu_is_capable(struct ivpu_device *vdev, u32 capability); + +static inline u8 ivpu_revision(struct ivpu_device *vdev) +{ + return to_pci_dev(vdev->drm.dev)->revision; +} + +static inline u16 ivpu_device_id(struct ivpu_device *vdev) +{ + return to_pci_dev(vdev->drm.dev)->device; +} + +static inline int ivpu_hw_ip_gen(struct ivpu_device *vdev) +{ + switch (ivpu_device_id(vdev)) { + case PCI_DEVICE_ID_MTL: + case PCI_DEVICE_ID_ARL: + return IVPU_HW_IP_37XX; + case PCI_DEVICE_ID_LNL: + return IVPU_HW_IP_40XX; + case PCI_DEVICE_ID_PTL_P: + case PCI_DEVICE_ID_WCL: + return IVPU_HW_IP_50XX; + case PCI_DEVICE_ID_NVL: + return IVPU_HW_IP_60XX; + default: + dump_stack(); + ivpu_err(vdev, "Unknown NPU IP generation\n"); + return 0; + } +} + +static inline int ivpu_hw_btrs_gen(struct ivpu_device *vdev) +{ + switch (ivpu_device_id(vdev)) { + case PCI_DEVICE_ID_MTL: + case PCI_DEVICE_ID_ARL: + return IVPU_HW_BTRS_MTL; + case PCI_DEVICE_ID_LNL: + case PCI_DEVICE_ID_PTL_P: + case PCI_DEVICE_ID_WCL: + case PCI_DEVICE_ID_NVL: + return IVPU_HW_BTRS_LNL; + default: + dump_stack(); + ivpu_err(vdev, "Unknown buttress generation\n"); + return 0; + } +} + +static inline struct ivpu_device *to_ivpu_device(struct drm_device *dev) +{ + return container_of(dev, struct ivpu_device, drm); +} + +static inline u32 ivpu_get_context_count(struct ivpu_device *vdev) +{ + struct xa_limit ctx_limit = vdev->context_xa_limit; + + return (ctx_limit.max - ctx_limit.min + 1); +} + +static inline u32 ivpu_get_doorbell_count(struct ivpu_device *vdev) +{ + struct xa_limit db_limit = vdev->db_limit; + + return (db_limit.max - db_limit.min + 1); +} + +static inline u32 ivpu_get_platform(struct ivpu_device *vdev) +{ + WARN_ON_ONCE(vdev->platform == IVPU_PLATFORM_INVALID); + return vdev->platform; +} + +static inline bool ivpu_is_silicon(struct ivpu_device *vdev) +{ + return ivpu_get_platform(vdev) == IVPU_PLATFORM_SILICON; +} + +static inline bool ivpu_is_simics(struct ivpu_device *vdev) +{ + return ivpu_get_platform(vdev) == IVPU_PLATFORM_SIMICS; +} + +static inline bool ivpu_is_fpga(struct ivpu_device *vdev) +{ + return ivpu_get_platform(vdev) == IVPU_PLATFORM_FPGA || + ivpu_get_platform(vdev) == IVPU_PLATFORM_HSLE; +} + +static inline bool ivpu_is_force_snoop_enabled(struct ivpu_device *vdev) +{ + return ivpu_force_snoop; +} + +#endif /* __IVPU_DRV_H__ */ diff --git a/drivers/accel/ivpu/ivpu_fw.c b/drivers/accel/ivpu/ivpu_fw.c new file mode 100644 index 000000000..5a8f7dfb5 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_fw.c @@ -0,0 +1,724 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include +#include +#include +#include + +#include "vpu_boot_api.h" +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_fw_log.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_ipc.h" +#include "ivpu_pm.h" + +#define FW_SHAVE_NN_MAX_SIZE SZ_2M +#define FW_FILE_IMAGE_OFFSET (VPU_FW_HEADER_SIZE + FW_VERSION_HEADER_SIZE) +#define FW_PREEMPT_BUF_MIN_SIZE SZ_4K +#define FW_PREEMPT_BUF_MAX_SIZE SZ_32M + +#define WATCHDOG_MSS_REDIRECT 32 +#define WATCHDOG_NCE_REDIRECT 33 + +#define ADDR_TO_L2_CACHE_CFG(addr) ((addr) >> 31) + +/* Check if FW API is compatible with the driver */ +#define IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, name, min_major) \ + ivpu_fw_check_api(vdev, fw_hdr, #name, \ + VPU_##name##_API_VER_INDEX, \ + VPU_##name##_API_VER_MAJOR, \ + VPU_##name##_API_VER_MINOR, min_major) + +/* Check if API version is lower that the given version */ +#define IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, name, major, minor) \ + ivpu_fw_check_api_ver_lt(vdev, fw_hdr, #name, VPU_##name##_API_VER_INDEX, major, minor) + +#define IVPU_FOCUS_PRESENT_TIMER_MS 1000 + +static char *ivpu_firmware; +#if IS_ENABLED(CONFIG_DRM_ACCEL_IVPU_DEBUG) +module_param_named_unsafe(firmware, ivpu_firmware, charp, 0644); +MODULE_PARM_DESC(firmware, "NPU firmware binary in /lib/firmware/.."); +#endif + +static struct { + int gen; + const char *name; +} fw_names[] = { + { IVPU_HW_IP_37XX, "intel/vpu/vpu_37xx_v1.bin" }, + { IVPU_HW_IP_37XX, "intel/vpu/vpu_37xx_v0.0.bin" }, + { IVPU_HW_IP_40XX, "intel/vpu/vpu_40xx_v1.bin" }, + { IVPU_HW_IP_40XX, "intel/vpu/vpu_40xx_v0.0.bin" }, + { IVPU_HW_IP_50XX, "intel/vpu/vpu_50xx_v1.bin" }, + { IVPU_HW_IP_50XX, "intel/vpu/vpu_50xx_v0.0.bin" }, + { IVPU_HW_IP_60XX, "intel/vpu/vpu_60xx_v1.bin" }, +}; + +/* Production fw_names from the table above */ +MODULE_FIRMWARE("intel/vpu/vpu_37xx_v1.bin"); +MODULE_FIRMWARE("intel/vpu/vpu_40xx_v1.bin"); +MODULE_FIRMWARE("intel/vpu/vpu_50xx_v1.bin"); +MODULE_FIRMWARE("intel/vpu/vpu_60xx_v1.bin"); + +static int ivpu_fw_request(struct ivpu_device *vdev) +{ + int ret = -ENOENT; + int i; + + if (ivpu_firmware) { + ret = request_firmware(&vdev->fw->file, ivpu_firmware, vdev->drm.dev); + if (!ret) + vdev->fw->name = ivpu_firmware; + return ret; + } + + for (i = 0; i < ARRAY_SIZE(fw_names); i++) { + if (fw_names[i].gen != ivpu_hw_ip_gen(vdev)) + continue; + + ret = firmware_request_nowarn(&vdev->fw->file, fw_names[i].name, vdev->drm.dev); + if (!ret) { + vdev->fw->name = fw_names[i].name; + return 0; + } + } + + ivpu_err(vdev, "Failed to request firmware: %d\n", ret); + return ret; +} + +static int +ivpu_fw_check_api(struct ivpu_device *vdev, const struct vpu_firmware_header *fw_hdr, + const char *str, int index, u16 expected_major, u16 expected_minor, + u16 min_major) +{ + u16 major = (u16)(fw_hdr->api_version[index] >> 16); + u16 minor = (u16)(fw_hdr->api_version[index]); + + if (major < min_major) { + ivpu_err(vdev, "Incompatible FW %s API version: %d.%d, required %d.0 or later\n", + str, major, minor, min_major); + return -EINVAL; + } + if (major != expected_major) { + ivpu_warn(vdev, "Major FW %s API version different: %d.%d (expected %d.%d)\n", + str, major, minor, expected_major, expected_minor); + } + ivpu_dbg(vdev, FW_BOOT, "FW %s API version: %d.%d (expected %d.%d)\n", + str, major, minor, expected_major, expected_minor); + + return 0; +} + +static bool +ivpu_fw_check_api_ver_lt(struct ivpu_device *vdev, const struct vpu_firmware_header *fw_hdr, + const char *str, int index, u16 major, u16 minor) +{ + u16 fw_major = (u16)(fw_hdr->api_version[index] >> 16); + u16 fw_minor = (u16)(fw_hdr->api_version[index]); + + if (fw_major < major || (fw_major == major && fw_minor < minor)) + return true; + + return false; +} + +bool ivpu_is_within_range(u64 addr, size_t size, struct ivpu_addr_range *range) +{ + u64 addr_end; + + if (!range || check_add_overflow(addr, size, &addr_end)) + return false; + + if (addr < range->start || addr_end > range->end) + return false; + + return true; +} + +static u32 +ivpu_fw_sched_mode_select(struct ivpu_device *vdev, const struct vpu_firmware_header *fw_hdr) +{ + if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_60XX && + ivpu_sched_mode == VPU_SCHEDULING_MODE_OS) { + ivpu_warn(vdev, "OS sched mode is not supported, using HW mode\n"); + return VPU_SCHEDULING_MODE_HW; + } + + if (ivpu_sched_mode != IVPU_SCHED_MODE_AUTO) + return ivpu_sched_mode; + + if (IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, JSM, 3, 24)) + return VPU_SCHEDULING_MODE_OS; + + return VPU_SCHEDULING_MODE_HW; +} + +static void +ivpu_preemption_config_parse(struct ivpu_device *vdev, const struct vpu_firmware_header *fw_hdr) +{ + struct ivpu_fw_info *fw = vdev->fw; + u32 primary_preempt_buf_size, secondary_preempt_buf_size; + + if (fw_hdr->preemption_buffer_1_max_size) + primary_preempt_buf_size = fw_hdr->preemption_buffer_1_max_size; + else + primary_preempt_buf_size = fw_hdr->preemption_buffer_1_size; + + if (fw_hdr->preemption_buffer_2_max_size) + secondary_preempt_buf_size = fw_hdr->preemption_buffer_2_max_size; + else + secondary_preempt_buf_size = fw_hdr->preemption_buffer_2_size; + + ivpu_dbg(vdev, FW_BOOT, "Preemption buffer size, primary: %u, secondary: %u\n", + primary_preempt_buf_size, secondary_preempt_buf_size); + + if (primary_preempt_buf_size < FW_PREEMPT_BUF_MIN_SIZE || + secondary_preempt_buf_size < FW_PREEMPT_BUF_MIN_SIZE) { + ivpu_warn(vdev, "Preemption buffers size too small\n"); + return; + } + + if (primary_preempt_buf_size > FW_PREEMPT_BUF_MAX_SIZE || + secondary_preempt_buf_size > FW_PREEMPT_BUF_MAX_SIZE) { + ivpu_warn(vdev, "Preemption buffers size too big\n"); + return; + } + + if (fw->sched_mode != VPU_SCHEDULING_MODE_HW) + return; + + if (ivpu_test_mode & IVPU_TEST_MODE_MIP_DISABLE) + return; + + vdev->fw->primary_preempt_buf_size = ALIGN(primary_preempt_buf_size, PAGE_SIZE); + vdev->fw->secondary_preempt_buf_size = ALIGN(secondary_preempt_buf_size, PAGE_SIZE); +} + +static int ivpu_fw_parse(struct ivpu_device *vdev) +{ + struct ivpu_fw_info *fw = vdev->fw; + const struct vpu_firmware_header *fw_hdr = (const void *)fw->file->data; + struct ivpu_addr_range fw_image_range; + u64 boot_params_addr, boot_params_size; + u64 fw_version_addr, fw_version_size; + u64 runtime_addr, runtime_size; + u64 image_load_addr, image_size; + + if (fw->file->size <= FW_FILE_IMAGE_OFFSET) { + ivpu_err(vdev, "Firmware file is too small: %zu\n", fw->file->size); + return -EINVAL; + } + + if (fw_hdr->header_version != VPU_FW_HEADER_VERSION) { + ivpu_err(vdev, "Invalid firmware header version: %u\n", fw_hdr->header_version); + return -EINVAL; + } + + boot_params_addr = fw_hdr->boot_params_load_address; + boot_params_size = SZ_4K; + + if (!ivpu_is_within_range(boot_params_addr, boot_params_size, &vdev->hw->ranges.runtime)) { + ivpu_err(vdev, "Invalid boot params address: 0x%llx\n", boot_params_addr); + return -EINVAL; + } + + fw_version_addr = fw_hdr->firmware_version_load_address; + fw_version_size = ALIGN(fw_hdr->firmware_version_size, SZ_4K); + + if (fw_version_size != SZ_4K) { + ivpu_err(vdev, "Invalid firmware version size: %u\n", + fw_hdr->firmware_version_size); + return -EINVAL; + } + + if (!ivpu_is_within_range(fw_version_addr, fw_version_size, &vdev->hw->ranges.runtime)) { + ivpu_err(vdev, "Invalid firmware version address: 0x%llx\n", fw_version_addr); + return -EINVAL; + } + + runtime_addr = fw_hdr->image_load_address; + runtime_size = fw_hdr->runtime_size - boot_params_size - fw_version_size; + + image_load_addr = fw_hdr->image_load_address; + image_size = fw_hdr->image_size; + + if (!ivpu_is_within_range(runtime_addr, runtime_size, &vdev->hw->ranges.runtime)) { + ivpu_err(vdev, "Invalid firmware runtime address: 0x%llx and size %llu\n", + runtime_addr, runtime_size); + return -EINVAL; + } + + if (FW_FILE_IMAGE_OFFSET + image_size > fw->file->size) { + ivpu_err(vdev, "Invalid image size: %llu\n", image_size); + return -EINVAL; + } + + if (!PAGE_ALIGNED(runtime_addr)) { + ivpu_err(vdev, "Runtime address 0x%llx not page aligned\n", runtime_addr); + return -EINVAL; + } + + if (!PAGE_ALIGNED(runtime_size)) { + ivpu_err(vdev, "Runtime size %llu not page aligned\n", runtime_size); + return -EINVAL; + } + + if (runtime_size < image_size) { + ivpu_err(vdev, "Runtime size too small: %llu, image size: %llu\n", + runtime_size, image_size); + return -EINVAL; + } + + if (!ivpu_is_within_range(image_load_addr, image_size, &vdev->hw->ranges.runtime)) { + ivpu_err(vdev, "Invalid firmware load address: 0x%llx and size %llu\n", + image_load_addr, image_size); + return -EINVAL; + } + + if (ivpu_hw_range_init(vdev, &fw_image_range, image_load_addr, image_size)) + return -EINVAL; + + if (!ivpu_is_within_range(fw_hdr->entry_point, SZ_4K, &fw_image_range)) { + ivpu_err(vdev, "Invalid entry point: 0x%llx\n", fw_hdr->entry_point); + return -EINVAL; + } + + if (fw_hdr->shave_nn_fw_size > FW_SHAVE_NN_MAX_SIZE) { + ivpu_err(vdev, "SHAVE NN firmware is too big: %u\n", fw_hdr->shave_nn_fw_size); + return -EINVAL; + } + + ivpu_dbg(vdev, FW_BOOT, "Header version: 0x%x, format 0x%x\n", + fw_hdr->header_version, fw_hdr->image_format); + + if (!scnprintf(fw->version, sizeof(fw->version), "%s", fw->file->data + VPU_FW_HEADER_SIZE)) + ivpu_warn(vdev, "Missing firmware version\n"); + + ivpu_info(vdev, "Firmware: %s, version: %s\n", fw->name, fw->version); + + if (IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, BOOT, 3)) + return -EINVAL; + if (IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, JSM, 3)) + return -EINVAL; + + fw->boot_params_addr = boot_params_addr; + fw->boot_params_size = boot_params_size; + fw->fw_version_addr = fw_version_addr; + fw->fw_version_size = fw_version_size; + fw->runtime_addr = runtime_addr; + fw->runtime_size = runtime_size; + fw->image_load_offset = image_load_addr - runtime_addr; + fw->image_size = image_size; + fw->shave_nn_size = PAGE_ALIGN(fw_hdr->shave_nn_fw_size); + fw->cold_boot_entry_point = fw_hdr->entry_point; + + fw->trace_level = min_t(u32, ivpu_fw_log_level, IVPU_FW_LOG_FATAL); + fw->trace_destination_mask = VPU_TRACE_DESTINATION_VERBOSE_TRACING; + fw->trace_hw_component_mask = -1; + + fw->dvfs_mode = 0; + + fw->sched_mode = ivpu_fw_sched_mode_select(vdev, fw_hdr); + ivpu_info(vdev, "Scheduler mode: %s\n", fw->sched_mode ? "HW" : "OS"); + + ivpu_preemption_config_parse(vdev, fw_hdr); + ivpu_dbg(vdev, FW_BOOT, "Mid-inference preemption %s supported\n", + ivpu_fw_preempt_buf_size(vdev) ? "is" : "is not"); + + if (fw_hdr->ro_section_start_address && + !ivpu_is_within_range(fw_hdr->ro_section_start_address, fw_hdr->ro_section_size, + &fw_image_range)) { + ivpu_err(vdev, "Invalid read-only section: start address 0x%llx, size %u\n", + fw_hdr->ro_section_start_address, fw_hdr->ro_section_size); + return -EINVAL; + } + + fw->read_only_addr = fw_hdr->ro_section_start_address; + fw->read_only_size = fw_hdr->ro_section_size; + + ivpu_dbg(vdev, FW_BOOT, "Boot params: address 0x%llx, size %llu\n", + fw->boot_params_addr, fw->boot_params_size); + ivpu_dbg(vdev, FW_BOOT, "FW version: address 0x%llx, size %llu\n", + fw->fw_version_addr, fw->fw_version_size); + ivpu_dbg(vdev, FW_BOOT, "Runtime: address 0x%llx, size %u\n", + fw->runtime_addr, fw->runtime_size); + ivpu_dbg(vdev, FW_BOOT, "Image load offset: 0x%llx, size %u\n", + fw->image_load_offset, fw->image_size); + ivpu_dbg(vdev, FW_BOOT, "Read-only section: address 0x%llx, size %u\n", + fw->read_only_addr, fw->read_only_size); + ivpu_dbg(vdev, FW_BOOT, "FW cold boot entry point: 0x%llx\n", fw->cold_boot_entry_point); + ivpu_dbg(vdev, FW_BOOT, "SHAVE NN size: %u\n", fw->shave_nn_size); + + return 0; +} + +static void ivpu_fw_release(struct ivpu_device *vdev) +{ + release_firmware(vdev->fw->file); +} + + +static int ivpu_fw_mem_init(struct ivpu_device *vdev) +{ + struct ivpu_fw_info *fw = vdev->fw; + int log_verb_size; + int ret; + + fw->mem_bp = ivpu_bo_create_runtime(vdev, fw->boot_params_addr, fw->boot_params_size, + DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); + if (!fw->mem_bp) { + ivpu_err(vdev, "Failed to create firmware boot params memory buffer\n"); + return -ENOMEM; + } + + fw->mem_fw_ver = ivpu_bo_create_runtime(vdev, fw->fw_version_addr, fw->fw_version_size, + DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); + if (!fw->mem_fw_ver) { + ivpu_err(vdev, "Failed to create firmware version memory buffer\n"); + ret = -ENOMEM; + goto err_free_bp; + } + + fw->mem = ivpu_bo_create_runtime(vdev, fw->runtime_addr, fw->runtime_size, + DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); + if (!fw->mem) { + ivpu_err(vdev, "Failed to create firmware runtime memory buffer\n"); + ret = -ENOMEM; + goto err_free_fw_ver; + } + + ret = ivpu_mmu_context_set_pages_ro(vdev, &vdev->gctx, fw->read_only_addr, + fw->read_only_size); + if (ret) { + ivpu_err(vdev, "Failed to set firmware image read-only\n"); + goto err_free_fw_mem; + } + + fw->mem_log_crit = ivpu_bo_create_global(vdev, IVPU_FW_CRITICAL_BUFFER_SIZE, + DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE); + if (!fw->mem_log_crit) { + ivpu_err(vdev, "Failed to create critical log buffer\n"); + ret = -ENOMEM; + goto err_free_fw_mem; + } + + if (ivpu_fw_log_level <= IVPU_FW_LOG_INFO) + log_verb_size = IVPU_FW_VERBOSE_BUFFER_LARGE_SIZE; + else + log_verb_size = IVPU_FW_VERBOSE_BUFFER_SMALL_SIZE; + + fw->mem_log_verb = ivpu_bo_create_global(vdev, log_verb_size, + DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE); + if (!fw->mem_log_verb) { + ivpu_err(vdev, "Failed to create verbose log buffer\n"); + ret = -ENOMEM; + goto err_free_log_crit; + } + + if (fw->shave_nn_size) { + fw->mem_shave_nn = ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.shave, + fw->shave_nn_size, DRM_IVPU_BO_WC); + if (!fw->mem_shave_nn) { + ivpu_err(vdev, "Failed to create shavenn buffer\n"); + ret = -ENOMEM; + goto err_free_log_verb; + } + } + + return 0; + +err_free_log_verb: + ivpu_bo_free(fw->mem_log_verb); +err_free_log_crit: + ivpu_bo_free(fw->mem_log_crit); +err_free_fw_mem: + ivpu_bo_free(fw->mem); +err_free_fw_ver: + ivpu_bo_free(fw->mem_fw_ver); +err_free_bp: + ivpu_bo_free(fw->mem_bp); + return ret; +} + +static void ivpu_fw_mem_fini(struct ivpu_device *vdev) +{ + struct ivpu_fw_info *fw = vdev->fw; + + if (fw->mem_shave_nn) { + ivpu_bo_free(fw->mem_shave_nn); + fw->mem_shave_nn = NULL; + } + + ivpu_bo_free(fw->mem_log_verb); + ivpu_bo_free(fw->mem_log_crit); + ivpu_bo_free(fw->mem); + ivpu_bo_free(fw->mem_fw_ver); + ivpu_bo_free(fw->mem_bp); + + fw->mem_log_verb = NULL; + fw->mem_log_crit = NULL; + fw->mem = NULL; + fw->mem_fw_ver = NULL; + fw->mem_bp = NULL; +} + +int ivpu_fw_init(struct ivpu_device *vdev) +{ + int ret; + + ret = ivpu_fw_request(vdev); + if (ret) + return ret; + + ret = ivpu_fw_parse(vdev); + if (ret) + goto err_fw_release; + + ret = ivpu_fw_mem_init(vdev); + if (ret) + goto err_fw_release; + + ivpu_fw_load(vdev); + + return 0; + +err_fw_release: + ivpu_fw_release(vdev); + return ret; +} + +void ivpu_fw_fini(struct ivpu_device *vdev) +{ + ivpu_fw_mem_fini(vdev); + ivpu_fw_release(vdev); +} + +void ivpu_fw_load(struct ivpu_device *vdev) +{ + struct ivpu_fw_info *fw = vdev->fw; + u64 image_end_offset = fw->image_load_offset + fw->image_size; + + memset(ivpu_bo_vaddr(fw->mem), 0, fw->image_load_offset); + memcpy(ivpu_bo_vaddr(fw->mem) + fw->image_load_offset, + fw->file->data + FW_FILE_IMAGE_OFFSET, fw->image_size); + + if (IVPU_WA(clear_runtime_mem)) { + u8 *start = ivpu_bo_vaddr(fw->mem) + image_end_offset; + u64 size = ivpu_bo_size(fw->mem) - image_end_offset; + + memset(start, 0, size); + } + + wmb(); /* Flush WC buffers after writing fw->mem */ +} + +static void ivpu_fw_boot_params_print(struct ivpu_device *vdev, struct vpu_boot_params *boot_params) +{ + ivpu_dbg(vdev, FW_BOOT, "boot_params.magic = 0x%x\n", + boot_params->magic); + ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_id = 0x%x\n", + boot_params->vpu_id); + ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_count = 0x%x\n", + boot_params->vpu_count); + ivpu_dbg(vdev, FW_BOOT, "boot_params.frequency = %u\n", + boot_params->frequency); + ivpu_dbg(vdev, FW_BOOT, "boot_params.perf_clk_frequency = %u\n", + boot_params->perf_clk_frequency); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.ipc_header_area_start = 0x%llx\n", + boot_params->ipc_header_area_start); + ivpu_dbg(vdev, FW_BOOT, "boot_params.ipc_header_area_size = 0x%x\n", + boot_params->ipc_header_area_size); + ivpu_dbg(vdev, FW_BOOT, "boot_params.shared_region_base = 0x%llx\n", + boot_params->shared_region_base); + ivpu_dbg(vdev, FW_BOOT, "boot_params.shared_region_size = 0x%x\n", + boot_params->shared_region_size); + ivpu_dbg(vdev, FW_BOOT, "boot_params.ipc_payload_area_start = 0x%llx\n", + boot_params->ipc_payload_area_start); + ivpu_dbg(vdev, FW_BOOT, "boot_params.ipc_payload_area_size = 0x%x\n", + boot_params->ipc_payload_area_size); + ivpu_dbg(vdev, FW_BOOT, "boot_params.global_aliased_pio_base = 0x%llx\n", + boot_params->global_aliased_pio_base); + ivpu_dbg(vdev, FW_BOOT, "boot_params.global_aliased_pio_size = 0x%x\n", + boot_params->global_aliased_pio_size); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.autoconfig = 0x%x\n", + boot_params->autoconfig); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.cache_defaults[VPU_BOOT_L2_CACHE_CFG_NN].use = 0x%x\n", + boot_params->cache_defaults[VPU_BOOT_L2_CACHE_CFG_NN].use); + ivpu_dbg(vdev, FW_BOOT, "boot_params.cache_defaults[VPU_BOOT_L2_CACHE_CFG_NN].cfg = 0x%x\n", + boot_params->cache_defaults[VPU_BOOT_L2_CACHE_CFG_NN].cfg); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.shave_nn_fw_base = 0x%llx\n", + boot_params->shave_nn_fw_base); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.watchdog_irq_mss = 0x%x\n", + boot_params->watchdog_irq_mss); + ivpu_dbg(vdev, FW_BOOT, "boot_params.watchdog_irq_nce = 0x%x\n", + boot_params->watchdog_irq_nce); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.host_version_id = 0x%x\n", + boot_params->host_version_id); + ivpu_dbg(vdev, FW_BOOT, "boot_params.si_stepping = 0x%x\n", + boot_params->si_stepping); + ivpu_dbg(vdev, FW_BOOT, "boot_params.device_id = 0x%llx\n", + boot_params->device_id); + ivpu_dbg(vdev, FW_BOOT, "boot_params.feature_exclusion = 0x%llx\n", + boot_params->feature_exclusion); + ivpu_dbg(vdev, FW_BOOT, "boot_params.sku = 0x%llx\n", + boot_params->sku); + ivpu_dbg(vdev, FW_BOOT, "boot_params.min_freq_pll_ratio = 0x%x\n", + boot_params->min_freq_pll_ratio); + ivpu_dbg(vdev, FW_BOOT, "boot_params.pn_freq_pll_ratio = 0x%x\n", + boot_params->pn_freq_pll_ratio); + ivpu_dbg(vdev, FW_BOOT, "boot_params.max_freq_pll_ratio = 0x%x\n", + boot_params->max_freq_pll_ratio); + ivpu_dbg(vdev, FW_BOOT, "boot_params.default_trace_level = 0x%x\n", + boot_params->default_trace_level); + ivpu_dbg(vdev, FW_BOOT, "boot_params.tracing_buff_message_format_mask = 0x%llx\n", + boot_params->tracing_buff_message_format_mask); + ivpu_dbg(vdev, FW_BOOT, "boot_params.trace_destination_mask = 0x%x\n", + boot_params->trace_destination_mask); + ivpu_dbg(vdev, FW_BOOT, "boot_params.trace_hw_component_mask = 0x%llx\n", + boot_params->trace_hw_component_mask); + ivpu_dbg(vdev, FW_BOOT, "boot_params.boot_type = 0x%x\n", + boot_params->boot_type); + ivpu_dbg(vdev, FW_BOOT, "boot_params.punit_telemetry_sram_base = 0x%llx\n", + boot_params->punit_telemetry_sram_base); + ivpu_dbg(vdev, FW_BOOT, "boot_params.punit_telemetry_sram_size = 0x%llx\n", + boot_params->punit_telemetry_sram_size); + ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_telemetry_enable = 0x%x\n", + boot_params->vpu_telemetry_enable); + ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_scheduling_mode = 0x%x\n", + boot_params->vpu_scheduling_mode); + ivpu_dbg(vdev, FW_BOOT, "boot_params.dvfs_mode = %u\n", + boot_params->dvfs_mode); + ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_delayed_entry = %d\n", + boot_params->d0i3_delayed_entry); + ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_residency_time_us = %lld\n", + boot_params->d0i3_residency_time_us); + ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_entry_vpu_ts = %llu\n", + boot_params->d0i3_entry_vpu_ts); + ivpu_dbg(vdev, FW_BOOT, "boot_params.system_time_us = %llu\n", + boot_params->system_time_us); + ivpu_dbg(vdev, FW_BOOT, "boot_params.power_profile = 0x%x\n", + boot_params->power_profile); + ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_uses_ecc_mca_signal = 0x%x\n", + boot_params->vpu_uses_ecc_mca_signal); + ivpu_dbg(vdev, FW_BOOT, "boot_params.boot_type = 0x%x\n", boot_params->boot_type); +} + +void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params *boot_params) +{ + struct ivpu_bo *ipc_mem_rx = vdev->ipc->mem_rx; + + /* In case of warm boot only update variable params */ + if (ivpu_fw_is_warm_boot(vdev)) { + boot_params->d0i3_residency_time_us = + ktime_us_delta(ktime_get_boottime(), vdev->hw->d0i3_entry_host_ts); + boot_params->d0i3_entry_vpu_ts = vdev->hw->d0i3_entry_vpu_ts; + boot_params->system_time_us = ktime_to_us(ktime_get_real()); + + ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_residency_time_us = %lld\n", + boot_params->d0i3_residency_time_us); + ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_entry_vpu_ts = %llu\n", + boot_params->d0i3_entry_vpu_ts); + ivpu_dbg(vdev, FW_BOOT, "boot_params.system_time_us = %llu\n", + boot_params->system_time_us); + ivpu_dbg(vdev, FW_BOOT, "boot_params.boot_type = 0x%x\n", boot_params->boot_type); + + boot_params->save_restore_ret_address = 0; + boot_params->boot_type = VPU_BOOT_TYPE_WARMBOOT; + wmb(); /* Flush WC buffers after writing save_restore_ret_address */ + return; + } + + memset(boot_params, 0, sizeof(*boot_params)); + boot_params->boot_type = VPU_BOOT_TYPE_COLDBOOT; + boot_params->magic = VPU_BOOT_PARAMS_MAGIC; + boot_params->vpu_id = to_pci_dev(vdev->drm.dev)->bus->number; + + /* + * This param is a debug firmware feature. It switches default clock + * to higher resolution one for fine-grained and more accurate firmware + * task profiling. + */ + boot_params->perf_clk_frequency = ivpu_hw_profiling_freq_get(vdev); + + /* + * Uncached region of VPU address space, covers IPC buffers, job queues + * and log buffers, programmable to L2$ Uncached by VPU MTRR + */ + boot_params->shared_region_base = vdev->hw->ranges.global.start; + boot_params->shared_region_size = vdev->hw->ranges.global.end - + vdev->hw->ranges.global.start; + + boot_params->ipc_header_area_start = ipc_mem_rx->vpu_addr; + boot_params->ipc_header_area_size = ivpu_bo_size(ipc_mem_rx) / 2; + + boot_params->ipc_payload_area_start = ipc_mem_rx->vpu_addr + ivpu_bo_size(ipc_mem_rx) / 2; + boot_params->ipc_payload_area_size = ivpu_bo_size(ipc_mem_rx) / 2; + + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + boot_params->global_aliased_pio_base = vdev->hw->ranges.user.start; + boot_params->global_aliased_pio_size = ivpu_hw_range_size(&vdev->hw->ranges.user); + } + + /* Allow configuration for L2C_PAGE_TABLE with boot param value */ + boot_params->autoconfig = 1; + + /* Enable L2 cache for first 2GB of high memory */ + boot_params->cache_defaults[VPU_BOOT_L2_CACHE_CFG_NN].use = 1; + boot_params->cache_defaults[VPU_BOOT_L2_CACHE_CFG_NN].cfg = + ADDR_TO_L2_CACHE_CFG(vdev->hw->ranges.shave.start); + + if (vdev->fw->mem_shave_nn) + boot_params->shave_nn_fw_base = vdev->fw->mem_shave_nn->vpu_addr; + + boot_params->watchdog_irq_mss = WATCHDOG_MSS_REDIRECT; + boot_params->watchdog_irq_nce = WATCHDOG_NCE_REDIRECT; + boot_params->si_stepping = ivpu_revision(vdev); + boot_params->device_id = ivpu_device_id(vdev); + boot_params->feature_exclusion = vdev->hw->tile_fuse; + boot_params->sku = vdev->hw->sku; + + boot_params->min_freq_pll_ratio = vdev->hw->pll.min_ratio; + boot_params->pn_freq_pll_ratio = vdev->hw->pll.pn_ratio; + boot_params->max_freq_pll_ratio = vdev->hw->pll.max_ratio; + + boot_params->default_trace_level = vdev->fw->trace_level; + boot_params->tracing_buff_message_format_mask = BIT(VPU_TRACING_FORMAT_STRING); + boot_params->trace_destination_mask = vdev->fw->trace_destination_mask; + boot_params->trace_hw_component_mask = vdev->fw->trace_hw_component_mask; + boot_params->crit_tracing_buff_addr = vdev->fw->mem_log_crit->vpu_addr; + boot_params->crit_tracing_buff_size = ivpu_bo_size(vdev->fw->mem_log_crit); + boot_params->verbose_tracing_buff_addr = vdev->fw->mem_log_verb->vpu_addr; + boot_params->verbose_tracing_buff_size = ivpu_bo_size(vdev->fw->mem_log_verb); + + boot_params->punit_telemetry_sram_base = ivpu_hw_telemetry_offset_get(vdev); + boot_params->punit_telemetry_sram_size = ivpu_hw_telemetry_size_get(vdev); + boot_params->vpu_telemetry_enable = ivpu_hw_telemetry_enable_get(vdev); + boot_params->vpu_scheduling_mode = vdev->fw->sched_mode; + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) + boot_params->vpu_focus_present_timer_ms = IVPU_FOCUS_PRESENT_TIMER_MS; + boot_params->dvfs_mode = vdev->fw->dvfs_mode; + boot_params->d0i3_delayed_entry = 1; + boot_params->d0i3_residency_time_us = 0; + boot_params->d0i3_entry_vpu_ts = 0; + if (IVPU_WA(disable_d0i2)) + boot_params->power_profile |= BIT(1); + boot_params->vpu_uses_ecc_mca_signal = + ivpu_hw_uses_ecc_mca_signal(vdev) ? VPU_BOOT_MCA_ECC_BOTH : 0; + + boot_params->system_time_us = ktime_to_us(ktime_get_real()); + wmb(); /* Flush WC buffers after writing bootparams */ + + ivpu_fw_boot_params_print(vdev, boot_params); +} diff --git a/drivers/accel/ivpu/ivpu_fw.h b/drivers/accel/ivpu/ivpu_fw.h new file mode 100644 index 000000000..d3c410912 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_fw.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2025 Intel Corporation + */ + +#ifndef __IVPU_FW_H__ +#define __IVPU_FW_H__ + +#include "vpu_boot_api.h" +#include "vpu_jsm_api.h" + +#define FW_VERSION_HEADER_SIZE SZ_4K +#define FW_VERSION_STR_SIZE SZ_256 + +struct ivpu_device; +struct ivpu_bo; +struct vpu_boot_params; + +struct ivpu_fw_info { + const struct firmware *file; + const char *name; + char version[FW_VERSION_STR_SIZE]; + struct ivpu_bo *mem_bp; + struct ivpu_bo *mem_fw_ver; + struct ivpu_bo *mem; + struct ivpu_bo *mem_shave_nn; + struct ivpu_bo *mem_log_crit; + struct ivpu_bo *mem_log_verb; + u64 boot_params_addr; + u64 boot_params_size; + u64 fw_version_addr; + u64 fw_version_size; + u64 runtime_addr; + u32 runtime_size; + u64 image_load_offset; + u32 image_size; + u32 shave_nn_size; + u64 warm_boot_entry_point; + u64 cold_boot_entry_point; + u8 last_boot_mode; + u8 next_boot_mode; + u32 trace_level; + u32 trace_destination_mask; + u64 trace_hw_component_mask; + u32 dvfs_mode; + u32 primary_preempt_buf_size; + u32 secondary_preempt_buf_size; + u64 read_only_addr; + u32 read_only_size; + u32 sched_mode; + u64 last_heartbeat; +}; + +bool ivpu_is_within_range(u64 addr, size_t size, struct ivpu_addr_range *range); +int ivpu_fw_init(struct ivpu_device *vdev); +void ivpu_fw_fini(struct ivpu_device *vdev); +void ivpu_fw_load(struct ivpu_device *vdev); +void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params *boot_params); + +static inline bool ivpu_fw_is_warm_boot(struct ivpu_device *vdev) +{ + return vdev->fw->next_boot_mode == VPU_BOOT_TYPE_WARMBOOT; +} + +static inline u32 ivpu_fw_preempt_buf_size(struct ivpu_device *vdev) +{ + return vdev->fw->primary_preempt_buf_size + vdev->fw->secondary_preempt_buf_size; +} + +#endif /* __IVPU_FW_H__ */ diff --git a/drivers/accel/ivpu/ivpu_fw_log.c b/drivers/accel/ivpu/ivpu_fw_log.c new file mode 100644 index 000000000..716467aa3 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_fw_log.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include "vpu_boot_api.h" +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_fw_log.h" +#include "ivpu_gem.h" + +#define IVPU_FW_LOG_LINE_LENGTH 256 + +unsigned int ivpu_fw_log_level = IVPU_FW_LOG_ERROR; +module_param_named(fw_log_level, ivpu_fw_log_level, uint, 0444); +MODULE_PARM_DESC(fw_log_level, + "NPU firmware default log level: debug=" __stringify(IVPU_FW_LOG_DEBUG) + " info=" __stringify(IVPU_FW_LOG_INFO) + " warn=" __stringify(IVPU_FW_LOG_WARN) + " error=" __stringify(IVPU_FW_LOG_ERROR) + " fatal=" __stringify(IVPU_FW_LOG_FATAL)); + +static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *offset, + struct vpu_tracing_buffer_header **out_log) +{ + struct vpu_tracing_buffer_header *log; + + if ((*offset + sizeof(*log)) > ivpu_bo_size(bo)) + return -EINVAL; + + log = ivpu_bo_vaddr(bo) + *offset; + + if (log->vpu_canary_start != VPU_TRACING_BUFFER_CANARY) + return -EINVAL; + + if (log->header_size < sizeof(*log) || log->header_size > 1024) { + ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", log->header_size); + return -EINVAL; + } + if (log->size < log->header_size) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + return -EINVAL; + } + if ((char *)log + log->size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + return -EINVAL; + } + + *out_log = log; + *offset += log->size; + + ivpu_dbg(vdev, FW_BOOT, + "FW log name \"%s\", write offset 0x%x size 0x%x, wrap count %d, hdr version %d size %d format %d, alignment %d", + log->name, log->write_index, log->size, log->wrap_count, log->header_version, + log->header_size, log->format, log->alignment); + + return 0; +} + +static void fw_log_print_lines(char *buffer, u32 size, struct drm_printer *p) +{ + char line[IVPU_FW_LOG_LINE_LENGTH]; + u32 index = 0; + + if (!size || !buffer) + return; + + while (size--) { + if (*buffer == '\n' || *buffer == 0) { + line[index] = 0; + if (index != 0) + drm_printf(p, "%s\n", line); + index = 0; + buffer++; + continue; + } + if (index == IVPU_FW_LOG_LINE_LENGTH - 1) { + line[index] = 0; + index = 0; + drm_printf(p, "%s\n", line); + } + if (*buffer != '\r' && (isprint(*buffer) || iscntrl(*buffer))) + line[index++] = *buffer; + buffer++; + } + line[index] = 0; + if (index != 0) + drm_printf(p, "%s", line); +} + +static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const char *prefix, + bool only_new_msgs, struct drm_printer *p) +{ + char *log_data = (void *)log + log->header_size; + u32 data_size = log->size - log->header_size; + u32 log_start = only_new_msgs ? READ_ONCE(log->read_index) : 0; + u32 log_end = READ_ONCE(log->write_index); + + if (log_start >= data_size) + log_start = 0; + if (log_end > data_size) + log_end = data_size; + + if (log->wrap_count == log->read_wrap_count) { + if (log_end <= log_start) { + drm_printf(p, "==== %s \"%s\" log empty ====\n", prefix, log->name); + return; + } + } else if (log->wrap_count == log->read_wrap_count + 1) { + if (log_end > log_start) + log_start = log_end; + } else { + log_start = log_end; + } + + drm_printf(p, "==== %s \"%s\" log start ====\n", prefix, log->name); + if (log_end > log_start) { + fw_log_print_lines(log_data + log_start, log_end - log_start, p); + } else { + fw_log_print_lines(log_data + log_start, data_size - log_start, p); + fw_log_print_lines(log_data, log_end, p); + } + drm_printf(p, "\n\x1b[0m"); /* add new line and clear formatting */ + drm_printf(p, "==== %s \"%s\" log end ====\n", prefix, log->name); +} + +static void +fw_log_print_all_in_bo(struct ivpu_device *vdev, const char *name, + struct ivpu_bo *bo, bool only_new_msgs, struct drm_printer *p) +{ + struct vpu_tracing_buffer_header *log; + u32 next = 0; + + while (fw_log_from_bo(vdev, bo, &next, &log) == 0) + fw_log_print_buffer(log, name, only_new_msgs, p); +} + +void ivpu_fw_log_print(struct ivpu_device *vdev, bool only_new_msgs, struct drm_printer *p) +{ + fw_log_print_all_in_bo(vdev, "NPU critical", vdev->fw->mem_log_crit, only_new_msgs, p); + fw_log_print_all_in_bo(vdev, "NPU verbose", vdev->fw->mem_log_verb, only_new_msgs, p); +} + +void ivpu_fw_log_mark_read(struct ivpu_device *vdev) +{ + struct vpu_tracing_buffer_header *log; + u32 next; + + next = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &log) == 0) { + log->read_index = READ_ONCE(log->write_index); + log->read_wrap_count = READ_ONCE(log->wrap_count); + } + + next = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &log) == 0) { + log->read_index = READ_ONCE(log->write_index); + log->read_wrap_count = READ_ONCE(log->wrap_count); + } +} + +void ivpu_fw_log_reset(struct ivpu_device *vdev) +{ + struct vpu_tracing_buffer_header *log; + u32 next; + + next = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &log) == 0) { + log->read_index = 0; + log->read_wrap_count = 0; + } + + next = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &log) == 0) { + log->read_index = 0; + log->read_wrap_count = 0; + } +} diff --git a/drivers/accel/ivpu/ivpu_fw_log.h b/drivers/accel/ivpu/ivpu_fw_log.h new file mode 100644 index 000000000..8bb528a73 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_fw_log.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __IVPU_FW_LOG_H__ +#define __IVPU_FW_LOG_H__ + +#include + +#include "ivpu_drv.h" + +#define IVPU_FW_LOG_DEFAULT 0 +#define IVPU_FW_LOG_DEBUG 1 +#define IVPU_FW_LOG_INFO 2 +#define IVPU_FW_LOG_WARN 3 +#define IVPU_FW_LOG_ERROR 4 +#define IVPU_FW_LOG_FATAL 5 + +#define IVPU_FW_VERBOSE_BUFFER_SMALL_SIZE SZ_1M +#define IVPU_FW_VERBOSE_BUFFER_LARGE_SIZE SZ_8M +#define IVPU_FW_CRITICAL_BUFFER_SIZE SZ_512K + +extern unsigned int ivpu_fw_log_level; + +void ivpu_fw_log_print(struct ivpu_device *vdev, bool only_new_msgs, struct drm_printer *p); +void ivpu_fw_log_mark_read(struct ivpu_device *vdev); +void ivpu_fw_log_reset(struct ivpu_device *vdev); + + +#endif /* __IVPU_FW_LOG_H__ */ diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c new file mode 100644 index 000000000..4f2005a8d --- /dev/null +++ b/drivers/accel/ivpu/ivpu_gem.c @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_mmu.h" +#include "ivpu_mmu_context.h" + +MODULE_IMPORT_NS("DMA_BUF"); + +static const struct drm_gem_object_funcs ivpu_gem_funcs; + +static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) +{ + ivpu_dbg(vdev, BO, + "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n", + action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr, + (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, + drm_gem_is_imported(&bo->base.base)); +} + +static inline int ivpu_bo_lock(struct ivpu_bo *bo) +{ + return dma_resv_lock(bo->base.base.resv, NULL); +} + +static inline void ivpu_bo_unlock(struct ivpu_bo *bo) +{ + dma_resv_unlock(bo->base.base.resv); +} + +static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo) +{ + struct sg_table *sgt; + + drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base)); + + ivpu_bo_lock(bo); + + sgt = bo->base.sgt; + if (!sgt) { + sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) + ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt)); + else + bo->base.sgt = sgt; + } + + ivpu_bo_unlock(bo); + + return sgt; +} + +/* + * ivpu_bo_bind() - pin the backing physical pages and map them to VPU. + * + * This function pins physical memory pages, then maps the physical pages + * to IOMMU address space and finally updates the VPU MMU page tables + * to allow the VPU to translate VPU address to IOMMU address. + */ +int __must_check ivpu_bo_bind(struct ivpu_bo *bo) +{ + struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); + struct sg_table *sgt; + int ret = 0; + + ivpu_dbg_bo(vdev, bo, "bind"); + + if (drm_gem_is_imported(&bo->base.base)) + sgt = ivpu_bo_map_attachment(vdev, bo); + else + sgt = drm_gem_shmem_get_pages_sgt(&bo->base); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); + return ret; + } + + ivpu_bo_lock(bo); + + if (!bo->mmu_mapped) { + drm_WARN_ON(&vdev->drm, !bo->ctx); + ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, ivpu_bo_size(bo), + ivpu_bo_is_snooped(bo), ivpu_bo_is_read_only(bo)); + if (ret) { + ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); + goto unlock; + } + bo->mmu_mapped = true; + } + +unlock: + ivpu_bo_unlock(bo); + + return ret; +} + +static int +ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, + const struct ivpu_addr_range *range) +{ + struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); + int idx, ret; + + if (!drm_dev_enter(&vdev->drm, &idx)) + return -ENODEV; + + ivpu_bo_lock(bo); + + ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); + if (!ret) { + bo->ctx = ctx; + bo->ctx_id = ctx->id; + bo->vpu_addr = bo->mm_node.start; + ivpu_dbg_bo(vdev, bo, "vaddr"); + } + + ivpu_bo_unlock(bo); + + drm_dev_exit(idx); + + return ret; +} + +static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) +{ + struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); + + dma_resv_assert_held(bo->base.base.resv); + + if (bo->mmu_mapped) { + drm_WARN_ON(&vdev->drm, !bo->ctx); + drm_WARN_ON(&vdev->drm, !bo->vpu_addr); + drm_WARN_ON(&vdev->drm, !bo->base.sgt); + ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); + bo->mmu_mapped = false; + } + + if (bo->ctx) { + ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); + bo->ctx = NULL; + } + + if (bo->base.sgt) { + if (drm_gem_is_imported(&bo->base.base)) { + dma_buf_unmap_attachment(bo->base.base.import_attach, + bo->base.sgt, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); + sg_free_table(bo->base.sgt); + kfree(bo->base.sgt); + } + bo->base.sgt = NULL; + } +} + +void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) +{ + struct ivpu_bo *bo; + + if (drm_WARN_ON(&vdev->drm, !ctx)) + return; + + mutex_lock(&vdev->bo_list_lock); + list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { + ivpu_bo_lock(bo); + if (bo->ctx == ctx) { + ivpu_dbg_bo(vdev, bo, "unbind"); + ivpu_bo_unbind_locked(bo); + } + ivpu_bo_unlock(bo); + } + mutex_unlock(&vdev->bo_list_lock); +} + +struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) +{ + struct ivpu_bo *bo; + + if (size == 0 || !PAGE_ALIGNED(size)) + return ERR_PTR(-EINVAL); + + bo = kzalloc_obj(*bo); + if (!bo) + return ERR_PTR(-ENOMEM); + + bo->base.base.funcs = &ivpu_gem_funcs; + bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ + + INIT_LIST_HEAD(&bo->bo_list_node); + + return &bo->base.base; +} + +struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf) +{ + struct ivpu_device *vdev = to_ivpu_device(dev); + struct device *attach_dev = dev->dev; + struct dma_buf_attachment *attach; + struct drm_gem_object *obj; + struct ivpu_bo *bo; + int ret; + + attach = dma_buf_attach(dma_buf, attach_dev); + if (IS_ERR(attach)) + return ERR_CAST(attach); + + get_dma_buf(dma_buf); + + obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL); + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); + goto fail_detach; + } + + obj->import_attach = attach; + obj->resv = dma_buf->resv; + + bo = to_ivpu_bo(obj); + + mutex_lock(&vdev->bo_list_lock); + list_add_tail(&bo->bo_list_node, &vdev->bo_list); + mutex_unlock(&vdev->bo_list_lock); + + ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo)); + + return obj; + +fail_detach: + dma_buf_detach(dma_buf, attach); + dma_buf_put(dma_buf); + + return ERR_PTR(ret); +} + +static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) +{ + struct drm_gem_shmem_object *shmem; + struct ivpu_bo *bo; + + switch (flags & DRM_IVPU_BO_CACHE_MASK) { + case DRM_IVPU_BO_CACHED: + case DRM_IVPU_BO_WC: + break; + default: + return ERR_PTR(-EINVAL); + } + + shmem = drm_gem_shmem_create(&vdev->drm, size); + if (IS_ERR(shmem)) + return ERR_CAST(shmem); + + bo = to_ivpu_bo(&shmem->base); + bo->base.map_wc = flags & DRM_IVPU_BO_WC; + bo->flags = flags; + + mutex_lock(&vdev->bo_list_lock); + list_add_tail(&bo->bo_list_node, &vdev->bo_list); + mutex_unlock(&vdev->bo_list_lock); + + ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size); + + return bo; +} + +static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_bo *bo = to_ivpu_bo(obj); + struct ivpu_addr_range *range; + + if (bo->ctx) { + ivpu_dbg(vdev, IOCTL, "Can't add BO %pe to ctx %u: already in ctx %u\n", + bo, file_priv->ctx.id, bo->ctx->id); + return -EALREADY; + } + + if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) + range = &vdev->hw->ranges.shave; + else if (bo->flags & DRM_IVPU_BO_DMA_MEM) + range = &vdev->hw->ranges.dma; + else + range = &vdev->hw->ranges.user; + + return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); +} + +static void ivpu_gem_bo_free(struct drm_gem_object *obj) +{ + struct ivpu_device *vdev = to_ivpu_device(obj->dev); + struct ivpu_bo *bo = to_ivpu_bo(obj); + + ivpu_dbg_bo(vdev, bo, "free"); + + drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node)); + + mutex_lock(&vdev->bo_list_lock); + list_del(&bo->bo_list_node); + + drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) && + !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); + drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0); + drm_WARN_ON(&vdev->drm, bo->base.vaddr); + + ivpu_bo_lock(bo); + ivpu_bo_unbind_locked(bo); + ivpu_bo_unlock(bo); + + mutex_unlock(&vdev->bo_list_lock); + + drm_WARN_ON(&vdev->drm, bo->mmu_mapped); + drm_WARN_ON(&vdev->drm, bo->ctx); + + drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1); + drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node); + drm_gem_shmem_free(&bo->base); +} + +static enum drm_gem_object_status ivpu_gem_status(struct drm_gem_object *obj) +{ + struct ivpu_bo *bo = to_ivpu_bo(obj); + enum drm_gem_object_status status = 0; + + if (ivpu_bo_is_resident(bo)) + status |= DRM_GEM_OBJECT_RESIDENT; + + return status; +} + +static const struct drm_gem_object_funcs ivpu_gem_funcs = { + .free = ivpu_gem_bo_free, + .open = ivpu_gem_bo_open, + .print_info = drm_gem_shmem_object_print_info, + .pin = drm_gem_shmem_object_pin, + .unpin = drm_gem_shmem_object_unpin, + .get_sg_table = drm_gem_shmem_object_get_sg_table, + .vmap = drm_gem_shmem_object_vmap, + .vunmap = drm_gem_shmem_object_vunmap, + .mmap = drm_gem_shmem_object_mmap, + .status = ivpu_gem_status, + .vm_ops = &drm_gem_shmem_vm_ops, +}; + +int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct drm_ivpu_bo_create *args = data; + u64 size = PAGE_ALIGN(args->size); + struct ivpu_bo *bo; + int ret; + + if (args->flags & ~DRM_IVPU_BO_FLAGS) { + ivpu_dbg(vdev, IOCTL, "Invalid BO flags 0x%x\n", args->flags); + return -EINVAL; + } + + if (size == 0) { + ivpu_dbg(vdev, IOCTL, "Invalid BO size %llu\n", args->size); + return -EINVAL; + } + + bo = ivpu_bo_alloc(vdev, size, args->flags); + if (IS_ERR(bo)) { + ivpu_dbg(vdev, IOCTL, "Failed to allocate BO: %pe ctx %u size %llu flags 0x%x\n", + bo, file_priv->ctx.id, args->size, args->flags); + return PTR_ERR(bo); + } + + drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0); + + ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); + if (ret) { + ivpu_dbg(vdev, IOCTL, "Failed to create handle for BO: %pe ctx %u size %llu flags 0x%x\n", + bo, file_priv->ctx.id, args->size, args->flags); + } else { + args->vpu_addr = bo->vpu_addr; + drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1); + } + + drm_gem_object_put(&bo->base.base); + + return ret; +} + +struct ivpu_bo * +ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + struct ivpu_addr_range *range, u64 size, u32 flags) +{ + struct iosys_map map; + struct ivpu_bo *bo; + int ret; + + if (drm_WARN_ON(&vdev->drm, !range)) + return NULL; + + drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); + drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); + drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); + + bo = ivpu_bo_alloc(vdev, size, flags); + if (IS_ERR(bo)) { + ivpu_err(vdev, "Failed to allocate BO: %pe vpu_addr 0x%llx size %llu flags 0x%x\n", + bo, range->start, size, flags); + return NULL; + } + + ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); + if (ret) { + ivpu_err(vdev, "Failed to allocate NPU address for BO: %pe ctx %u size %llu: %d\n", + bo, ctx->id, size, ret); + goto err_put; + } + + ret = ivpu_bo_bind(bo); + if (ret) + goto err_put; + + if (flags & DRM_IVPU_BO_MAPPABLE) { + ivpu_bo_lock(bo); + ret = drm_gem_shmem_vmap_locked(&bo->base, &map); + ivpu_bo_unlock(bo); + + if (ret) + goto err_put; + } + + return bo; + +err_put: + drm_gem_object_put(&bo->base.base); + return NULL; +} + +struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags) +{ + struct ivpu_addr_range range; + + if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) { + ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size); + return NULL; + } + + if (ivpu_hw_range_init(vdev, &range, addr, size)) + return NULL; + + return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags); +} + +struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) +{ + return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); +} + +void ivpu_bo_free(struct ivpu_bo *bo) +{ + struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); + + if (bo->flags & DRM_IVPU_BO_MAPPABLE) { + ivpu_bo_lock(bo); + drm_gem_shmem_vunmap_locked(&bo->base, &map); + ivpu_bo_unlock(bo); + } + + drm_gem_object_put(&bo->base.base); +} + +int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_ivpu_bo_info *args = data; + struct drm_gem_object *obj; + struct ivpu_bo *bo; + int ret = 0; + + obj = drm_gem_object_lookup(file, args->handle); + if (!obj) + return -ENOENT; + + bo = to_ivpu_bo(obj); + + ivpu_bo_lock(bo); + args->flags = bo->flags; + args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); + args->vpu_addr = bo->vpu_addr; + args->size = obj->size; + ivpu_bo_unlock(bo); + + drm_gem_object_put(obj); + return ret; +} + +int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_ivpu_bo_wait *args = data; + struct drm_gem_object *obj; + unsigned long timeout; + long ret; + + timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); + + /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */ + timeout += 1; + + obj = drm_gem_object_lookup(file, args->handle); + if (!obj) + return -EINVAL; + + ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); + if (ret == 0) { + ret = -ETIMEDOUT; + } else if (ret > 0) { + ret = 0; + args->job_status = to_ivpu_bo(obj)->job_status; + } + + drm_gem_object_put(obj); + + return ret; +} + +static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) +{ + ivpu_bo_lock(bo); + + drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", + bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size, + bo->flags, kref_read(&bo->base.base.refcount)); + + if (bo->base.pages) + drm_printf(p, " has_pages"); + + if (bo->mmu_mapped) + drm_printf(p, " mmu_mapped"); + + if (drm_gem_is_imported(&bo->base.base)) + drm_printf(p, " imported"); + + drm_printf(p, "\n"); + + ivpu_bo_unlock(bo); +} + +void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) +{ + struct ivpu_device *vdev = to_ivpu_device(dev); + struct ivpu_bo *bo; + + drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", + "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); + + mutex_lock(&vdev->bo_list_lock); + list_for_each_entry(bo, &vdev->bo_list, bo_list_node) + ivpu_bo_print_info(bo, p); + mutex_unlock(&vdev->bo_list_lock); +} + +void ivpu_bo_list_print(struct drm_device *dev) +{ + struct drm_printer p = drm_info_printer(dev->dev); + + ivpu_bo_list(dev, &p); +} diff --git a/drivers/accel/ivpu/ivpu_gem.h b/drivers/accel/ivpu/ivpu_gem.h new file mode 100644 index 000000000..0c3350f22 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_gem.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2025 Intel Corporation + */ +#ifndef __IVPU_GEM_H__ +#define __IVPU_GEM_H__ + +#include +#include +#include + +struct ivpu_file_priv; + +struct ivpu_bo { + struct drm_gem_shmem_object base; + struct ivpu_mmu_context *ctx; + struct list_head bo_list_node; + struct drm_mm_node mm_node; + + u64 vpu_addr; + u32 flags; + u32 job_status; /* Valid only for command buffer */ + u32 ctx_id; + bool mmu_mapped; +}; + +int ivpu_bo_bind(struct ivpu_bo *bo); +void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx); + +struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size); +struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf); +struct ivpu_bo *ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + struct ivpu_addr_range *range, u64 size, u32 flags); +struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags); +struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags); +void ivpu_bo_free(struct ivpu_bo *bo); + +int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_bo_create_from_userptr_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p); +void ivpu_bo_list_print(struct drm_device *dev); + +static inline struct ivpu_bo *to_ivpu_bo(struct drm_gem_object *obj) +{ + return container_of(obj, struct ivpu_bo, base.base); +} + +static inline void *ivpu_bo_vaddr(struct ivpu_bo *bo) +{ + return bo->base.vaddr; +} + +static inline size_t ivpu_bo_size(struct ivpu_bo *bo) +{ + return bo->base.base.size; +} + +static inline u32 ivpu_bo_cache_mode(struct ivpu_bo *bo) +{ + return bo->flags & DRM_IVPU_BO_CACHE_MASK; +} + +static inline struct ivpu_device *ivpu_bo_to_vdev(struct ivpu_bo *bo) +{ + return to_ivpu_device(bo->base.base.dev); +} + +static inline bool ivpu_bo_is_snooped(struct ivpu_bo *bo) +{ + if (ivpu_is_force_snoop_enabled(ivpu_bo_to_vdev(bo))) + return true; + + return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED; +} + +static inline bool ivpu_bo_is_read_only(struct ivpu_bo *bo) +{ + return bo->flags & DRM_IVPU_BO_READ_ONLY; +} + +static inline bool ivpu_bo_is_resident(struct ivpu_bo *bo) +{ + return !!bo->base.pages; +} + +static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr) +{ + if (vpu_addr < bo->vpu_addr) + return NULL; + + if (vpu_addr >= (bo->vpu_addr + ivpu_bo_size(bo))) + return NULL; + + return ivpu_bo_vaddr(bo) + (vpu_addr - bo->vpu_addr); +} + +static inline u32 cpu_to_vpu_addr(struct ivpu_bo *bo, void *cpu_addr) +{ + if (cpu_addr < ivpu_bo_vaddr(bo)) + return 0; + + if (cpu_addr >= (ivpu_bo_vaddr(bo) + ivpu_bo_size(bo))) + return 0; + + return bo->vpu_addr + (cpu_addr - ivpu_bo_vaddr(bo)); +} + +static inline bool ivpu_bo_is_mappable(struct ivpu_bo *bo) +{ + return bo->flags & DRM_IVPU_BO_MAPPABLE; +} + +#endif /* __IVPU_GEM_H__ */ diff --git a/drivers/accel/ivpu/ivpu_gem_userptr.c b/drivers/accel/ivpu/ivpu_gem_userptr.c new file mode 100644 index 000000000..7cbf3a4cd --- /dev/null +++ b/drivers/accel/ivpu/ivpu_gem_userptr.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2025 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_gem.h" + +static struct sg_table * +ivpu_gem_userptr_dmabuf_map(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct sg_table *sgt = attachment->dmabuf->priv; + int ret; + + ret = dma_map_sgtable(attachment->dev, sgt, direction, DMA_ATTR_SKIP_CPU_SYNC); + if (ret) + return ERR_PTR(ret); + + return sgt; +} + +static void ivpu_gem_userptr_dmabuf_unmap(struct dma_buf_attachment *attachment, + struct sg_table *sgt, + enum dma_data_direction direction) +{ + dma_unmap_sgtable(attachment->dev, sgt, direction, DMA_ATTR_SKIP_CPU_SYNC); +} + +static void ivpu_gem_userptr_dmabuf_release(struct dma_buf *dma_buf) +{ + struct sg_table *sgt = dma_buf->priv; + struct sg_page_iter page_iter; + struct page *page; + + for_each_sgtable_page(sgt, &page_iter, 0) { + page = sg_page_iter_page(&page_iter); + unpin_user_page(page); + } + + sg_free_table(sgt); + kfree(sgt); +} + +static const struct dma_buf_ops ivpu_gem_userptr_dmabuf_ops = { + .map_dma_buf = ivpu_gem_userptr_dmabuf_map, + .unmap_dma_buf = ivpu_gem_userptr_dmabuf_unmap, + .release = ivpu_gem_userptr_dmabuf_release, +}; + +static struct dma_buf * +ivpu_create_userptr_dmabuf(struct ivpu_device *vdev, void __user *user_ptr, + size_t size, uint32_t flags) +{ + struct dma_buf_export_info exp_info = {}; + struct dma_buf *dma_buf; + struct sg_table *sgt; + struct page **pages; + unsigned long nr_pages = size >> PAGE_SHIFT; + unsigned int gup_flags = FOLL_LONGTERM; + int ret, i, pinned; + + /* Add FOLL_WRITE only if the BO is not read-only */ + if (!(flags & DRM_IVPU_BO_READ_ONLY)) + gup_flags |= FOLL_WRITE; + + pages = kvmalloc_objs(*pages, nr_pages); + if (!pages) + return ERR_PTR(-ENOMEM); + + pinned = pin_user_pages_fast((unsigned long)user_ptr, nr_pages, gup_flags, pages); + if (pinned < 0) { + ret = pinned; + ivpu_dbg(vdev, IOCTL, "Failed to pin user pages: %d\n", ret); + goto free_pages_array; + } + + if (pinned != nr_pages) { + ivpu_dbg(vdev, IOCTL, "Pinned %d pages, expected %lu\n", pinned, nr_pages); + ret = -EFAULT; + goto unpin_pages; + } + + sgt = kmalloc_obj(*sgt); + if (!sgt) { + ret = -ENOMEM; + goto unpin_pages; + } + + ret = sg_alloc_table_from_pages(sgt, pages, nr_pages, 0, size, GFP_KERNEL); + if (ret) { + ivpu_dbg(vdev, IOCTL, "Failed to create sg table: %d\n", ret); + goto free_sgt; + } + + exp_info.exp_name = "ivpu_userptr_dmabuf"; + exp_info.owner = THIS_MODULE; + exp_info.ops = &ivpu_gem_userptr_dmabuf_ops; + exp_info.size = size; + exp_info.flags = O_RDWR | O_CLOEXEC; + exp_info.priv = sgt; + + dma_buf = dma_buf_export(&exp_info); + if (IS_ERR(dma_buf)) { + ret = PTR_ERR(dma_buf); + ivpu_dbg(vdev, IOCTL, "Failed to export userptr dma-buf: %d\n", ret); + goto free_sg_table; + } + + kvfree(pages); + return dma_buf; + +free_sg_table: + sg_free_table(sgt); +free_sgt: + kfree(sgt); +unpin_pages: + for (i = 0; i < pinned; i++) + unpin_user_page(pages[i]); +free_pages_array: + kvfree(pages); + return ERR_PTR(ret); +} + +static struct ivpu_bo * +ivpu_bo_create_from_userptr(struct ivpu_device *vdev, void __user *user_ptr, + size_t size, uint32_t flags) +{ + struct dma_buf *dma_buf; + struct drm_gem_object *obj; + struct ivpu_bo *bo; + + dma_buf = ivpu_create_userptr_dmabuf(vdev, user_ptr, size, flags); + if (IS_ERR(dma_buf)) + return ERR_CAST(dma_buf); + + obj = ivpu_gem_prime_import(&vdev->drm, dma_buf); + if (IS_ERR(obj)) { + dma_buf_put(dma_buf); + return ERR_CAST(obj); + } + + dma_buf_put(dma_buf); + + bo = to_ivpu_bo(obj); + bo->flags = flags; + + return bo; +} + +int ivpu_bo_create_from_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_ivpu_bo_create_from_userptr *args = data; + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = to_ivpu_device(dev); + void __user *user_ptr = u64_to_user_ptr(args->user_ptr); + struct ivpu_bo *bo; + int ret; + + if (args->flags & ~(DRM_IVPU_BO_HIGH_MEM | DRM_IVPU_BO_DMA_MEM | DRM_IVPU_BO_READ_ONLY)) { + ivpu_dbg(vdev, IOCTL, "Invalid BO flags: 0x%x\n", args->flags); + return -EINVAL; + } + + if (!args->user_ptr || !args->size) { + ivpu_dbg(vdev, IOCTL, "Userptr or size are zero: ptr %llx size %llu\n", + args->user_ptr, args->size); + return -EINVAL; + } + + if (!PAGE_ALIGNED(args->user_ptr) || !PAGE_ALIGNED(args->size)) { + ivpu_dbg(vdev, IOCTL, "Userptr or size not page aligned: ptr %llx size %llu\n", + args->user_ptr, args->size); + return -EINVAL; + } + + if (!access_ok(user_ptr, args->size)) { + ivpu_dbg(vdev, IOCTL, "Userptr is not accessible: ptr %llx size %llu\n", + args->user_ptr, args->size); + return -EFAULT; + } + + bo = ivpu_bo_create_from_userptr(vdev, user_ptr, args->size, args->flags); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); + if (ret) { + ivpu_dbg(vdev, IOCTL, "Failed to create handle for BO: %pe ctx %u size %llu flags 0x%x\n", + bo, file_priv->ctx.id, args->size, args->flags); + } else { + ivpu_dbg(vdev, BO, "Created userptr BO: handle=%u vpu_addr=0x%llx size=%llu flags=0x%x\n", + args->handle, bo->vpu_addr, args->size, bo->flags); + args->vpu_addr = bo->vpu_addr; + } + + drm_gem_object_put(&bo->base.base); + + return ret; +} diff --git a/drivers/accel/ivpu/ivpu_hw.c b/drivers/accel/ivpu/ivpu_hw.c new file mode 100644 index 000000000..d4a9bcda4 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 - 2024 Intel Corporation + */ + +#include "ivpu_drv.h" +#include "ivpu_hw.h" +#include "ivpu_hw_btrs.h" +#include "ivpu_hw_ip.h" + +#include +#include +#include +#include +#include + +#ifdef CONFIG_FAULT_INJECTION +DECLARE_FAULT_ATTR(ivpu_hw_failure); + +static char *ivpu_fail_hw; +module_param_named_unsafe(fail_hw, ivpu_fail_hw, charp, 0444); +MODULE_PARM_DESC(fail_hw, ",,,"); +#endif + +#define FW_SHARED_MEM_ALIGNMENT SZ_512K /* VPU MTRR limitation */ + +#define ECC_MCA_SIGNAL_ENABLE_MASK 0xff + +static char *platform_to_str(u32 platform) +{ + switch (platform) { + case IVPU_PLATFORM_SILICON: + return "SILICON"; + case IVPU_PLATFORM_SIMICS: + return "SIMICS"; + case IVPU_PLATFORM_FPGA: + return "FPGA"; + case IVPU_PLATFORM_HSLE: + return "HSLE"; + default: + return "Invalid platform"; + } +} + +static void platform_init(struct ivpu_device *vdev) +{ + int platform = ivpu_hw_btrs_platform_read(vdev); + + ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", platform_to_str(platform), platform); + + switch (platform) { + case IVPU_PLATFORM_SILICON: + case IVPU_PLATFORM_SIMICS: + case IVPU_PLATFORM_FPGA: + case IVPU_PLATFORM_HSLE: + vdev->platform = platform; + break; + + default: + ivpu_err(vdev, "Invalid platform type: %d\n", platform); + break; + } +} + +static void wa_init(struct ivpu_device *vdev) +{ + vdev->wa.punit_disabled = false; + vdev->wa.clear_runtime_mem = false; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev); + + if ((ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && + ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) || + (ivpu_device_id(vdev) == PCI_DEVICE_ID_NVL && + ivpu_revision(vdev) == IVPU_HW_IP_REV_NVL_A0)) + vdev->wa.disable_clock_relinquish = true; + + if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE) + vdev->wa.disable_clock_relinquish = false; + + if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_DISABLE) + vdev->wa.disable_clock_relinquish = true; + + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + vdev->wa.wp0_during_power_up = true; + + if (ivpu_test_mode & IVPU_TEST_MODE_D0I2_DISABLE) + vdev->wa.disable_d0i2 = true; + + IVPU_PRINT_WA(punit_disabled); + IVPU_PRINT_WA(clear_runtime_mem); + IVPU_PRINT_WA(interrupt_clear_with_0); + IVPU_PRINT_WA(disable_clock_relinquish); + IVPU_PRINT_WA(wp0_during_power_up); + IVPU_PRINT_WA(disable_d0i2); +} + +static void timeouts_init(struct ivpu_device *vdev) +{ + if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) { + vdev->timeout.boot = -1; + vdev->timeout.jsm = -1; + vdev->timeout.tdr = -1; + vdev->timeout.inference = -1; + vdev->timeout.autosuspend = -1; + vdev->timeout.d0i3_entry_msg = -1; + } else if (ivpu_is_fpga(vdev)) { + vdev->timeout.boot = 50; + vdev->timeout.jsm = 15000; + vdev->timeout.tdr = 30000; + vdev->timeout.inference = 900000; + vdev->timeout.autosuspend = -1; + vdev->timeout.d0i3_entry_msg = 500; + vdev->timeout.state_dump_msg = 10000; + } else if (ivpu_is_simics(vdev)) { + vdev->timeout.boot = 50; + vdev->timeout.jsm = 500; + vdev->timeout.tdr = 10000; + vdev->timeout.inference = 300000; + vdev->timeout.autosuspend = 100; + vdev->timeout.d0i3_entry_msg = 100; + vdev->timeout.state_dump_msg = 10; + } else { + vdev->timeout.boot = 1000; + vdev->timeout.jsm = 500; + vdev->timeout.tdr = 2000; + vdev->timeout.inference = 60000; + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + vdev->timeout.autosuspend = 10; + else + vdev->timeout.autosuspend = 100; + vdev->timeout.d0i3_entry_msg = 5; + vdev->timeout.state_dump_msg = 100; + } +} + +static void priority_bands_init(struct ivpu_device *vdev) +{ + /* Idle */ + vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 0; + vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 50000; + vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 160000; + /* Normal */ + vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000; + vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000; + vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 300000; + /* Focus */ + vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000; + vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000; + vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 200000; + /* Realtime */ + vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 0; + vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 50000; + vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 200000; +} + +int ivpu_hw_range_init(struct ivpu_device *vdev, struct ivpu_addr_range *range, u64 start, u64 size) +{ + u64 end; + + if (!range || check_add_overflow(start, size, &end)) { + ivpu_err(vdev, "Invalid range: start 0x%llx size %llu\n", start, size); + return -EINVAL; + } + + range->start = start; + range->end = end; + + return 0; +} + +static void memory_ranges_init(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x84800000, SZ_64M); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0xa0000000, 511 * SZ_1M); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x180000000, SZ_2G); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.dma, 0x200000000, SZ_128G); + } else { + ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x80000000, SZ_64M); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x80000000, SZ_2G); + ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0x100000000, SZ_256G); + vdev->hw->ranges.dma = vdev->hw->ranges.user; + } + + drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vdev->hw->ranges.global.start, + FW_SHARED_MEM_ALIGNMENT)); +} + +static int wp_enable(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_wp_drive(vdev, true); +} + +static int wp_disable(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_wp_drive(vdev, false); +} + +int ivpu_hw_power_up(struct ivpu_device *vdev) +{ + int ret; + + if (IVPU_WA(wp0_during_power_up)) { + /* WP requests may fail when powering down, so issue WP 0 here */ + ret = wp_disable(vdev); + if (ret) + ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret); + } + + ret = ivpu_hw_btrs_d0i3_disable(vdev); + if (ret) + ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret); + + ret = wp_enable(vdev); + if (ret) { + ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret); + return ret; + } + + if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) { + if (IVPU_WA(disable_clock_relinquish)) + ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev); + ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev); + ivpu_hw_btrs_ats_print_lnl(vdev); + } + + ret = ivpu_hw_ip_host_ss_configure(vdev); + if (ret) { + ivpu_err(vdev, "Failed to configure host SS: %d\n", ret); + return ret; + } + + ivpu_hw_ip_idle_gen_disable(vdev); + + ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev); + if (ret) { + ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n"); + return ret; + } + + ret = ivpu_hw_ip_pwr_domain_enable(vdev); + if (ret) { + ivpu_err(vdev, "Failed to enable power domain: %d\n", ret); + return ret; + } + + ret = ivpu_hw_ip_host_ss_axi_enable(vdev); + if (ret) { + ivpu_err(vdev, "Failed to enable AXI: %d\n", ret); + return ret; + } + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL) + ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev); + + ret = ivpu_hw_ip_top_noc_enable(vdev); + if (ret) + ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret); + + return ret; +} + +static void save_d0i3_entry_timestamp(struct ivpu_device *vdev) +{ + vdev->hw->d0i3_entry_host_ts = ktime_get_boottime(); + vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev); +} + +int ivpu_hw_reset(struct ivpu_device *vdev) +{ + int ret = 0; + + if (ivpu_hw_btrs_ip_reset(vdev)) { + ivpu_err(vdev, "Failed to reset NPU IP\n"); + ret = -EIO; + } + + if (wp_disable(vdev)) { + ivpu_err(vdev, "Failed to disable workpoint\n"); + ret = -EIO; + } + + return ret; +} + +int ivpu_hw_power_down(struct ivpu_device *vdev) +{ + int ret = 0; + + save_d0i3_entry_timestamp(vdev); + + if (!ivpu_hw_is_idle(vdev)) + ivpu_warn(vdev, "NPU not idle during power down\n"); + + if (ivpu_hw_reset(vdev)) { + ivpu_err(vdev, "Failed to reset NPU\n"); + ret = -EIO; + } + + if (ivpu_hw_btrs_d0i3_enable(vdev)) { + ivpu_err(vdev, "Failed to enter D0I3\n"); + ret = -EIO; + } + + return ret; +} + +int ivpu_hw_init(struct ivpu_device *vdev) +{ + ivpu_hw_btrs_info_init(vdev); + ivpu_hw_btrs_freq_ratios_init(vdev); + priority_bands_init(vdev); + memory_ranges_init(vdev); + platform_init(vdev); + wa_init(vdev); + timeouts_init(vdev); + atomic_set(&vdev->hw->firewall_irq_counter, 0); + +#ifdef CONFIG_FAULT_INJECTION + if (ivpu_fail_hw) + setup_fault_attr(&ivpu_hw_failure, ivpu_fail_hw); +#endif + + return 0; +} + +int ivpu_hw_boot_fw(struct ivpu_device *vdev) +{ + int ret; + + ivpu_hw_ip_snoop_disable(vdev); + ivpu_hw_ip_tbu_mmu_enable(vdev); + ret = ivpu_hw_ip_soc_cpu_boot(vdev); + if (ret) + ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret); + + return ret; +} + +void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; + return; + } + + if (enable) + vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH; + else + vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; +} + +void ivpu_irq_handlers_init(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx; + else + vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl; + else + vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl; +} + +void ivpu_hw_irq_enable(struct ivpu_device *vdev) +{ + ivpu_hw_ip_irq_enable(vdev); + ivpu_hw_btrs_irq_enable(vdev); +} + +void ivpu_hw_irq_disable(struct ivpu_device *vdev) +{ + ivpu_hw_btrs_irq_disable(vdev); + ivpu_hw_ip_irq_disable(vdev); +} + +irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr) +{ + struct ivpu_device *vdev = ptr; + bool ip_handled, btrs_handled; + + ivpu_hw_btrs_global_int_disable(vdev); + + btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq); + if (!ivpu_hw_is_idle((vdev)) || !btrs_handled) + ip_handled = ivpu_hw_ip_irq_handler(vdev, irq); + else + ip_handled = false; + + /* Re-enable global interrupts to re-trigger MSI for pending interrupts */ + ivpu_hw_btrs_global_int_enable(vdev); + + if (!ip_handled && !btrs_handled) + return IRQ_NONE; + + pm_runtime_mark_last_busy(vdev->drm.dev); + return IRQ_HANDLED; +} + +bool ivpu_hw_uses_ecc_mca_signal(struct ivpu_device *vdev) +{ + unsigned long long msr_integrity_caps; + int ret; + + if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX) + return false; + + ret = rdmsrq_safe(MSR_INTEGRITY_CAPS, &msr_integrity_caps); + if (ret) { + ivpu_warn(vdev, "Error reading MSR_INTEGRITY_CAPS: %d", ret); + return false; + } + + ivpu_dbg(vdev, MISC, "MSR_INTEGRITY_CAPS: 0x%llx\n", msr_integrity_caps); + + return msr_integrity_caps & ECC_MCA_SIGNAL_ENABLE_MASK; +} diff --git a/drivers/accel/ivpu/ivpu_hw.h b/drivers/accel/ivpu/ivpu_hw.h new file mode 100644 index 000000000..487a918e2 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#ifndef __IVPU_HW_H__ +#define __IVPU_HW_H__ + +#include "ivpu_drv.h" +#include "ivpu_hw_btrs.h" +#include "ivpu_hw_ip.h" + +struct ivpu_addr_range { + resource_size_t start; + resource_size_t end; +}; + +struct ivpu_hw_info { + struct { + bool (*btrs_irq_handler)(struct ivpu_device *vdev, int irq); + bool (*ip_irq_handler)(struct ivpu_device *vdev, int irq); + } irq; + struct { + struct ivpu_addr_range runtime; + struct ivpu_addr_range global; + struct ivpu_addr_range user; + struct ivpu_addr_range shave; + struct ivpu_addr_range dma; + } ranges; + struct { + /* Hardware min and max pll ratio */ + u8 min_ratio; + u8 max_ratio; + /* + * Pll ratio for the efficiency frequency. The VPU has optimum + * performance to power ratio at this frequency. + */ + u8 pn_ratio; + /* Pll ratios configured via sysfs interface */ + u8 cfg_min_ratio; + u8 cfg_max_ratio; + u32 profiling_freq; + } pll; + struct { + u32 grace_period[VPU_HWS_NUM_PRIORITY_BANDS]; + u32 process_quantum[VPU_HWS_NUM_PRIORITY_BANDS]; + u32 process_grace_period[VPU_HWS_NUM_PRIORITY_BANDS]; + } hws; + u32 tile_fuse; + u32 sku; + u16 config; + int dma_bits; + ktime_t d0i3_entry_host_ts; + u64 d0i3_entry_vpu_ts; + atomic_t firewall_irq_counter; +}; + +int ivpu_hw_init(struct ivpu_device *vdev); +int ivpu_hw_range_init(struct ivpu_device *vdev, struct ivpu_addr_range *range, u64 start, + u64 size); +int ivpu_hw_power_up(struct ivpu_device *vdev); +int ivpu_hw_power_down(struct ivpu_device *vdev); +int ivpu_hw_reset(struct ivpu_device *vdev); +int ivpu_hw_boot_fw(struct ivpu_device *vdev); +void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable); +void ivpu_irq_handlers_init(struct ivpu_device *vdev); +void ivpu_hw_irq_enable(struct ivpu_device *vdev); +void ivpu_hw_irq_disable(struct ivpu_device *vdev); +irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr); +bool ivpu_hw_uses_ecc_mca_signal(struct ivpu_device *vdev); + +static inline u32 ivpu_hw_btrs_irq_handler(struct ivpu_device *vdev, int irq) +{ + return vdev->hw->irq.btrs_irq_handler(vdev, irq); +} + +static inline u32 ivpu_hw_ip_irq_handler(struct ivpu_device *vdev, int irq) +{ + return vdev->hw->irq.ip_irq_handler(vdev, irq); +} + +static inline u64 ivpu_hw_range_size(const struct ivpu_addr_range *range) +{ + return range->end - range->start; +} + +static inline void ivpu_hw_irq_clear(struct ivpu_device *vdev) +{ + ivpu_hw_ip_irq_clear(vdev); +} + +static inline u32 ivpu_hw_profiling_freq_get(struct ivpu_device *vdev) +{ + return vdev->hw->pll.profiling_freq; +} + +static inline void ivpu_hw_diagnose_failure(struct ivpu_device *vdev) +{ + ivpu_hw_ip_diagnose_failure(vdev); + ivpu_hw_btrs_diagnose_failure(vdev); +} + +static inline u32 ivpu_hw_telemetry_offset_get(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_telemetry_offset_get(vdev); +} + +static inline u32 ivpu_hw_telemetry_size_get(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_telemetry_size_get(vdev); +} + +static inline u32 ivpu_hw_telemetry_enable_get(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_telemetry_enable_get(vdev); +} + +static inline bool ivpu_hw_is_idle(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_is_idle(vdev); +} + +static inline int ivpu_hw_wait_for_idle(struct ivpu_device *vdev) +{ + return ivpu_hw_btrs_wait_for_idle(vdev); +} + +static inline void ivpu_hw_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr) +{ + ivpu_hw_ip_ipc_tx_set(vdev, vpu_addr); +} + +static inline void ivpu_hw_db_set(struct ivpu_device *vdev, u32 db_id) +{ + ivpu_hw_ip_db_set(vdev, db_id); +} + +static inline u32 ivpu_hw_ipc_rx_addr_get(struct ivpu_device *vdev) +{ + return ivpu_hw_ip_ipc_rx_addr_get(vdev); +} + +static inline u32 ivpu_hw_ipc_rx_count_get(struct ivpu_device *vdev) +{ + return ivpu_hw_ip_ipc_rx_count_get(vdev); +} + +#endif /* __IVPU_HW_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_37xx_reg.h b/drivers/accel/ivpu/ivpu_hw_37xx_reg.h new file mode 100644 index 000000000..cf5e2f010 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_37xx_reg.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_HW_37XX_REG_H__ +#define __IVPU_HW_37XX_REG_H__ + +#include + +#define VPU_37XX_HOST_SS_CPR_CLK_SET 0x00000084u +#define VPU_37XX_HOST_SS_CPR_CLK_SET_TOP_NOC_MASK BIT_MASK(1) +#define VPU_37XX_HOST_SS_CPR_CLK_SET_DSS_MAS_MASK BIT_MASK(10) +#define VPU_37XX_HOST_SS_CPR_CLK_SET_MSS_MAS_MASK BIT_MASK(11) + +#define VPU_37XX_HOST_SS_CPR_RST_SET 0x00000094u +#define VPU_37XX_HOST_SS_CPR_RST_SET_TOP_NOC_MASK BIT_MASK(1) +#define VPU_37XX_HOST_SS_CPR_RST_SET_DSS_MAS_MASK BIT_MASK(10) +#define VPU_37XX_HOST_SS_CPR_RST_SET_MSS_MAS_MASK BIT_MASK(11) + +#define VPU_37XX_HOST_SS_CPR_RST_CLR 0x00000098u +#define VPU_37XX_HOST_SS_CPR_RST_CLR_AON_MASK BIT_MASK(0) +#define VPU_37XX_HOST_SS_CPR_RST_CLR_TOP_NOC_MASK BIT_MASK(1) +#define VPU_37XX_HOST_SS_CPR_RST_CLR_DSS_MAS_MASK BIT_MASK(10) +#define VPU_37XX_HOST_SS_CPR_RST_CLR_MSS_MAS_MASK BIT_MASK(11) + +#define VPU_37XX_HOST_SS_HW_VERSION 0x00000108u +#define VPU_37XX_HOST_SS_HW_VERSION_SOC_REVISION_MASK GENMASK(7, 0) +#define VPU_37XX_HOST_SS_HW_VERSION_SOC_NUMBER_MASK GENMASK(15, 8) +#define VPU_37XX_HOST_SS_HW_VERSION_VPU_GENERATION_MASK GENMASK(23, 16) + +#define VPU_37XX_HOST_SS_GEN_CTRL 0x00000118u +#define VPU_37XX_HOST_SS_GEN_CTRL_PS_MASK GENMASK(31, 29) + +#define VPU_37XX_HOST_SS_NOC_QREQN 0x00000154u +#define VPU_37XX_HOST_SS_NOC_QREQN_TOP_SOCMMIO_MASK BIT_MASK(0) + +#define VPU_37XX_HOST_SS_NOC_QACCEPTN 0x00000158u +#define VPU_37XX_HOST_SS_NOC_QACCEPTN_TOP_SOCMMIO_MASK BIT_MASK(0) + +#define VPU_37XX_HOST_SS_NOC_QDENY 0x0000015cu +#define VPU_37XX_HOST_SS_NOC_QDENY_TOP_SOCMMIO_MASK BIT_MASK(0) + +#define VPU_37XX_TOP_NOC_QREQN 0x00000160u +#define VPU_37XX_TOP_NOC_QREQN_CPU_CTRL_MASK BIT_MASK(0) +#define VPU_37XX_TOP_NOC_QREQN_HOSTIF_L2CACHE_MASK BIT_MASK(1) + +#define VPU_37XX_TOP_NOC_QACCEPTN 0x00000164u +#define VPU_37XX_TOP_NOC_QACCEPTN_CPU_CTRL_MASK BIT_MASK(0) +#define VPU_37XX_TOP_NOC_QACCEPTN_HOSTIF_L2CACHE_MASK BIT_MASK(1) + +#define VPU_37XX_TOP_NOC_QDENY 0x00000168u +#define VPU_37XX_TOP_NOC_QDENY_CPU_CTRL_MASK BIT_MASK(0) +#define VPU_37XX_TOP_NOC_QDENY_HOSTIF_L2CACHE_MASK BIT_MASK(1) + +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN 0x00000170u +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_CSS_ROM_CMX_MASK BIT_MASK(0) +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_CSS_DBG_MASK BIT_MASK(1) +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_CSS_CTRL_MASK BIT_MASK(2) +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_DEC400_MASK BIT_MASK(3) +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_MSS_NCE_MASK BIT_MASK(4) +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_MSS_MBI_MASK BIT_MASK(5) +#define VPU_37XX_HOST_SS_FW_SOC_IRQ_EN_MSS_MBI_CMX_MASK BIT_MASK(6) + +#define VPU_37XX_HOST_SS_ICB_STATUS_0 0x00010210u +#define VPU_37XX_HOST_SS_ICB_STATUS_0_TIMER_0_INT_MASK BIT_MASK(0) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_TIMER_1_INT_MASK BIT_MASK(1) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_TIMER_2_INT_MASK BIT_MASK(2) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_TIMER_3_INT_MASK BIT_MASK(3) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_HOST_IPC_FIFO_INT_MASK BIT_MASK(4) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_MMU_IRQ_0_INT_MASK BIT_MASK(5) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_MMU_IRQ_1_INT_MASK BIT_MASK(6) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_MMU_IRQ_2_INT_MASK BIT_MASK(7) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_NOC_FIREWALL_INT_MASK BIT_MASK(8) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_CPU_INT_REDIRECT_0_INT_MASK BIT_MASK(30) +#define VPU_37XX_HOST_SS_ICB_STATUS_0_CPU_INT_REDIRECT_1_INT_MASK BIT_MASK(31) + +#define VPU_37XX_HOST_SS_ICB_STATUS_1 0x00010214u +#define VPU_37XX_HOST_SS_ICB_STATUS_1_CPU_INT_REDIRECT_2_INT_MASK BIT_MASK(0) +#define VPU_37XX_HOST_SS_ICB_STATUS_1_CPU_INT_REDIRECT_3_INT_MASK BIT_MASK(1) +#define VPU_37XX_HOST_SS_ICB_STATUS_1_CPU_INT_REDIRECT_4_INT_MASK BIT_MASK(2) + +#define VPU_37XX_HOST_SS_ICB_CLEAR_0 0x00010220u +#define VPU_37XX_HOST_SS_ICB_CLEAR_1 0x00010224u +#define VPU_37XX_HOST_SS_ICB_ENABLE_0 0x00010240u + +#define VPU_37XX_HOST_SS_TIM_IPC_FIFO_ATM 0x000200f4u + +#define VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT 0x000200fcu +#define VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT_READ_POINTER_MASK GENMASK(7, 0) +#define VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT_WRITE_POINTER_MASK GENMASK(15, 8) +#define VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT_FILL_LEVEL_MASK GENMASK(23, 16) +#define VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT_RSVD0_MASK GENMASK(31, 24) + +#define VPU_37XX_HOST_SS_AON_PWR_ISO_EN0 0x00030020u +#define VPU_37XX_HOST_SS_AON_PWR_ISO_EN0_MSS_CPU_MASK BIT_MASK(3) + +#define VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0 0x00030024u +#define VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0_MSS_CPU_MASK BIT_MASK(3) + +#define VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0 0x00030028u +#define VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0_MSS_CPU_MASK BIT_MASK(3) + +#define VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0 0x0003002cu +#define VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0_MSS_CPU_MASK BIT_MASK(3) + +#define VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN 0x00030200u +#define VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN_EN_MASK BIT_MASK(0) + +#define VPU_37XX_HOST_SS_AON_DPU_ACTIVE 0x00030204u +#define VPU_37XX_HOST_SS_AON_DPU_ACTIVE_DPU_ACTIVE_MASK BIT_MASK(0) + +#define VPU_37XX_HOST_SS_LOADING_ADDRESS_LO 0x00041040u +#define VPU_37XX_HOST_SS_LOADING_ADDRESS_LO_DONE_MASK BIT_MASK(0) +#define VPU_37XX_HOST_SS_LOADING_ADDRESS_LO_IOSF_RS_ID_MASK GENMASK(2, 1) +#define VPU_37XX_HOST_SS_LOADING_ADDRESS_LO_IMAGE_LOCATION_MASK GENMASK(31, 3) + +#define VPU_37XX_HOST_SS_WORKPOINT_CONFIG_MIRROR 0x00082020u +#define VPU_37XX_HOST_SS_WORKPOINT_CONFIG_MIRROR_FINAL_PLL_FREQ_MASK GENMASK(15, 0) +#define VPU_37XX_HOST_SS_WORKPOINT_CONFIG_MIRROR_CONFIG_ID_MASK GENMASK(31, 16) + +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES 0x00360000u +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_CACHE_OVERRIDE_EN_MASK BIT_MASK(0) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_AWCACHE_OVERRIDE_MASK BIT_MASK(1) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_ARCACHE_OVERRIDE_MASK BIT_MASK(2) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_NOSNOOP_OVERRIDE_EN_MASK BIT_MASK(3) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_AW_NOSNOOP_OVERRIDE_MASK BIT_MASK(4) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_AR_NOSNOOP_OVERRIDE_MASK BIT_MASK(5) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_PTW_AW_CONTEXT_FLAG_MASK GENMASK(10, 6) +#define VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES_PTW_AR_CONTEXT_FLAG_MASK GENMASK(15, 11) + +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV 0x00360004u +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU0_AWMMUSSIDV_MASK BIT_MASK(0) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU0_ARMMUSSIDV_MASK BIT_MASK(1) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU1_AWMMUSSIDV_MASK BIT_MASK(2) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU1_ARMMUSSIDV_MASK BIT_MASK(3) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU2_AWMMUSSIDV_MASK BIT_MASK(4) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU2_ARMMUSSIDV_MASK BIT_MASK(5) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU3_AWMMUSSIDV_MASK BIT_MASK(6) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU3_ARMMUSSIDV_MASK BIT_MASK(7) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU4_AWMMUSSIDV_MASK BIT_MASK(8) +#define VPU_37XX_HOST_IF_TBU_MMUSSIDV_TBU4_ARMMUSSIDV_MASK BIT_MASK(9) + +#define VPU_37XX_CPU_SS_DSU_LEON_RT_BASE 0x04000000u +#define VPU_37XX_CPU_SS_DSU_LEON_RT_DSU_CTRL 0x04000000u +#define VPU_37XX_CPU_SS_DSU_LEON_RT_PC_REG 0x04400010u +#define VPU_37XX_CPU_SS_DSU_LEON_RT_NPC_REG 0x04400014u +#define VPU_37XX_CPU_SS_DSU_LEON_RT_DSU_TRAP_REG 0x04400020u + +#define VPU_37XX_CPU_SS_MSSCPU_CPR_CLK_SET 0x06010004u +#define VPU_37XX_CPU_SS_MSSCPU_CPR_CLK_SET_CPU_DSU_MASK BIT_MASK(1) + +#define VPU_37XX_CPU_SS_MSSCPU_CPR_RST_CLR 0x06010018u +#define VPU_37XX_CPU_SS_MSSCPU_CPR_RST_CLR_CPU_DSU_MASK BIT_MASK(1) + +#define VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC 0x06010040u +#define VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC_IRQI_RSTRUN0_MASK BIT_MASK(0) +#define VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC_IRQI_RESUME0_MASK BIT_MASK(1) +#define VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC_IRQI_RSTRUN1_MASK BIT_MASK(2) +#define VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC_IRQI_RESUME1_MASK BIT_MASK(3) +#define VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC_IRQI_RSTVEC_MASK GENMASK(31, 4) + +#define VPU_37XX_CPU_SS_TIM_WATCHDOG 0x0602009cu +#define VPU_37XX_CPU_SS_TIM_WDOG_EN 0x060200a4u +#define VPU_37XX_CPU_SS_TIM_SAFE 0x060200a8u +#define VPU_37XX_CPU_SS_TIM_IPC_FIFO 0x060200f0u + +#define VPU_37XX_CPU_SS_TIM_GEN_CONFIG 0x06021008u +#define VPU_37XX_CPU_SS_TIM_GEN_CONFIG_WDOG_TO_INT_CLR_MASK BIT_MASK(9) + +#define VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT 0x06029000u + +#define VPU_37XX_CPU_SS_DOORBELL_0 0x06300000u +#define VPU_37XX_CPU_SS_DOORBELL_0_SET_MASK BIT_MASK(0) + +#define VPU_37XX_CPU_SS_DOORBELL_1 0x06301000u + +#endif /* __IVPU_HW_37XX_REG_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_40xx_reg.h b/drivers/accel/ivpu/ivpu_hw_40xx_reg.h new file mode 100644 index 000000000..fc0ee8d63 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_40xx_reg.h @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_HW_40XX_REG_H__ +#define __IVPU_HW_40XX_REG_H__ + +#include + +#define VPU_40XX_HOST_SS_CPR_CLK_EN 0x00000080u +#define VPU_40XX_HOST_SS_CPR_CLK_EN_TOP_NOC_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_CPR_CLK_EN_DSS_MAS_MASK BIT_MASK(10) +#define VPU_40XX_HOST_SS_CPR_CLK_EN_CSS_MAS_MASK BIT_MASK(11) + +#define VPU_40XX_HOST_SS_CPR_CLK_SET 0x00000084u +#define VPU_40XX_HOST_SS_CPR_CLK_SET_TOP_NOC_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_CPR_CLK_SET_DSS_MAS_MASK BIT_MASK(10) +#define VPU_40XX_HOST_SS_CPR_CLK_SET_MSS_MAS_MASK BIT_MASK(11) + +#define VPU_40XX_HOST_SS_CPR_RST_EN 0x00000090u +#define VPU_40XX_HOST_SS_CPR_RST_EN_TOP_NOC_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_CPR_RST_EN_DSS_MAS_MASK BIT_MASK(10) +#define VPU_40XX_HOST_SS_CPR_RST_EN_CSS_MAS_MASK BIT_MASK(11) + +#define VPU_40XX_HOST_SS_CPR_RST_SET 0x00000094u +#define VPU_40XX_HOST_SS_CPR_RST_SET_TOP_NOC_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_CPR_RST_SET_DSS_MAS_MASK BIT_MASK(10) +#define VPU_40XX_HOST_SS_CPR_RST_SET_MSS_MAS_MASK BIT_MASK(11) + +#define VPU_40XX_HOST_SS_CPR_RST_CLR 0x00000098u +#define VPU_40XX_HOST_SS_CPR_RST_CLR_TOP_NOC_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_CPR_RST_CLR_DSS_MAS_MASK BIT_MASK(10) +#define VPU_40XX_HOST_SS_CPR_RST_CLR_MSS_MAS_MASK BIT_MASK(11) + +#define VPU_40XX_HOST_SS_HW_VERSION 0x00000108u +#define VPU_40XX_HOST_SS_HW_VERSION_SOC_REVISION_MASK GENMASK(7, 0) +#define VPU_40XX_HOST_SS_HW_VERSION_SOC_NUMBER_MASK GENMASK(15, 8) +#define VPU_40XX_HOST_SS_HW_VERSION_VPU_GENERATION_MASK GENMASK(23, 16) + +#define VPU_40XX_HOST_SS_SW_VERSION 0x0000010cu + +#define VPU_40XX_HOST_SS_GEN_CTRL 0x00000118u +#define VPU_40XX_HOST_SS_GEN_CTRL_PS_MASK GENMASK(31, 29) + +#define VPU_40XX_HOST_SS_NOC_QREQN 0x00000154u +#define VPU_40XX_HOST_SS_NOC_QREQN_TOP_SOCMMIO_MASK BIT_MASK(0) + +#define VPU_40XX_HOST_SS_NOC_QACCEPTN 0x00000158u +#define VPU_40XX_HOST_SS_NOC_QACCEPTN_TOP_SOCMMIO_MASK BIT_MASK(0) + +#define VPU_40XX_HOST_SS_NOC_QDENY 0x0000015cu +#define VPU_40XX_HOST_SS_NOC_QDENY_TOP_SOCMMIO_MASK BIT_MASK(0) + +#define VPU_40XX_TOP_NOC_QREQN 0x00000160u +#define VPU_40XX_TOP_NOC_QREQN_CPU_CTRL_MASK BIT_MASK(0) +#define VPU_40XX_TOP_NOC_QREQN_HOSTIF_L2CACHE_MASK BIT_MASK(2) + +#define VPU_40XX_TOP_NOC_QACCEPTN 0x00000164u +#define VPU_40XX_TOP_NOC_QACCEPTN_CPU_CTRL_MASK BIT_MASK(0) +#define VPU_40XX_TOP_NOC_QACCEPTN_HOSTIF_L2CACHE_MASK BIT_MASK(2) + +#define VPU_40XX_TOP_NOC_QDENY 0x00000168u +#define VPU_40XX_TOP_NOC_QDENY_CPU_CTRL_MASK BIT_MASK(0) +#define VPU_40XX_TOP_NOC_QDENY_HOSTIF_L2CACHE_MASK BIT_MASK(2) + +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN 0x00000170u +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_CSS_ROM_CMX_MASK BIT_MASK(0) +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_CSS_DBG_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_CSS_CTRL_MASK BIT_MASK(2) +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_DEC400_MASK BIT_MASK(3) +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_MSS_NCE_MASK BIT_MASK(4) +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_MSS_MBI_MASK BIT_MASK(5) +#define VPU_40XX_HOST_SS_FW_SOC_IRQ_EN_MSS_MBI_CMX_MASK BIT_MASK(6) + +#define VPU_40XX_HOST_SS_ICB_STATUS_0 0x00010210u +#define VPU_40XX_HOST_SS_ICB_STATUS_0_TIMER_0_INT_MASK BIT_MASK(0) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_TIMER_1_INT_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_TIMER_2_INT_MASK BIT_MASK(2) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_TIMER_3_INT_MASK BIT_MASK(3) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_HOST_IPC_FIFO_INT_MASK BIT_MASK(4) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_MMU_IRQ_0_INT_MASK BIT_MASK(5) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_MMU_IRQ_1_INT_MASK BIT_MASK(6) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_MMU_IRQ_2_INT_MASK BIT_MASK(7) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_NOC_FIREWALL_INT_MASK BIT_MASK(8) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_CPU_INT_REDIRECT_0_INT_MASK BIT_MASK(30) +#define VPU_40XX_HOST_SS_ICB_STATUS_0_CPU_INT_REDIRECT_1_INT_MASK BIT_MASK(31) + +#define VPU_40XX_HOST_SS_ICB_STATUS_1 0x00010214u +#define VPU_40XX_HOST_SS_ICB_STATUS_1_CPU_INT_REDIRECT_2_INT_MASK BIT_MASK(0) +#define VPU_40XX_HOST_SS_ICB_STATUS_1_CPU_INT_REDIRECT_3_INT_MASK BIT_MASK(1) +#define VPU_40XX_HOST_SS_ICB_STATUS_1_CPU_INT_REDIRECT_4_INT_MASK BIT_MASK(2) + +#define VPU_40XX_HOST_SS_ICB_CLEAR_0 0x00010220u +#define VPU_40XX_HOST_SS_ICB_CLEAR_1 0x00010224u +#define VPU_40XX_HOST_SS_ICB_ENABLE_0 0x00010240u +#define VPU_40XX_HOST_SS_ICB_ENABLE_1 0x00010244u + +#define VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM 0x000200f4u + +#define VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT 0x000200fcu +#define VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT_FILL_LEVEL_MASK GENMASK(23, 16) + +#define VPU_40XX_HOST_SS_AON_PWR_ISO_EN0 0x00030020u +#define VPU_40XX_HOST_SS_AON_PWR_ISO_EN0_CSS_CPU_MASK BIT_MASK(3) + +#define VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0 0x00030024u +#define VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0_CSS_CPU_MASK BIT_MASK(3) + +#define VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0 0x00030028u +#define VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0_CSS_CPU_MASK BIT_MASK(3) + +#define VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0 0x0003002cu +#define VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0_CSS_CPU_MASK BIT_MASK(3) + +#define VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY 0x00030068u +#define VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY_POST_DLY_MASK GENMASK(7, 0) +#define VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY_POST1_DLY_MASK GENMASK(15, 8) +#define VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY_POST2_DLY_MASK GENMASK(23, 16) + +#define VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY 0x0003006cu +#define VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY_STATUS_DLY_MASK GENMASK(7, 0) + +#define VPU_40XX_HOST_SS_AON_IDLE_GEN 0x00030200u +#define VPU_40XX_HOST_SS_AON_IDLE_GEN_EN_MASK BIT_MASK(0) +#define VPU_40XX_HOST_SS_AON_IDLE_GEN_HW_PG_EN_MASK BIT_MASK(1) + +#define VPU_40XX_HOST_SS_AON_DPU_ACTIVE 0x00030204u +#define VPU_40XX_HOST_SS_AON_DPU_ACTIVE_DPU_ACTIVE_MASK BIT_MASK(0) + +#define VPU_50XX_HOST_SS_AON_FABRIC_REQ_OVERRIDE 0x00030210u +#define VPU_50XX_HOST_SS_AON_FABRIC_REQ_OVERRIDE_REQ_OVERRIDE_MASK BIT_MASK(0) + +#define VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO 0x00040040u +#define VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_DONE_MASK BIT_MASK(0) +#define VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IOSF_RS_ID_MASK GENMASK(2, 1) +#define VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK GENMASK(31, 3) + +#define VPU_40XX_HOST_SS_WORKPOINT_CONFIG_MIRROR 0x00082020u +#define VPU_40XX_HOST_SS_WORKPOINT_CONFIG_MIRROR_FINAL_PLL_FREQ_MASK GENMASK(15, 0) +#define VPU_40XX_HOST_SS_WORKPOINT_CONFIG_MIRROR_CONFIG_ID_MASK GENMASK(31, 16) + +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES 0x00360000u +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_CACHE_OVERRIDE_EN_MASK BIT_MASK(0) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_AWCACHE_OVERRIDE_MASK BIT_MASK(1) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_ARCACHE_OVERRIDE_MASK BIT_MASK(2) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_SNOOP_OVERRIDE_EN_MASK BIT_MASK(3) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_AW_SNOOP_OVERRIDE_MASK BIT_MASK(4) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_AR_SNOOP_OVERRIDE_MASK BIT_MASK(5) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_PTW_AW_CONTEXT_FLAG_MASK GENMASK(10, 6) +#define VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES_PTW_AR_CONTEXT_FLAG_MASK GENMASK(15, 11) + +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV 0x00360004u +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU0_AWMMUSSIDV_MASK BIT_MASK(0) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU0_ARMMUSSIDV_MASK BIT_MASK(1) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU1_AWMMUSSIDV_MASK BIT_MASK(2) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU1_ARMMUSSIDV_MASK BIT_MASK(3) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU2_AWMMUSSIDV_MASK BIT_MASK(4) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU2_ARMMUSSIDV_MASK BIT_MASK(5) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU3_AWMMUSSIDV_MASK BIT_MASK(6) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU3_ARMMUSSIDV_MASK BIT_MASK(7) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU4_AWMMUSSIDV_MASK BIT_MASK(8) +#define VPU_40XX_HOST_IF_TBU_MMUSSIDV_TBU4_ARMMUSSIDV_MASK BIT_MASK(9) + +#define VPU_40XX_CPU_SS_DSU_LEON_RT_BASE 0x04000000u +#define VPU_40XX_CPU_SS_DSU_LEON_RT_DSU_CTRL 0x04000000u +#define VPU_40XX_CPU_SS_DSU_LEON_RT_PC_REG 0x04400010u +#define VPU_40XX_CPU_SS_DSU_LEON_RT_NPC_REG 0x04400014u +#define VPU_40XX_CPU_SS_DSU_LEON_RT_DSU_TRAP_REG 0x04400020u + +#define VPU_40XX_CPU_SS_TIM_WATCHDOG 0x0102009cu +#define VPU_40XX_CPU_SS_TIM_WDOG_EN 0x010200a4u +#define VPU_40XX_CPU_SS_TIM_SAFE 0x010200a8u + +#define VPU_40XX_CPU_SS_TIM_GEN_CONFIG 0x01021008u +#define VPU_40XX_CPU_SS_TIM_GEN_CONFIG_WDOG_TO_INT_CLR_MASK BIT_MASK(9) + +#define VPU_40XX_CPU_SS_CPR_NOC_QREQN 0x01010030u +#define VPU_40XX_CPU_SS_CPR_NOC_QREQN_TOP_MMIO_MASK BIT_MASK(0) + +#define VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN 0x01010034u +#define VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN_TOP_MMIO_MASK BIT_MASK(0) + +#define VPU_40XX_CPU_SS_CPR_NOC_QDENY 0x01010038u +#define VPU_40XX_CPU_SS_CPR_NOC_QDENY_TOP_MMIO_MASK BIT_MASK(0) + +#define VPU_40XX_CPU_SS_TIM_IPC_FIFO 0x010200f0u +#define VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT 0x01029008u + +#define VPU_40XX_CPU_SS_DOORBELL_0 0x01300000u +#define VPU_40XX_CPU_SS_DOORBELL_0_SET_MASK BIT_MASK(0) + +#define VPU_40XX_CPU_SS_DOORBELL_1 0x01301000u + +#endif /* __IVPU_HW_40XX_REG_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c new file mode 100644 index 000000000..a17c829ad --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_btrs.c @@ -0,0 +1,971 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include + +#include "ivpu_drv.h" +#include "ivpu_hw.h" +#include "ivpu_hw_btrs.h" +#include "ivpu_hw_btrs_lnl_reg.h" +#include "ivpu_hw_btrs_mtl_reg.h" +#include "ivpu_hw_reg_io.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_pm.h" + +#define BTRS_MTL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR)) | \ + (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR))) + +#define BTRS_LNL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR)) | \ + (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR)) | \ + (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR)) | \ + (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR)) | \ + (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR)) | \ + (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR))) + +#define BTRS_MTL_ALL_IRQ_MASK (BTRS_MTL_IRQ_MASK | (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, \ + FREQ_CHANGE))) + +#define BTRS_IRQ_DISABLE_MASK ((u32)-1) + +#define BTRS_LNL_ALL_IRQ_MASK ((u32)-1) + + +#define PLL_CDYN_DEFAULT 0x80 +#define PLL_EPP_DEFAULT 0x80 +#define PLL_REF_CLK_FREQ_MHZ 50 + +#define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC) +#define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC) +#define TIMEOUT_US (150 * USEC_PER_MSEC) + +/* Work point configuration values */ +#define WP_CONFIG(tile, ratio) (((tile) << 8) | (ratio)) +#define MTL_CONFIG_1_TILE 0x01 +#define MTL_CONFIG_2_TILE 0x02 +#define MTL_PLL_RATIO_5_3 0x01 +#define MTL_PLL_RATIO_4_3 0x02 +#define BTRS_MTL_TILE_FUSE_ENABLE_BOTH 0x0 +#define BTRS_MTL_TILE_SKU_BOTH 0x3630 + +#define BTRS_LNL_TILE_MAX_NUM 6 +#define BTRS_LNL_TILE_MAX_MASK 0x3f + +#define WEIGHTS_DEFAULT 0xf711f711u +#define WEIGHTS_ATS_DEFAULT 0x0000f711u + +#define DCT_REQ 0x2 +#define DCT_ENABLE 0x1 +#define DCT_DISABLE 0x0 + +int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev) +{ + REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, BTRS_MTL_ALL_IRQ_MASK); + if (REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) == BTRS_MTL_ALL_IRQ_MASK) { + /* Writing 1s does not clear the interrupt status register */ + REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0); + return true; + } + + return false; +} + +static void freq_ratios_init_mtl(struct ivpu_device *vdev) +{ + struct ivpu_hw_info *hw = vdev->hw; + u32 fmin_fuse, fmax_fuse; + + fmin_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMIN_FUSE); + hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, MIN_RATIO, fmin_fuse); + hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, PN_RATIO, fmin_fuse); + + fmax_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMAX_FUSE); + hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMAX_FUSE, MAX_RATIO, fmax_fuse); +} + +static void freq_ratios_init_lnl(struct ivpu_device *vdev) +{ + struct ivpu_hw_info *hw = vdev->hw; + u32 fmin_fuse, fmax_fuse; + + fmin_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMIN_FUSE); + hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, MIN_RATIO, fmin_fuse); + hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, PN_RATIO, fmin_fuse); + + fmax_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMAX_FUSE); + hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMAX_FUSE, MAX_RATIO, fmax_fuse); +} + +void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev) +{ + struct ivpu_hw_info *hw = vdev->hw; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + freq_ratios_init_mtl(vdev); + else + freq_ratios_init_lnl(vdev); + + hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, hw->pll.min_ratio, hw->pll.max_ratio); + hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, hw->pll.max_ratio); + hw->pll.pn_ratio = clamp_t(u8, hw->pll.pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio); + hw->pll.cfg_max_ratio = hw->pll.max_ratio; + hw->pll.cfg_min_ratio = hw->pll.min_ratio; +} + +static bool tile_disable_check(u32 config) +{ + /* Allowed values: 0 or one bit from range 0-5 (6 tiles) */ + if (config == 0) + return true; + + if (config > BIT(BTRS_LNL_TILE_MAX_NUM - 1)) + return false; + + if ((config & (config - 1)) == 0) + return true; + + return false; +} + +static int read_tile_config_fuse(struct ivpu_device *vdev, u32 *tile_fuse_config) +{ + u32 fuse; + u32 config; + + fuse = REGB_RD32(VPU_HW_BTRS_LNL_TILE_FUSE); + if (!REG_TEST_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, VALID, fuse)) { + ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse); + return -EIO; + } + + config = REG_GET_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, CONFIG, fuse); + if (!tile_disable_check(config)) + ivpu_warn(vdev, "More than 1 tile disabled, tile fuse config mask: 0x%x\n", config); + + ivpu_dbg(vdev, MISC, "Tile disable config mask: 0x%x\n", config); + + *tile_fuse_config = config; + return 0; +} + +static int info_init_mtl(struct ivpu_device *vdev) +{ + struct ivpu_hw_info *hw = vdev->hw; + + hw->tile_fuse = BTRS_MTL_TILE_FUSE_ENABLE_BOTH; + hw->sku = BTRS_MTL_TILE_SKU_BOTH; + hw->config = WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_4_3); + + return 0; +} + +static int info_init_lnl(struct ivpu_device *vdev) +{ + struct ivpu_hw_info *hw = vdev->hw; + u32 tile_fuse_config; + int ret; + + ret = read_tile_config_fuse(vdev, &tile_fuse_config); + if (ret) + return ret; + + hw->tile_fuse = tile_fuse_config; + hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; + + return 0; +} + +int ivpu_hw_btrs_info_init(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return info_init_mtl(vdev); + else + return info_init_lnl(vdev); +} + +static int wp_request_sync(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return REGB_POLL_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); + else + return REGB_POLL_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); +} + +static int wait_for_status_ready(struct ivpu_device *vdev, bool enable) +{ + u32 exp_val = enable ? 0x1 : 0x0; + + if (IVPU_WA(punit_disabled)) + return 0; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US); + else + return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US); +} + +struct wp_request { + u16 min; + u16 max; + u16 target; + u16 cfg; + u16 epp; + u16 cdyn; +}; + +static void wp_request_mtl(struct ivpu_device *vdev, struct wp_request *wp) +{ + u32 val; + + val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val); + REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, val); + + val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, EPP, PLL_EPP_DEFAULT, val); + REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, val); + + val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val); + REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, val); + + val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_CMD); + val = REG_SET_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, val); + REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_CMD, val); +} + +static void wp_request_lnl(struct ivpu_device *vdev, struct wp_request *wp) +{ + u32 val; + + val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val); + REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, val); + + val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, EPP, wp->epp, val); + REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, val); + + val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CDYN, wp->cdyn, val); + REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, val); + + val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_CMD); + val = REG_SET_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, val); + REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_CMD, val); +} + +static void wp_request(struct ivpu_device *vdev, struct wp_request *wp) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + wp_request_mtl(vdev, wp); + else + wp_request_lnl(vdev, wp); +} + +static int wp_request_send(struct ivpu_device *vdev, struct wp_request *wp) +{ + int ret; + + ret = wp_request_sync(vdev); + if (ret) { + ivpu_err(vdev, "Failed to sync before workpoint request: %d\n", ret); + return ret; + } + + wp_request(vdev, wp); + + ret = wp_request_sync(vdev); + if (ret) + ivpu_err(vdev, "Failed to sync after workpoint request: %d\n", ret); + + return ret; +} + +static void prepare_wp_request(struct ivpu_device *vdev, struct wp_request *wp, bool enable) +{ + struct ivpu_hw_info *hw = vdev->hw; + + wp->min = hw->pll.min_ratio; + wp->max = hw->pll.max_ratio; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { + wp->target = enable ? hw->pll.pn_ratio : 0; + wp->cfg = enable ? hw->config : 0; + wp->cdyn = 0; + wp->epp = 0; + } else { + wp->target = hw->pll.pn_ratio; + wp->cfg = 0; + wp->cdyn = enable ? PLL_CDYN_DEFAULT : 0; + wp->epp = enable ? PLL_EPP_DEFAULT : 0; + } +} + +static int wait_for_pll_lock(struct ivpu_device *vdev, bool enable) +{ + u32 exp_val = enable ? 0x1 : 0x0; + + if (ivpu_hw_btrs_gen(vdev) != IVPU_HW_BTRS_MTL) + return 0; + + if (IVPU_WA(punit_disabled)) + return 0; + + return REGB_POLL_FLD(VPU_HW_BTRS_MTL_PLL_STATUS, LOCK, exp_val, PLL_TIMEOUT_US); +} + +static int wait_for_cdyn_deassert(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return 0; + + return REGB_POLL_FLD(VPU_HW_BTRS_LNL_CDYN, CDYN, 0, PLL_TIMEOUT_US); +} + +int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable) +{ + struct wp_request wp; + int ret; + + if (IVPU_WA(punit_disabled)) { + ivpu_dbg(vdev, PM, "Skipping workpoint request\n"); + return 0; + } + + prepare_wp_request(vdev, &wp, enable); + + ivpu_dbg(vdev, PM, "PLL workpoint request: %u MHz, config: 0x%x, epp: 0x%x, cdyn: 0x%x\n", + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, wp.target), wp.cfg, wp.epp, wp.cdyn); + + ret = wp_request_send(vdev, &wp); + if (ret) { + ivpu_err(vdev, "Failed to send workpoint request: %d\n", ret); + return ret; + } + + ret = wait_for_pll_lock(vdev, enable); + if (ret) { + ivpu_err(vdev, "Timed out waiting for PLL lock\n"); + return ret; + } + + ret = wait_for_status_ready(vdev, enable); + if (ret) { + ivpu_err(vdev, "Timed out waiting for NPU ready status\n"); + return ret; + } + + if (!enable) { + ret = wait_for_cdyn_deassert(vdev); + if (ret) { + ivpu_err(vdev, "Timed out waiting for CDYN deassert\n"); + return ret; + } + } + + return 0; +} + +static int d0i3_drive_mtl(struct ivpu_device *vdev, bool enable) +{ + int ret; + u32 val; + + ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); + if (ret) { + ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); + return ret; + } + + val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL); + if (enable) + val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val); + else + val = REG_CLR_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val); + REGB_WR32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, val); + + ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); + if (ret) + ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); + + return ret; +} + +static int d0i3_drive_lnl(struct ivpu_device *vdev, bool enable) +{ + int ret; + u32 val; + + ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); + if (ret) { + ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); + return ret; + } + + val = REGB_RD32(VPU_HW_BTRS_LNL_D0I3_CONTROL); + if (enable) + val = REG_SET_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val); + else + val = REG_CLR_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val); + REGB_WR32(VPU_HW_BTRS_LNL_D0I3_CONTROL, val); + + ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); + if (ret) { + ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); + return ret; + } + + return 0; +} + +static int d0i3_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return d0i3_drive_mtl(vdev, enable); + else + return d0i3_drive_lnl(vdev, enable); +} + +int ivpu_hw_btrs_d0i3_enable(struct ivpu_device *vdev) +{ + int ret; + + if (IVPU_WA(punit_disabled)) + return 0; + + ret = d0i3_drive(vdev, true); + if (ret) + ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret); + + udelay(5); /* VPU requires 5 us to complete the transition */ + + return ret; +} + +int ivpu_hw_btrs_d0i3_disable(struct ivpu_device *vdev) +{ + int ret; + + if (IVPU_WA(punit_disabled)) + return 0; + + ret = d0i3_drive(vdev, false); + if (ret) + ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret); + + return ret; +} + +int ivpu_hw_btrs_wait_for_clock_res_own_ack(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return 0; + + return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US); +} + +void ivpu_hw_btrs_set_port_arbitration_weights_lnl(struct ivpu_device *vdev) +{ + REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT); + REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT); +} + +static int ip_reset_mtl(struct ivpu_device *vdev) +{ + int ret; + u32 val; + + ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US); + if (ret) { + ivpu_err(vdev, "Timed out waiting for TRIGGER bit\n"); + return ret; + } + + val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_IP_RESET); + val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, val); + REGB_WR32(VPU_HW_BTRS_MTL_VPU_IP_RESET, val); + + ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US); + if (ret) + ivpu_err(vdev, "Timed out waiting for RESET completion\n"); + + return ret; +} + +static int ip_reset_lnl(struct ivpu_device *vdev) +{ + int ret; + u32 val; + + ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev); + + ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US); + if (ret) { + ivpu_err(vdev, "Wait for *_TRIGGER timed out\n"); + return ret; + } + + val = REGB_RD32(VPU_HW_BTRS_LNL_IP_RESET); + val = REG_SET_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, val); + REGB_WR32(VPU_HW_BTRS_LNL_IP_RESET, val); + + ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US); + if (ret) + ivpu_err(vdev, "Timed out waiting for RESET completion\n"); + + return ret; +} + +int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev) +{ + if (IVPU_WA(punit_disabled)) + return 0; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return ip_reset_mtl(vdev); + else + return ip_reset_lnl(vdev); +} + +void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev) +{ + u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); + + if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT) + val = REG_CLR_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val); + else + val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val); + + REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val); +} + +void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev) +{ + ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n", + REGB_RD32(VPU_HW_BTRS_LNL_HM_ATS) ? "Enable" : "Disable"); +} + +void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev) +{ + u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); + + val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, DISABLE_CLK_RELINQUISH, val); + REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val); +} + +bool ivpu_hw_btrs_is_idle(struct ivpu_device *vdev) +{ + u32 val; + + if (IVPU_WA(punit_disabled)) + return true; + + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { + val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_STATUS); + + return REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, val) && + REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, val); + } else { + val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); + + return REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, val) && + REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, val); + } +} + +int ivpu_hw_btrs_wait_for_idle(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); + else + return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); +} + +static u32 pll_config_get_mtl(struct ivpu_device *vdev) +{ + return REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL); +} + +static u32 pll_config_get_lnl(struct ivpu_device *vdev) +{ + return REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ); +} + +static u32 pll_ratio_to_mhz_mtl(u8 pll_ratio) +{ + return (pll_ratio * PLL_REF_CLK_FREQ_MHZ * 2) / 3; +} + +static u32 pll_ratio_to_mhz_lnl(u8 pll_ratio) +{ + return (pll_ratio * PLL_REF_CLK_FREQ_MHZ) / 2; +} + +u32 ivpu_hw_btrs_pll_ratio_to_mhz(struct ivpu_device *vdev, u8 pll_ratio) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return pll_ratio_to_mhz_mtl(pll_ratio); + else + return pll_ratio_to_mhz_lnl(pll_ratio); +} + +u32 ivpu_hw_btrs_pll_ratio_to_hz(struct ivpu_device *vdev, u8 pll_ratio) +{ + return ivpu_hw_btrs_pll_ratio_to_mhz(vdev, pll_ratio) * HZ_PER_MHZ; +} + +u32 ivpu_hw_btrs_current_freq_get(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return pll_ratio_to_mhz_mtl(pll_config_get_mtl(vdev)); + else + return pll_ratio_to_mhz_lnl(pll_config_get_lnl(vdev)); +} + +static int ivpu_hw_btrs_cfg_freq_set(struct ivpu_device *vdev, u8 cfg_min_ratio, u8 cfg_max_ratio) +{ + u8 min_ratio = clamp_t(u8, cfg_min_ratio, vdev->hw->pll.min_ratio, cfg_max_ratio); + u8 pn_ratio = clamp_t(u8, vdev->hw->pll.pn_ratio, min_ratio, cfg_max_ratio); + int ret; + + ivpu_dbg(vdev, PM, "Set frequency range to min: %u, pn: %u, max: %u MHz\n", + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, min_ratio), + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, pn_ratio), + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, cfg_max_ratio)); + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + ret = ivpu_jsm_msg_freq_config(vdev, min_ratio, pn_ratio, cfg_max_ratio); + ivpu_rpm_put(vdev); + + if (ret) { + ivpu_warn(vdev, + "Failed to set frequency to min: %u, pn: %u, max: %u MHz, ret %d\n", + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, min_ratio), + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, pn_ratio), + ivpu_hw_btrs_pll_ratio_to_mhz(vdev, cfg_max_ratio), + ret); + return ret; + } + + vdev->hw->pll.cfg_min_ratio = cfg_min_ratio; + vdev->hw->pll.cfg_max_ratio = cfg_max_ratio; + + return 0; +} + +static u8 dpu_mhz_to_pll_ratio_lnl(u32 freq_mhz) +{ + return clamp_t(u32, freq_mhz / (PLL_REF_CLK_FREQ_MHZ / 2), 0, U8_MAX); +} + +int ivpu_hw_btrs_cfg_max_freq_set(struct ivpu_device *vdev, u32 max_freq_mhz) +{ + u8 ratio = dpu_mhz_to_pll_ratio_lnl(max_freq_mhz); + u8 cfg_max_ratio = clamp_t(u8, ratio, vdev->hw->pll.min_ratio, vdev->hw->pll.max_ratio); + + return ivpu_hw_btrs_cfg_freq_set(vdev, vdev->hw->pll.cfg_min_ratio, cfg_max_ratio); +} + +int ivpu_hw_btrs_cfg_min_freq_set(struct ivpu_device *vdev, u32 min_freq_mhz) +{ + u8 ratio = dpu_mhz_to_pll_ratio_lnl(min_freq_mhz); + u8 cfg_min_ratio = clamp_t(u8, ratio, vdev->hw->pll.min_ratio, vdev->hw->pll.max_ratio); + + return ivpu_hw_btrs_cfg_freq_set(vdev, cfg_min_ratio, vdev->hw->pll.cfg_max_ratio); +} + +int ivpu_hw_btrs_cfg_freq_init(struct ivpu_device *vdev) +{ + if (vdev->hw->pll.min_ratio == vdev->hw->pll.cfg_min_ratio && + vdev->hw->pll.max_ratio == vdev->hw->pll.cfg_max_ratio) + return 0; + + return ivpu_hw_btrs_cfg_freq_set(vdev, + vdev->hw->pll.cfg_min_ratio, + vdev->hw->pll.cfg_max_ratio); +} + +/* Handler for IRQs from Buttress core (irqB) */ +bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq) +{ + u32 status = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK; + bool schedule_recovery = false; + + if (!status) + return false; + + if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, FREQ_CHANGE, status)) { + u32 pll = pll_config_get_mtl(vdev); + + ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq, wp %08x, %u MHz", + pll, pll_ratio_to_mhz_mtl(pll)); + } + + if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, status)) { + ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0)); + REGB_WR32(VPU_HW_BTRS_MTL_ATS_ERR_CLEAR, 0x1); + schedule_recovery = true; + } + + if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, status)) { + u32 ufi_log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG); + + ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx", + ufi_log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, ufi_log), + REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, ufi_log), + REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, ufi_log)); + REGB_WR32(VPU_HW_BTRS_MTL_UFI_ERR_CLEAR, 0x1); + schedule_recovery = true; + } + + /* This must be done after interrupts are cleared at the source. */ + if (IVPU_WA(interrupt_clear_with_0)) + /* + * Writing 1 triggers an interrupt, so we can't perform read update write. + * Clear local interrupt status by writing 0 to all bits. + */ + REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0); + else + REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, status); + + if (schedule_recovery) + ivpu_pm_trigger_recovery(vdev, "Buttress IRQ"); + + return true; +} + +/* Handler for IRQs from Buttress core (irqB) */ +bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq) +{ + u32 status = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; + bool schedule_recovery = false; + + if (!status) + return false; + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status)) { + ivpu_dbg(vdev, IRQ, "Survivability IRQ\n"); + queue_work(system_percpu_wq, &vdev->irq_dct_work); + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status)) { + u32 pll = pll_config_get_lnl(vdev); + + ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq, wp %08x, %u MHz", + pll, pll_ratio_to_mhz_lnl(pll)); + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, status)) { + ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", + REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1), + REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2)); + REGB_WR32(VPU_HW_BTRS_LNL_ATS_ERR_CLEAR, 0x1); + schedule_recovery = true; + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, status)) { + ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG)); + REGB_WR32(VPU_HW_BTRS_LNL_CFI0_ERR_CLEAR, 0x1); + schedule_recovery = true; + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, status)) { + ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG)); + REGB_WR32(VPU_HW_BTRS_LNL_CFI1_ERR_CLEAR, 0x1); + schedule_recovery = true; + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, status)) { + ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x", + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW), + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH)); + REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_CLEAR, 0x1); + schedule_recovery = true; + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, status)) { + ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x", + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW), + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH)); + REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR, 0x1); + schedule_recovery = true; + } + + /* This must be done after interrupts are cleared at the source. */ + REGB_WR32(VPU_HW_BTRS_LNL_INTERRUPT_STAT, status); + + if (schedule_recovery) + ivpu_pm_trigger_recovery(vdev, "Buttress IRQ"); + + return true; +} + +int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable) +{ + u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW); + u32 cmd = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, CMD, val); + u32 param1 = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, PARAM1, val); + + if (cmd != DCT_REQ) { + ivpu_err_ratelimited(vdev, "Unsupported PCODE command: 0x%x\n", cmd); + return -EBADR; + } + + switch (param1) { + case DCT_ENABLE: + *enable = true; + return 0; + case DCT_DISABLE: + *enable = false; + return 0; + default: + ivpu_err_ratelimited(vdev, "Invalid PARAM1 value: %u\n", param1); + return -EINVAL; + } +} + +void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u8 active_percent) +{ + u32 val = 0; + u32 cmd = enable ? DCT_ENABLE : DCT_DISABLE; + + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, CMD, DCT_REQ, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM1, cmd, val); + val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM2, active_percent, val); + + REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, val); +} + +u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_OFFSET); + else + return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_OFFSET); +} + +u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_SIZE); + else + return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_SIZE); +} + +u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_ENABLE); + else + return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_ENABLE); +} + +void ivpu_hw_btrs_global_int_disable(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1); + else + REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1); +} + +void ivpu_hw_btrs_global_int_enable(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0); + else + REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0); +} + +void ivpu_hw_btrs_irq_enable(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { + REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, (u32)(~BTRS_MTL_IRQ_MASK)); + REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0); + } else { + REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, (u32)(~BTRS_LNL_IRQ_MASK)); + REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0); + } +} + +void ivpu_hw_btrs_irq_disable(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { + REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1); + REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK); + } else { + REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1); + REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK); + } +} + +static void diagnose_failure_mtl(struct ivpu_device *vdev) +{ + u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK; + + if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, reg)) + ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0)); + + if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, reg)) { + u32 log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG); + + ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx", + log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, log), + REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, log), + REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, log)); + } +} + +static void diagnose_failure_lnl(struct ivpu_device *vdev) +{ + u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) { + ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", + REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1), + REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2)); + } + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, reg)) + ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG)); + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, reg)) + ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG)); + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, reg)) + ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n", + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW), + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH)); + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, reg)) + ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n", + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW), + REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH)); + + if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, reg)) + ivpu_err(vdev, "Survivability IRQ\n"); +} + +void ivpu_hw_btrs_diagnose_failure(struct ivpu_device *vdev) +{ + if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) + return diagnose_failure_mtl(vdev); + else + return diagnose_failure_lnl(vdev); +} + +int ivpu_hw_btrs_platform_read(struct ivpu_device *vdev) +{ + u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); + + return REG_GET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PLATFORM, reg); +} diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.h b/drivers/accel/ivpu/ivpu_hw_btrs.h new file mode 100644 index 000000000..d6343d734 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_btrs.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#ifndef __IVPU_HW_BTRS_H__ +#define __IVPU_HW_BTRS_H__ + +#include "ivpu_drv.h" +#include "ivpu_hw_37xx_reg.h" +#include "ivpu_hw_40xx_reg.h" +#include "ivpu_hw_reg_io.h" + +#define PLL_PROFILING_FREQ_DEFAULT 38400000 +#define PLL_PROFILING_FREQ_HIGH 400000000 + +#define DCT_DEFAULT_ACTIVE_PERCENT 30u +#define DCT_PERIOD_US 35300u + +int ivpu_hw_btrs_info_init(struct ivpu_device *vdev); +void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev); +int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev); +int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable); +int ivpu_hw_btrs_wait_for_clock_res_own_ack(struct ivpu_device *vdev); +int ivpu_hw_btrs_d0i3_enable(struct ivpu_device *vdev); +int ivpu_hw_btrs_d0i3_disable(struct ivpu_device *vdev); +void ivpu_hw_btrs_set_port_arbitration_weights_lnl(struct ivpu_device *vdev); +bool ivpu_hw_btrs_is_idle(struct ivpu_device *vdev); +int ivpu_hw_btrs_wait_for_idle(struct ivpu_device *vdev); +int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev); +void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev); +void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev); +void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev); +u32 ivpu_hw_btrs_pll_ratio_to_mhz(struct ivpu_device *vdev, u8 pll_ratio); +u32 ivpu_hw_btrs_pll_ratio_to_hz(struct ivpu_device *vdev, u8 pll_ratio); +u32 ivpu_hw_btrs_current_freq_get(struct ivpu_device *vdev); +int ivpu_hw_btrs_cfg_min_freq_set(struct ivpu_device *vdev, u32 freq_mhz); +int ivpu_hw_btrs_cfg_max_freq_set(struct ivpu_device *vdev, u32 freq_mhz); +int ivpu_hw_btrs_cfg_freq_init(struct ivpu_device *vdev); +bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq); +bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq); +int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable); +void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u8 active_percent); +u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev); +u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev); +u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev); +void ivpu_hw_btrs_global_int_enable(struct ivpu_device *vdev); +void ivpu_hw_btrs_global_int_disable(struct ivpu_device *vdev); +void ivpu_hw_btrs_irq_enable(struct ivpu_device *vdev); +void ivpu_hw_btrs_irq_disable(struct ivpu_device *vdev); +void ivpu_hw_btrs_diagnose_failure(struct ivpu_device *vdev); +int ivpu_hw_btrs_platform_read(struct ivpu_device *vdev); + +#endif /* __IVPU_HW_BTRS_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h b/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h new file mode 100644 index 000000000..a81a9ba54 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_btrs_lnl_reg.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __IVPU_HW_BTRS_LNL_REG_H__ +#define __IVPU_HW_BTRS_LNL_REG_H__ + +#include + +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT 0x00000000u +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_FREQ_CHANGE_MASK BIT_MASK(0) +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_ATS_ERR_MASK BIT_MASK(1) +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_CFI0_ERR_MASK BIT_MASK(2) +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_CFI1_ERR_MASK BIT_MASK(3) +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_IMR0_ERR_MASK BIT_MASK(4) +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_IMR1_ERR_MASK BIT_MASK(5) +#define VPU_HW_BTRS_LNL_INTERRUPT_STAT_SURV_ERR_MASK BIT_MASK(6) + +#define VPU_HW_BTRS_LNL_LOCAL_INT_MASK 0x00000004u +#define VPU_HW_BTRS_LNL_GLOBAL_INT_MASK 0x00000008u + +#define VPU_HW_BTRS_LNL_HM_ATS 0x0000000cu + +#define VPU_HW_BTRS_LNL_ATS_ERR_LOG1 0x00000010u +#define VPU_HW_BTRS_LNL_ATS_ERR_LOG2 0x00000014u +#define VPU_HW_BTRS_LNL_ATS_ERR_CLEAR 0x00000018u + +#define VPU_HW_BTRS_LNL_CFI0_ERR_LOG 0x0000001cu +#define VPU_HW_BTRS_LNL_CFI0_ERR_CLEAR 0x00000020u + +#define VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS_ATS 0x00000024u + +#define VPU_HW_BTRS_LNL_CFI1_ERR_LOG 0x00000040u +#define VPU_HW_BTRS_LNL_CFI1_ERR_CLEAR 0x00000044u + +#define VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW 0x00000048u +#define VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH 0x0000004cu +#define VPU_HW_BTRS_LNL_IMR_ERR_CFI0_CLEAR 0x00000050u + +#define VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS 0x00000054u + +#define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW 0x00000058u +#define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH 0x0000005cu +#define VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR 0x00000060u + +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS 0x00000070u +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_CMD_MASK GENMASK(7, 0) +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM1_MASK GENMASK(15, 8) +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM2_MASK GENMASK(23, 16) +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS_PARAM3_MASK GENMASK(31, 24) + +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW 0x00000074u +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW_CMD_MASK GENMASK(7, 0) +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW_PARAM1_MASK GENMASK(15, 8) +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW_PARAM2_MASK GENMASK(23, 16) +#define VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW_PARAM3_MASK GENMASK(31, 24) + +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0 0x00000130u +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0_MIN_RATIO_MASK GENMASK(15, 0) +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0_MAX_RATIO_MASK GENMASK(31, 16) + +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1 0x00000134u +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1_TARGET_RATIO_MASK GENMASK(15, 0) +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1_EPP_MASK GENMASK(31, 16) + +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2 0x00000138u +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2_CONFIG_MASK GENMASK(15, 0) +#define VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2_CDYN_MASK GENMASK(31, 16) + +#define VPU_HW_BTRS_LNL_WP_REQ_CMD 0x0000013cu +#define VPU_HW_BTRS_LNL_WP_REQ_CMD_SEND_MASK BIT_MASK(0) + +#define VPU_HW_BTRS_LNL_PLL_FREQ 0x00000148u +#define VPU_HW_BTRS_LNL_PLL_FREQ_RATIO_MASK GENMASK(15, 0) + +#define VPU_HW_BTRS_LNL_CDYN 0x0000014cu +#define VPU_HW_BTRS_LNL_CDYN_CDYN_MASK GENMASK(15, 0) + +#define VPU_HW_BTRS_LNL_TILE_FUSE 0x00000150u +#define VPU_HW_BTRS_LNL_TILE_FUSE_VALID_MASK BIT_MASK(0) +#define VPU_HW_BTRS_LNL_TILE_FUSE_CONFIG_MASK GENMASK(6, 1) + +#define VPU_HW_BTRS_LNL_VPU_STATUS 0x00000154u +#define VPU_HW_BTRS_LNL_VPU_STATUS_READY_MASK BIT_MASK(0) +#define VPU_HW_BTRS_LNL_VPU_STATUS_IDLE_MASK BIT_MASK(1) +#define VPU_HW_BTRS_LNL_VPU_STATUS_DUP_IDLE_MASK BIT_MASK(2) +#define VPU_HW_BTRS_LNL_VPU_STATUS_CLOCK_RESOURCE_OWN_ACK_MASK BIT_MASK(6) +#define VPU_HW_BTRS_LNL_VPU_STATUS_POWER_RESOURCE_OWN_ACK_MASK BIT_MASK(7) +#define VPU_HW_BTRS_LNL_VPU_STATUS_PERF_CLK_MASK BIT_MASK(11) +#define VPU_HW_BTRS_LNL_VPU_STATUS_DISABLE_CLK_RELINQUISH_MASK BIT_MASK(12) +#define VPU_HW_BTRS_LNL_VPU_STATUS_PLATFORM_MASK GENMASK(31, 29) + +#define VPU_HW_BTRS_LNL_IP_RESET 0x00000160u +#define VPU_HW_BTRS_LNL_IP_RESET_TRIGGER_MASK BIT_MASK(0) + +#define VPU_HW_BTRS_LNL_D0I3_CONTROL 0x00000164u +#define VPU_HW_BTRS_LNL_D0I3_CONTROL_INPROGRESS_MASK BIT_MASK(0) +#define VPU_HW_BTRS_LNL_D0I3_CONTROL_I3_MASK BIT_MASK(2) + +#define VPU_HW_BTRS_LNL_VPU_TELEMETRY_OFFSET 0x00000168u +#define VPU_HW_BTRS_LNL_VPU_TELEMETRY_SIZE 0x0000016cu +#define VPU_HW_BTRS_LNL_VPU_TELEMETRY_ENABLE 0x00000170u + +#define VPU_HW_BTRS_LNL_FMIN_FUSE 0x00000174u +#define VPU_HW_BTRS_LNL_FMIN_FUSE_MIN_RATIO_MASK GENMASK(7, 0) +#define VPU_HW_BTRS_LNL_FMIN_FUSE_PN_RATIO_MASK GENMASK(15, 8) + +#define VPU_HW_BTRS_LNL_FMAX_FUSE 0x00000178u +#define VPU_HW_BTRS_LNL_FMAX_FUSE_MAX_RATIO_MASK GENMASK(7, 0) + +#endif /* __IVPU_HW_BTRS_LNL_REG_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_btrs_mtl_reg.h b/drivers/accel/ivpu/ivpu_hw_btrs_mtl_reg.h new file mode 100644 index 000000000..e93d539e0 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_btrs_mtl_reg.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_HW_BTRS_MTL_REG_H__ +#define __IVPU_HW_BTRS_MTL_REG_H__ + +#include + +#define VPU_HW_BTRS_MTL_INTERRUPT_TYPE 0x00000000u + +#define VPU_HW_BTRS_MTL_INTERRUPT_STAT 0x00000004u +#define VPU_HW_BTRS_MTL_INTERRUPT_STAT_FREQ_CHANGE_MASK BIT_MASK(0) +#define VPU_HW_BTRS_MTL_INTERRUPT_STAT_ATS_ERR_MASK BIT_MASK(1) +#define VPU_HW_BTRS_MTL_INTERRUPT_STAT_UFI_ERR_MASK BIT_MASK(2) + +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0 0x00000008u +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0_MIN_RATIO_MASK GENMASK(15, 0) +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0_MAX_RATIO_MASK GENMASK(31, 16) + +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1 0x0000000cu +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1_TARGET_RATIO_MASK GENMASK(15, 0) +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1_EPP_MASK GENMASK(31, 16) + +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2 0x00000010u +#define VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2_CONFIG_MASK GENMASK(15, 0) + +#define VPU_HW_BTRS_MTL_WP_REQ_CMD 0x00000014u +#define VPU_HW_BTRS_MTL_WP_REQ_CMD_SEND_MASK BIT_MASK(0) + +#define VPU_HW_BTRS_MTL_WP_DOWNLOAD 0x00000018u +#define VPU_HW_BTRS_MTL_WP_DOWNLOAD_TARGET_RATIO_MASK GENMASK(15, 0) + +#define VPU_HW_BTRS_MTL_CURRENT_PLL 0x0000001cu +#define VPU_HW_BTRS_MTL_CURRENT_PLL_RATIO_MASK GENMASK(15, 0) + +#define VPU_HW_BTRS_MTL_PLL_ENABLE 0x00000020u + +#define VPU_HW_BTRS_MTL_FMIN_FUSE 0x00000024u +#define VPU_HW_BTRS_MTL_FMIN_FUSE_MIN_RATIO_MASK GENMASK(7, 0) +#define VPU_HW_BTRS_MTL_FMIN_FUSE_PN_RATIO_MASK GENMASK(15, 8) + +#define VPU_HW_BTRS_MTL_FMAX_FUSE 0x00000028u +#define VPU_HW_BTRS_MTL_FMAX_FUSE_MAX_RATIO_MASK GENMASK(7, 0) + +#define VPU_HW_BTRS_MTL_TILE_FUSE 0x0000002cu +#define VPU_HW_BTRS_MTL_TILE_FUSE_VALID_MASK BIT_MASK(0) +#define VPU_HW_BTRS_MTL_TILE_FUSE_SKU_MASK GENMASK(3, 2) + +#define VPU_HW_BTRS_MTL_LOCAL_INT_MASK 0x00000030u +#define VPU_HW_BTRS_MTL_GLOBAL_INT_MASK 0x00000034u + +#define VPU_HW_BTRS_MTL_PLL_STATUS 0x00000040u +#define VPU_HW_BTRS_MTL_PLL_STATUS_LOCK_MASK BIT_MASK(1) + +#define VPU_HW_BTRS_MTL_VPU_STATUS 0x00000044u +#define VPU_HW_BTRS_MTL_VPU_STATUS_READY_MASK BIT_MASK(0) +#define VPU_HW_BTRS_MTL_VPU_STATUS_IDLE_MASK BIT_MASK(1) + +#define VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL 0x00000060u +#define VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL_INPROGRESS_MASK BIT_MASK(0) +#define VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL_I3_MASK BIT_MASK(2) + +#define VPU_HW_BTRS_MTL_VPU_IP_RESET 0x00000050u +#define VPU_HW_BTRS_MTL_VPU_IP_RESET_TRIGGER_MASK BIT_MASK(0) + +#define VPU_HW_BTRS_MTL_VPU_TELEMETRY_OFFSET 0x00000080u +#define VPU_HW_BTRS_MTL_VPU_TELEMETRY_SIZE 0x00000084u +#define VPU_HW_BTRS_MTL_VPU_TELEMETRY_ENABLE 0x00000088u + +#define VPU_HW_BTRS_MTL_ATS_ERR_LOG_0 0x000000a0u +#define VPU_HW_BTRS_MTL_ATS_ERR_LOG_1 0x000000a4u +#define VPU_HW_BTRS_MTL_ATS_ERR_CLEAR 0x000000a8u + +#define VPU_HW_BTRS_MTL_UFI_ERR_LOG 0x000000b0u +#define VPU_HW_BTRS_MTL_UFI_ERR_LOG_CQ_ID_MASK GENMASK(11, 0) +#define VPU_HW_BTRS_MTL_UFI_ERR_LOG_AXI_ID_MASK GENMASK(19, 12) +#define VPU_HW_BTRS_MTL_UFI_ERR_LOG_OPCODE_MASK GENMASK(24, 20) + +#define VPU_HW_BTRS_MTL_UFI_ERR_CLEAR 0x000000b4u + +#endif /* __IVPU_HW_BTRS_MTL_REG_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_ip.c b/drivers/accel/ivpu/ivpu_hw_ip.c new file mode 100644 index 000000000..81f0b1f8f --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_ip.c @@ -0,0 +1,1226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_hw_37xx_reg.h" +#include "ivpu_hw_40xx_reg.h" +#include "ivpu_hw_btrs.h" +#include "ivpu_hw_ip.h" +#include "ivpu_hw_reg_io.h" +#include "ivpu_mmu.h" +#include "ivpu_pm.h" + +#define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC) + +#define TIM_SAFE_ENABLE 0xf1d0dead +#define TIM_WATCHDOG_RESET_VALUE 0xffffffff + +#define ICB_0_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT))) + +#define ICB_1_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \ + (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT))) + +#define ICB_0_1_IRQ_MASK_37XX ((((u64)ICB_1_IRQ_MASK_37XX) << 32) | ICB_0_IRQ_MASK_37XX) + +#define ICB_0_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT))) + +#define ICB_1_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \ + (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT))) + +#define ICB_0_1_IRQ_MASK_40XX ((((u64)ICB_1_IRQ_MASK_40XX) << 32) | ICB_0_IRQ_MASK_40XX) + +#define ITF_FIREWALL_VIOLATION_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \ + (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \ + (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \ + (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \ + (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \ + (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \ + (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX))) + +#define ITF_FIREWALL_VIOLATION_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \ + (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \ + (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \ + (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \ + (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \ + (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \ + (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX))) + +static int wait_for_ip_bar(struct ivpu_device *vdev) +{ + return REGV_POLL_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, AON, 0, 100); +} + +static void host_ss_rst_clr(struct ivpu_device *vdev) +{ + u32 val = 0; + + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, TOP_NOC, val); + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, DSS_MAS, val); + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, MSS_MAS, val); + + REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_CLR, val); +} + +static int host_ss_noc_qreqn_check_37xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN); + + if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int host_ss_noc_qreqn_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); + + if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int host_ss_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return host_ss_noc_qreqn_check_37xx(vdev, exp_val); + else + return host_ss_noc_qreqn_check_40xx(vdev, exp_val); +} + +static int host_ss_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QACCEPTN); + + if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int host_ss_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN); + + if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int host_ss_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return host_ss_noc_qacceptn_check_37xx(vdev, exp_val); + else + return host_ss_noc_qacceptn_check_40xx(vdev, exp_val); +} + +static int host_ss_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QDENY); + + if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int host_ss_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY); + + if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int host_ss_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return host_ss_noc_qdeny_check_37xx(vdev, exp_val); + else + return host_ss_noc_qdeny_check_40xx(vdev, exp_val); +} + +static int top_noc_qrenqn_check_37xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN); + + if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) || + !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val)) + return -EIO; + + return 0; +} + +static int top_noc_qrenqn_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); + + if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) || + !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val)) + return -EIO; + + return 0; +} + +static int top_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return top_noc_qrenqn_check_37xx(vdev, exp_val); + else + return top_noc_qrenqn_check_40xx(vdev, exp_val); +} + +int ivpu_hw_ip_host_ss_configure(struct ivpu_device *vdev) +{ + int ret; + + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + ret = wait_for_ip_bar(vdev); + if (ret) { + ivpu_err(vdev, "Timed out waiting for NPU IP bar\n"); + return ret; + } + host_ss_rst_clr(vdev); + } + + ret = host_ss_noc_qreqn_check(vdev, 0x0); + if (ret) { + ivpu_err(vdev, "Failed qreqn check: %d\n", ret); + return ret; + } + + ret = host_ss_noc_qacceptn_check(vdev, 0x0); + if (ret) { + ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); + return ret; + } + + ret = host_ss_noc_qdeny_check(vdev, 0x0); + if (ret) + ivpu_err(vdev, "Failed qdeny check %d\n", ret); + + return ret; +} + +static void idle_gen_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN); + + if (enable) + val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val); + else + val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val); + + REGV_WR32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, val); +} + +static void idle_gen_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN); + + if (enable) + val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); + else + val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); + + REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val); +} + +void ivpu_hw_ip_idle_gen_enable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + idle_gen_drive_37xx(vdev, true); + else + idle_gen_drive_40xx(vdev, true); +} + +void ivpu_hw_ip_idle_gen_disable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + idle_gen_drive_37xx(vdev, false); + else + idle_gen_drive_40xx(vdev, false); +} + +static void +pwr_island_delay_set_50xx(struct ivpu_device *vdev, u32 post, u32 post1, u32 post2, u32 status) +{ + u32 val; + + val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY); + val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST_DLY, post, val); + val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST1_DLY, post1, val); + val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST2_DLY, post2, val); + REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, val); + + val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY); + val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, STATUS_DLY, status, val); + REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, val); +} + +static void pwr_island_trickle_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0); + + if (enable) + val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val); + else + val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val); + + REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val); +} + +static void pwr_island_trickle_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0); + + if (enable) + val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); + else + val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); + + REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val); +} + +static void pwr_island_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0); + + if (enable) + val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val); + else + val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val); + + REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val); +} + +static void pwr_island_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0); + + if (enable) + val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); + else + val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); + + REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val); +} + +static void pwr_island_enable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + pwr_island_trickle_drive_37xx(vdev, true); + ndelay(500); + pwr_island_drive_37xx(vdev, true); + } else { + pwr_island_trickle_drive_40xx(vdev, true); + ndelay(500); + pwr_island_drive_40xx(vdev, true); + } +} + +static int wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val) +{ + if (IVPU_WA(punit_disabled)) + return 0; + + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return REGV_POLL_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0, MSS_CPU, exp_val, + PWR_ISLAND_STATUS_TIMEOUT_US); + else + return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, exp_val, + PWR_ISLAND_STATUS_TIMEOUT_US); +} + +static void pwr_island_isolation_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0); + + if (enable) + val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val); + else + val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val); + + REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, val); +} + +static void pwr_island_isolation_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0); + + if (enable) + val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); + else + val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); + + REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val); +} + +static void pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + pwr_island_isolation_drive_37xx(vdev, enable); + else + pwr_island_isolation_drive_40xx(vdev, enable); +} + +static void pwr_island_isolation_disable(struct ivpu_device *vdev) +{ + pwr_island_isolation_drive(vdev, false); +} + +static void host_ss_clk_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_CLK_SET); + + if (enable) { + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val); + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val); + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val); + } else { + val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val); + val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val); + val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val); + } + + REGV_WR32(VPU_37XX_HOST_SS_CPR_CLK_SET, val); +} + +static void host_ss_clk_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN); + + if (enable) { + val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); + val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); + val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); + } else { + val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); + val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); + val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); + } + + REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val); +} + +static void host_ss_clk_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + host_ss_clk_drive_37xx(vdev, enable); + else + host_ss_clk_drive_40xx(vdev, enable); +} + +static void host_ss_clk_enable(struct ivpu_device *vdev) +{ + host_ss_clk_drive(vdev, true); +} + +static void host_ss_rst_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_RST_SET); + + if (enable) { + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val); + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val); + val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val); + } else { + val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val); + val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val); + val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val); + } + + REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_SET, val); +} + +static void host_ss_rst_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN); + + if (enable) { + val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); + val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); + val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); + } else { + val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); + val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); + val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); + } + + REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val); +} + +static void host_ss_rst_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + host_ss_rst_drive_37xx(vdev, enable); + else + host_ss_rst_drive_40xx(vdev, enable); +} + +static void host_ss_rst_enable(struct ivpu_device *vdev) +{ + host_ss_rst_drive(vdev, true); +} + +static void host_ss_noc_qreqn_top_socmmio_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN); + + if (enable) + val = REG_SET_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); + else + val = REG_CLR_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); + REGV_WR32(VPU_37XX_HOST_SS_NOC_QREQN, val); +} + +static void host_ss_noc_qreqn_top_socmmio_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); + + if (enable) + val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); + else + val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); + REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val); +} + +static void host_ss_noc_qreqn_top_socmmio_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + host_ss_noc_qreqn_top_socmmio_drive_37xx(vdev, enable); + else + host_ss_noc_qreqn_top_socmmio_drive_40xx(vdev, enable); +} + +static int host_ss_axi_drive(struct ivpu_device *vdev, bool enable) +{ + int ret; + + host_ss_noc_qreqn_top_socmmio_drive(vdev, enable); + + ret = host_ss_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); + if (ret) { + ivpu_err(vdev, "Failed HOST SS NOC QACCEPTN check: %d\n", ret); + return ret; + } + + ret = host_ss_noc_qdeny_check(vdev, 0x0); + if (ret) + ivpu_err(vdev, "Failed HOST SS NOC QDENY check: %d\n", ret); + + return ret; +} + +static void top_noc_qreqn_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); + + if (enable) { + val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); + val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); + } else { + val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); + val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); + } + + REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val); +} + +static void top_noc_qreqn_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN); + + if (enable) { + val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val); + val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); + } else { + val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val); + val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); + } + + REGV_WR32(VPU_37XX_TOP_NOC_QREQN, val); +} + +static void top_noc_qreqn_drive(struct ivpu_device *vdev, bool enable) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + top_noc_qreqn_drive_37xx(vdev, enable); + else + top_noc_qreqn_drive_40xx(vdev, enable); +} + +int ivpu_hw_ip_host_ss_axi_enable(struct ivpu_device *vdev) +{ + return host_ss_axi_drive(vdev, true); +} + +static int top_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QACCEPTN); + + if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) || + !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val)) + return -EIO; + + return 0; +} + +static int top_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN); + + if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) || + !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val)) + return -EIO; + + return 0; +} + +static int top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return top_noc_qacceptn_check_37xx(vdev, exp_val); + else + return top_noc_qacceptn_check_40xx(vdev, exp_val); +} + +static int top_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QDENY); + + if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) || + !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val)) + return -EIO; + + return 0; +} + +static int top_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY); + + if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) || + !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val)) + return -EIO; + + return 0; +} + +static int top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return top_noc_qdeny_check_37xx(vdev, exp_val); + else + return top_noc_qdeny_check_40xx(vdev, exp_val); +} + +static int top_noc_drive(struct ivpu_device *vdev, bool enable) +{ + int ret; + + top_noc_qreqn_drive(vdev, enable); + + ret = top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); + if (ret) { + ivpu_err(vdev, "Failed TOP NOC QACCEPTN check: %d\n", ret); + return ret; + } + + ret = top_noc_qdeny_check(vdev, 0x0); + if (ret) + ivpu_err(vdev, "Failed TOP NOC QDENY check: %d\n", ret); + + return ret; +} + +int ivpu_hw_ip_top_noc_enable(struct ivpu_device *vdev) +{ + return top_noc_drive(vdev, true); +} + +static void dpu_active_drive_37xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE); + + if (enable) + val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val); + else + val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val); + + REGV_WR32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, val); +} + +static void pwr_island_delay_set(struct ivpu_device *vdev) +{ + bool high = vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_HIGH; + u32 post, post1, post2, status; + + if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX) + return; + + switch (ivpu_device_id(vdev)) { + case PCI_DEVICE_ID_WCL: + case PCI_DEVICE_ID_PTL_P: + post = high ? 18 : 0; + post1 = 0; + post2 = 0; + status = high ? 46 : 3; + break; + + case PCI_DEVICE_ID_NVL: + post = high ? 198 : 17; + post1 = 0; + post2 = high ? 198 : 17; + status = 0; + break; + + default: + dump_stack(); + ivpu_err(vdev, "Unknown device ID\n"); + return; + } + + pwr_island_delay_set_50xx(vdev, post, post1, post2, status); +} + +int ivpu_hw_ip_pwr_domain_enable(struct ivpu_device *vdev) +{ + int ret; + + pwr_island_delay_set(vdev); + pwr_island_enable(vdev); + + ret = wait_for_pwr_island_status(vdev, 0x1); + if (ret) { + ivpu_err(vdev, "Timed out waiting for power island status\n"); + return ret; + } + + ret = top_noc_qreqn_check(vdev, 0x0); + if (ret) { + ivpu_err(vdev, "Failed TOP NOC QREQN check %d\n", ret); + return ret; + } + + host_ss_clk_enable(vdev); + pwr_island_isolation_disable(vdev); + host_ss_rst_enable(vdev); + + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + dpu_active_drive_37xx(vdev, true); + + return ret; +} + +u64 ivpu_hw_ip_read_perf_timer_counter(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return REGV_RD64(VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT); + else + return REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT); +} + +static void ivpu_hw_ip_snoop_disable_37xx(struct ivpu_device *vdev) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES); + + val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, NOSNOOP_OVERRIDE_EN, val); + val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AW_NOSNOOP_OVERRIDE, val); + + if (ivpu_is_force_snoop_enabled(vdev)) + val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val); + else + val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val); + + REGV_WR32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, val); +} + +static void ivpu_hw_ip_snoop_disable_40xx(struct ivpu_device *vdev) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES); + + val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val); + val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val); + + if (ivpu_is_force_snoop_enabled(vdev)) + val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val); + else + val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val); + + REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val); +} + +void ivpu_hw_ip_snoop_disable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return ivpu_hw_ip_snoop_disable_37xx(vdev); + else + return ivpu_hw_ip_snoop_disable_40xx(vdev); +} + +static void ivpu_hw_ip_tbu_mmu_enable_37xx(struct ivpu_device *vdev) +{ + u32 val = REGV_RD32(VPU_37XX_HOST_IF_TBU_MMUSSIDV); + + val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); + val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); + val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); + val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); + + REGV_WR32(VPU_37XX_HOST_IF_TBU_MMUSSIDV, val); +} + +static void ivpu_hw_ip_tbu_mmu_enable_40xx(struct ivpu_device *vdev) +{ + u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV); + + val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); + val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); + val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val); + val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val); + val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); + val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); + + REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val); +} + +void ivpu_hw_ip_tbu_mmu_enable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return ivpu_hw_ip_tbu_mmu_enable_37xx(vdev); + else + return ivpu_hw_ip_tbu_mmu_enable_40xx(vdev); +} + +static inline u64 get_entry_point_addr(struct ivpu_device *vdev) +{ + if (ivpu_fw_is_warm_boot(vdev)) + return vdev->fw->warm_boot_entry_point; + else + return vdev->fw->cold_boot_entry_point; +} + +static int soc_cpu_boot_37xx(struct ivpu_device *vdev) +{ + u32 val; + + val = REGV_RD32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC); + val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTRUN0, val); + + val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTVEC, val); + REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val); + + val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val); + REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val); + + val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val); + REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val); + + val = get_entry_point_addr(vdev) >> 9; + REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val); + + val = REG_SET_FLD(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, DONE, val); + REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val); + + return 0; +} + +static int cpu_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN); + + if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static int cpu_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val) +{ + u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY); + + if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val)) + return -EIO; + + return 0; +} + +static void cpu_noc_top_mmio_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN); + + if (enable) + val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); + else + val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); + REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val); +} + +static int soc_cpu_drive_40xx(struct ivpu_device *vdev, bool enable) +{ + int ret; + + cpu_noc_top_mmio_drive_40xx(vdev, enable); + + ret = cpu_noc_qacceptn_check_40xx(vdev, enable ? 0x1 : 0x0); + if (ret) { + ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); + return ret; + } + + ret = cpu_noc_qdeny_check_40xx(vdev, 0x0); + if (ret) + ivpu_err(vdev, "Failed qdeny check: %d\n", ret); + + return ret; +} + +static void soc_cpu_set_entry_point_40xx(struct ivpu_device *vdev, u64 entry_point) +{ + u64 val64; + u32 val; + + val64 = entry_point; + val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1; + REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64); + + val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO); + val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val); + REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val); +} + +static int soc_cpu_boot_40xx(struct ivpu_device *vdev) +{ + int ret; + + ret = soc_cpu_drive_40xx(vdev, true); + if (ret) { + ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret); + return ret; + } + + soc_cpu_set_entry_point_40xx(vdev, get_entry_point_addr(vdev)); + + return 0; +} + +static int soc_cpu_boot_60xx(struct ivpu_device *vdev) +{ + soc_cpu_set_entry_point_40xx(vdev, vdev->fw->cold_boot_entry_point); + + return 0; +} + +int ivpu_hw_ip_soc_cpu_boot(struct ivpu_device *vdev) +{ + int ret; + + switch (ivpu_hw_ip_gen(vdev)) { + case IVPU_HW_IP_37XX: + ret = soc_cpu_boot_37xx(vdev); + break; + + case IVPU_HW_IP_40XX: + case IVPU_HW_IP_50XX: + ret = soc_cpu_boot_40xx(vdev); + break; + + default: + ret = soc_cpu_boot_60xx(vdev); + } + + if (ret) + return ret; + + ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n", + ivpu_fw_is_warm_boot(vdev) ? "warm boot" : "cold boot"); + + return 0; +} + +static void wdt_disable_37xx(struct ivpu_device *vdev) +{ + u32 val; + + /* Enable writing and set non-zero WDT value */ + REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); + REGV_WR32(VPU_37XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE); + + /* Enable writing and disable watchdog timer */ + REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); + REGV_WR32(VPU_37XX_CPU_SS_TIM_WDOG_EN, 0); + + /* Now clear the timeout interrupt */ + val = REGV_RD32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG); + val = REG_CLR_FLD(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val); + REGV_WR32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, val); +} + +static void wdt_disable_40xx(struct ivpu_device *vdev) +{ + u32 val; + + REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); + REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE); + + REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); + REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0); + + val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG); + val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val); + REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val); +} + +void ivpu_hw_ip_wdt_disable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return wdt_disable_37xx(vdev); + else + return wdt_disable_40xx(vdev); +} + +static u32 ipc_rx_count_get_37xx(struct ivpu_device *vdev) +{ + u32 count = readl(vdev->regv + VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT); + + return REG_GET_FLD(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count); +} + +static u32 ipc_rx_count_get_40xx(struct ivpu_device *vdev) +{ + u32 count = readl(vdev->regv + VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT); + + return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count); +} + +u32 ivpu_hw_ip_ipc_rx_count_get(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return ipc_rx_count_get_37xx(vdev); + else + return ipc_rx_count_get_40xx(vdev); +} + +void ivpu_hw_ip_irq_enable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_37XX); + REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_37XX); + } else { + REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_40XX); + REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_40XX); + } +} + +void ivpu_hw_ip_irq_disable(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { + REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, 0x0ull); + REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, 0x0); + } else { + REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull); + REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul); + } +} + +static void diagnose_failure_37xx(struct ivpu_device *vdev) +{ + u32 reg = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX; + + if (ipc_rx_count_get_37xx(vdev)) + ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ"); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg)) + ivpu_err(vdev, "WDT MSS timeout detected\n"); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg)) + ivpu_err(vdev, "WDT NCE timeout detected\n"); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg)) + ivpu_err(vdev, "NOC Firewall irq detected\n"); +} + +static void diagnose_failure_40xx(struct ivpu_device *vdev) +{ + u32 reg = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX; + + if (ipc_rx_count_get_40xx(vdev)) + ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ"); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg)) + ivpu_err(vdev, "WDT MSS timeout detected\n"); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg)) + ivpu_err(vdev, "WDT NCE timeout detected\n"); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg)) + ivpu_err(vdev, "NOC Firewall irq detected\n"); +} + +void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + diagnose_failure_37xx(vdev); + else + diagnose_failure_40xx(vdev); +} + +void ivpu_hw_ip_irq_clear(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + REGV_WR64(VPU_37XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_37XX); + else + REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_40XX); +} + +static void irq_wdt_nce_handler(struct ivpu_device *vdev) +{ + ivpu_pm_trigger_recovery(vdev, "WDT NCE IRQ"); +} + +static void irq_wdt_mss_handler(struct ivpu_device *vdev) +{ + ivpu_hw_ip_wdt_disable(vdev); + ivpu_pm_trigger_recovery(vdev, "WDT MSS IRQ"); +} + +static void irq_noc_firewall_handler(struct ivpu_device *vdev) +{ + atomic_inc(&vdev->hw->firewall_irq_counter); + + ivpu_dbg(vdev, IRQ, "NOC Firewall interrupt detected, counter %d\n", + atomic_read(&vdev->hw->firewall_irq_counter)); +} + +/* Handler for IRQs from NPU core */ +bool ivpu_hw_ip_irq_handler_37xx(struct ivpu_device *vdev, int irq) +{ + u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX; + + if (!status) + return false; + + REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status)) + ivpu_mmu_irq_evtq_handler(vdev); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status)) + ivpu_ipc_irq_handler(vdev); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status)) + ivpu_dbg(vdev, IRQ, "MMU sync complete\n"); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status)) + ivpu_mmu_irq_gerr_handler(vdev); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status)) + irq_wdt_mss_handler(vdev); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status)) + irq_wdt_nce_handler(vdev); + + if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status)) + irq_noc_firewall_handler(vdev); + + return true; +} + +/* Handler for IRQs from NPU core */ +bool ivpu_hw_ip_irq_handler_40xx(struct ivpu_device *vdev, int irq) +{ + u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX; + + if (!status) + return false; + + REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status)) + ivpu_mmu_irq_evtq_handler(vdev); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status)) + ivpu_ipc_irq_handler(vdev); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status)) + ivpu_dbg(vdev, IRQ, "MMU sync complete\n"); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status)) + ivpu_mmu_irq_gerr_handler(vdev); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status)) + irq_wdt_mss_handler(vdev); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status)) + irq_wdt_nce_handler(vdev); + + if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status)) + irq_noc_firewall_handler(vdev); + + return true; +} + +static void db_set_37xx(struct ivpu_device *vdev, u32 db_id) +{ + u32 reg_stride = VPU_37XX_CPU_SS_DOORBELL_1 - VPU_37XX_CPU_SS_DOORBELL_0; + u32 val = REG_FLD(VPU_37XX_CPU_SS_DOORBELL_0, SET); + + REGV_WR32I(VPU_37XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val); +} + +static void db_set_40xx(struct ivpu_device *vdev, u32 db_id) +{ + u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0; + u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET); + + REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val); +} + +void ivpu_hw_ip_db_set(struct ivpu_device *vdev, u32 db_id) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + db_set_37xx(vdev, db_id); + else + db_set_40xx(vdev, db_id); +} + +u32 ivpu_hw_ip_ipc_rx_addr_get(struct ivpu_device *vdev) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + return REGV_RD32(VPU_37XX_HOST_SS_TIM_IPC_FIFO_ATM); + else + return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM); +} + +void ivpu_hw_ip_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr) +{ + if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) + REGV_WR32(VPU_37XX_CPU_SS_TIM_IPC_FIFO, vpu_addr); + else + REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr); +} diff --git a/drivers/accel/ivpu/ivpu_hw_ip.h b/drivers/accel/ivpu/ivpu_hw_ip.h new file mode 100644 index 000000000..dbbcdd10a --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_ip.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __IVPU_HW_IP_H__ +#define __IVPU_HW_IP_H__ + +#include "ivpu_drv.h" + +int ivpu_hw_ip_host_ss_configure(struct ivpu_device *vdev); +void ivpu_hw_ip_idle_gen_enable(struct ivpu_device *vdev); +void ivpu_hw_ip_idle_gen_disable(struct ivpu_device *vdev); +int ivpu_hw_ip_pwr_domain_enable(struct ivpu_device *vdev); +int ivpu_hw_ip_host_ss_axi_enable(struct ivpu_device *vdev); +int ivpu_hw_ip_top_noc_enable(struct ivpu_device *vdev); +u64 ivpu_hw_ip_read_perf_timer_counter(struct ivpu_device *vdev); +void ivpu_hw_ip_snoop_disable(struct ivpu_device *vdev); +void ivpu_hw_ip_tbu_mmu_enable(struct ivpu_device *vdev); +int ivpu_hw_ip_soc_cpu_boot(struct ivpu_device *vdev); +void ivpu_hw_ip_wdt_disable(struct ivpu_device *vdev); +void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev); +u32 ivpu_hw_ip_ipc_rx_count_get(struct ivpu_device *vdev); +void ivpu_hw_ip_irq_clear(struct ivpu_device *vdev); +bool ivpu_hw_ip_irq_handler_37xx(struct ivpu_device *vdev, int irq); +bool ivpu_hw_ip_irq_handler_40xx(struct ivpu_device *vdev, int irq); +void ivpu_hw_ip_db_set(struct ivpu_device *vdev, u32 db_id); +u32 ivpu_hw_ip_ipc_rx_addr_get(struct ivpu_device *vdev); +void ivpu_hw_ip_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr); +void ivpu_hw_ip_irq_enable(struct ivpu_device *vdev); +void ivpu_hw_ip_irq_disable(struct ivpu_device *vdev); +void ivpu_hw_ip_fabric_req_override_enable_50xx(struct ivpu_device *vdev); +void ivpu_hw_ip_fabric_req_override_disable_50xx(struct ivpu_device *vdev); + +#endif /* __IVPU_HW_IP_H__ */ diff --git a/drivers/accel/ivpu/ivpu_hw_reg_io.h b/drivers/accel/ivpu/ivpu_hw_reg_io.h new file mode 100644 index 000000000..66259b0ea --- /dev/null +++ b/drivers/accel/ivpu/ivpu_hw_reg_io.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_HW_REG_IO_H__ +#define __IVPU_HW_REG_IO_H__ + +#include +#include +#include +#include + +#include "ivpu_drv.h" + +#define REG_POLL_SLEEP_US 50 +#define REG_IO_ERROR 0xffffffff + +#define REGB_RD32(reg) ivpu_hw_reg_rd32(vdev, vdev->regb, (reg), #reg, __func__) +#define REGB_RD64(reg) ivpu_hw_reg_rd64(vdev, vdev->regb, (reg), #reg, __func__) +#define REGB_WR32(reg, val) ivpu_hw_reg_wr32(vdev, vdev->regb, (reg), (val), #reg, __func__) +#define REGB_WR64(reg, val) ivpu_hw_reg_wr64(vdev, vdev->regb, (reg), (val), #reg, __func__) + +#define REGV_RD32(reg) ivpu_hw_reg_rd32(vdev, vdev->regv, (reg), #reg, __func__) +#define REGV_RD64(reg) ivpu_hw_reg_rd64(vdev, vdev->regv, (reg), #reg, __func__) +#define REGV_WR32(reg, val) ivpu_hw_reg_wr32(vdev, vdev->regv, (reg), (val), #reg, __func__) +#define REGV_WR64(reg, val) ivpu_hw_reg_wr64(vdev, vdev->regv, (reg), (val), #reg, __func__) + +#define REGV_WR32I(reg, stride, index, val) \ + ivpu_hw_reg_wr32_index(vdev, vdev->regv, (reg), (stride), (index), (val), #reg, __func__) + +#define REG_FLD(REG, FLD) \ + (REG##_##FLD##_MASK) +#define REG_FLD_NUM(REG, FLD, num) \ + FIELD_PREP(REG##_##FLD##_MASK, num) +#define REG_GET_FLD(REG, FLD, val) \ + FIELD_GET(REG##_##FLD##_MASK, val) +#define REG_CLR_FLD(REG, FLD, val) \ + ((val) & ~(REG##_##FLD##_MASK)) +#define REG_SET_FLD(REG, FLD, val) \ + ((val) | (REG##_##FLD##_MASK)) +#define REG_SET_FLD_NUM(REG, FLD, num, val) \ + (((val) & ~(REG##_##FLD##_MASK)) | FIELD_PREP(REG##_##FLD##_MASK, num)) +#define REG_TEST_FLD(REG, FLD, val) \ + ((REG##_##FLD##_MASK) == ((val) & (REG##_##FLD##_MASK))) +#define REG_TEST_FLD_NUM(REG, FLD, num, val) \ + ((num) == FIELD_GET(REG##_##FLD##_MASK, val)) + +#define REGB_POLL_FLD(reg, fld, exp_fld_val, timeout_us) \ + ivpu_hw_reg_poll_fld(vdev, vdev->regb, reg, reg##_##fld##_MASK, \ + FIELD_PREP(reg##_##fld##_MASK, exp_fld_val), timeout_us, \ + __func__, #reg, #fld) + +#define REGV_POLL_FLD(reg, fld, exp_fld_val, timeout_us) \ + ivpu_hw_reg_poll_fld(vdev, vdev->regv, reg, reg##_##fld##_MASK, \ + FIELD_PREP(reg##_##fld##_MASK, exp_fld_val), timeout_us, \ + __func__, #reg, #fld) + +extern struct fault_attr ivpu_hw_failure; + +static inline int __must_check +ivpu_hw_reg_poll_fld(struct ivpu_device *vdev, void __iomem *base, + u32 reg_offset, u32 reg_mask, u32 exp_masked_val, u32 timeout_us, + const char *func_name, const char *reg_name, const char *fld_name) +{ + u32 reg_val; + int ret; + + ivpu_dbg(vdev, REG, "%s : %s (0x%08x) POLL %s started (exp_val 0x%x)\n", + func_name, reg_name, reg_offset, fld_name, exp_masked_val); + + ret = read_poll_timeout(readl, reg_val, (reg_val & reg_mask) == exp_masked_val, + REG_POLL_SLEEP_US, timeout_us, false, base + reg_offset); + +#ifdef CONFIG_FAULT_INJECTION + if (should_fail(&ivpu_hw_failure, 1)) + ret = -ETIMEDOUT; +#endif + + ivpu_dbg(vdev, REG, "%s : %s (0x%08x) POLL %s %s (reg_val 0x%08x)\n", + func_name, reg_name, reg_offset, fld_name, ret ? "ETIMEDOUT" : "OK", reg_val); + + return ret; +} + +static inline u32 +ivpu_hw_reg_rd32(struct ivpu_device *vdev, void __iomem *base, u32 reg, + const char *name, const char *func) +{ + u32 val = readl(base + reg); + + ivpu_dbg(vdev, REG, "%s : %s (0x%08x) RD: 0x%08x\n", func, name, reg, val); + return val; +} + +static inline u64 +ivpu_hw_reg_rd64(struct ivpu_device *vdev, void __iomem *base, u32 reg, + const char *name, const char *func) +{ + u64 val = readq(base + reg); + + ivpu_dbg(vdev, REG, "%s : %s (0x%08x) RD: 0x%016llx\n", func, name, reg, val); + return val; +} + +static inline void +ivpu_hw_reg_wr32(struct ivpu_device *vdev, void __iomem *base, u32 reg, u32 val, + const char *name, const char *func) +{ + ivpu_dbg(vdev, REG, "%s : %s (0x%08x) WR: 0x%08x\n", func, name, reg, val); + writel(val, base + reg); +} + +static inline void +ivpu_hw_reg_wr64(struct ivpu_device *vdev, void __iomem *base, u32 reg, u64 val, + const char *name, const char *func) +{ + ivpu_dbg(vdev, REG, "%s : %s (0x%08x) WR: 0x%016llx\n", func, name, reg, val); + writeq(val, base + reg); +} + +static inline void +ivpu_hw_reg_wr32_index(struct ivpu_device *vdev, void __iomem *base, u32 reg, + u32 stride, u32 index, u32 val, const char *name, + const char *func) +{ + reg += index * stride; + + ivpu_dbg(vdev, REG, "%s WR: %s_%d (0x%08x) <= 0x%08x\n", func, name, index, reg, val); + writel(val, base + reg); +} + +#endif /* __IVPU_HW_REG_IO_H__ */ diff --git a/drivers/accel/ivpu/ivpu_ipc.c b/drivers/accel/ivpu/ivpu_ipc.c new file mode 100644 index 000000000..978bc3d87 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_ipc.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#include +#include +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_hw_reg_io.h" +#include "ivpu_ipc.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_pm.h" +#include "ivpu_trace.h" + +#define IPC_MAX_RX_MSG 128 + +struct ivpu_ipc_tx_buf { + struct ivpu_ipc_hdr ipc; + struct vpu_jsm_msg jsm; +}; + +static void ivpu_ipc_msg_dump(struct ivpu_device *vdev, char *c, + struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr) +{ + ivpu_dbg(vdev, IPC, + "%s: vpu:0x%x (data_addr:0x%08x, data_size:0x%x, channel:0x%x, src_node:0x%x, dst_node:0x%x, status:0x%x)", + c, vpu_addr, ipc_hdr->data_addr, ipc_hdr->data_size, ipc_hdr->channel, + ipc_hdr->src_node, ipc_hdr->dst_node, ipc_hdr->status); +} + +static void ivpu_jsm_msg_dump(struct ivpu_device *vdev, char *c, + struct vpu_jsm_msg *jsm_msg, u32 vpu_addr) +{ + u32 *payload = (u32 *)&jsm_msg->payload; + + ivpu_dbg(vdev, JSM, + "%s: vpu:0x%08x (type:%s, status:0x%x, id: 0x%x, result: 0x%x, payload:0x%x 0x%x 0x%x 0x%x 0x%x)\n", + c, vpu_addr, ivpu_jsm_msg_type_to_str(jsm_msg->type), + jsm_msg->status, jsm_msg->request_id, jsm_msg->result, + payload[0], payload[1], payload[2], payload[3], payload[4]); +} + +static void +ivpu_ipc_rx_mark_free(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, + struct vpu_jsm_msg *jsm_msg) +{ + ipc_hdr->status = IVPU_IPC_HDR_FREE; + if (jsm_msg) + jsm_msg->status = VPU_JSM_MSG_FREE; + wmb(); /* Flush WC buffers for message statuses */ +} + +static void ivpu_ipc_mem_fini(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + + ivpu_bo_free(ipc->mem_rx); + ivpu_bo_free(ipc->mem_tx); +} + +static int +ivpu_ipc_tx_prepare(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + struct vpu_jsm_msg *req) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + struct ivpu_ipc_tx_buf *tx_buf; + u32 tx_buf_vpu_addr; + u32 jsm_vpu_addr; + + tx_buf_vpu_addr = gen_pool_alloc(ipc->mm_tx, sizeof(*tx_buf)); + if (!tx_buf_vpu_addr) { + ivpu_err_ratelimited(vdev, "Failed to reserve IPC buffer, size %ld\n", + sizeof(*tx_buf)); + return -ENOMEM; + } + + tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr); + if (drm_WARN_ON(&vdev->drm, !tx_buf)) { + gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf)); + return -EIO; + } + + jsm_vpu_addr = tx_buf_vpu_addr + offsetof(struct ivpu_ipc_tx_buf, jsm); + + if (tx_buf->ipc.status != IVPU_IPC_HDR_FREE) + ivpu_warn_ratelimited(vdev, "IPC message vpu:0x%x not released by firmware\n", + tx_buf_vpu_addr); + + if (tx_buf->jsm.status != VPU_JSM_MSG_FREE) + ivpu_warn_ratelimited(vdev, "JSM message vpu:0x%x not released by firmware\n", + jsm_vpu_addr); + + memset(tx_buf, 0, sizeof(*tx_buf)); + tx_buf->ipc.data_addr = jsm_vpu_addr; + /* + * Firmware expects full JSM message size regardless of the payload size. + * Unused fields must be zeroed to allow future extensions of existing + * commands without breaking compatibility. + */ + tx_buf->ipc.data_size = sizeof(*req); + tx_buf->ipc.channel = cons->channel; + tx_buf->ipc.src_node = 0; + tx_buf->ipc.dst_node = 1; + tx_buf->ipc.status = IVPU_IPC_HDR_ALLOCATED; + tx_buf->jsm.type = req->type; + tx_buf->jsm.status = VPU_JSM_MSG_ALLOCATED; + tx_buf->jsm.payload = req->payload; + + req->request_id = atomic_inc_return(&ipc->request_id); + tx_buf->jsm.request_id = req->request_id; + cons->request_id = req->request_id; + wmb(); /* Flush WC buffers for IPC, JSM msgs */ + + cons->tx_vpu_addr = tx_buf_vpu_addr; + + ivpu_jsm_msg_dump(vdev, "TX", &tx_buf->jsm, jsm_vpu_addr); + ivpu_ipc_msg_dump(vdev, "TX", &tx_buf->ipc, tx_buf_vpu_addr); + + return 0; +} + +static void ivpu_ipc_tx_release(struct ivpu_device *vdev, u32 vpu_addr) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + + if (vpu_addr) + gen_pool_free(ipc->mm_tx, vpu_addr, sizeof(struct ivpu_ipc_tx_buf)); +} + +static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr) +{ + ivpu_hw_ipc_tx_set(vdev, vpu_addr); +} + +static void +ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + struct ivpu_ipc_rx_msg *rx_msg; + + lockdep_assert_held(&ipc->cons_lock); + + rx_msg = kzalloc_obj(*rx_msg, GFP_ATOMIC); + if (!rx_msg) { + ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); + return; + } + + atomic_inc(&ipc->rx_msg_count); + + rx_msg->ipc_hdr = ipc_hdr; + rx_msg->jsm_msg = jsm_msg; + rx_msg->callback = cons->rx_callback; + + if (rx_msg->callback) { + list_add_tail(&rx_msg->link, &ipc->cb_msg_list); + } else { + spin_lock(&cons->rx_lock); + list_add_tail(&rx_msg->link, &cons->rx_msg_list); + spin_unlock(&cons->rx_lock); + wake_up(&cons->rx_msg_wq); + } +} + +static void +ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg) +{ + list_del(&rx_msg->link); + ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg); + atomic_dec(&vdev->ipc->rx_msg_count); + kfree(rx_msg); +} + +void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + u32 channel, ivpu_ipc_rx_callback_t rx_callback) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + + INIT_LIST_HEAD(&cons->link); + cons->channel = channel; + cons->tx_vpu_addr = 0; + cons->request_id = 0; + cons->aborted = false; + cons->rx_callback = rx_callback; + spin_lock_init(&cons->rx_lock); + INIT_LIST_HEAD(&cons->rx_msg_list); + init_waitqueue_head(&cons->rx_msg_wq); + + spin_lock_irq(&ipc->cons_lock); + list_add_tail(&cons->link, &ipc->cons_list); + spin_unlock_irq(&ipc->cons_lock); +} + +void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + struct ivpu_ipc_rx_msg *rx_msg, *r; + + spin_lock_irq(&ipc->cons_lock); + list_del(&cons->link); + spin_unlock_irq(&ipc->cons_lock); + + spin_lock_irq(&cons->rx_lock); + list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link) + ivpu_ipc_rx_msg_del(vdev, rx_msg); + spin_unlock_irq(&cons->rx_lock); + + ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr); +} + +int ivpu_ipc_send(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, struct vpu_jsm_msg *req) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + int ret; + + mutex_lock(&ipc->lock); + + if (!ipc->on) { + ret = -EAGAIN; + goto unlock; + } + + ret = ivpu_ipc_tx_prepare(vdev, cons, req); + if (ret) + goto unlock; + + ivpu_ipc_tx(vdev, cons->tx_vpu_addr); + trace_jsm("[tx]", req); + +unlock: + mutex_unlock(&ipc->lock); + return ret; +} + +static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons) +{ + bool ret; + + spin_lock_irq(&cons->rx_lock); + ret = !list_empty(&cons->rx_msg_list) || cons->aborted; + spin_unlock_irq(&cons->rx_lock); + + return ret; +} + +int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + struct ivpu_ipc_hdr *ipc_buf, + struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms) +{ + struct ivpu_ipc_rx_msg *rx_msg; + int wait_ret, ret = 0; + + if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n")) + return -EINVAL; + + wait_ret = wait_event_timeout(cons->rx_msg_wq, + ivpu_ipc_rx_need_wakeup(cons), + msecs_to_jiffies(timeout_ms)); + + if (wait_ret == 0) + return -ETIMEDOUT; + + spin_lock_irq(&cons->rx_lock); + if (cons->aborted) { + spin_unlock_irq(&cons->rx_lock); + return -ECANCELED; + } + rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link); + if (!rx_msg) { + spin_unlock_irq(&cons->rx_lock); + return -EAGAIN; + } + + if (ipc_buf) + memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf)); + if (rx_msg->jsm_msg) { + u32 size = min(rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg)); + + if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) { + ivpu_err(vdev, "IPC resp result error: %d\n", rx_msg->jsm_msg->result); + ret = -EBADMSG; + } + + if (jsm_msg) + memcpy(jsm_msg, rx_msg->jsm_msg, size); + trace_jsm("[rx]", rx_msg->jsm_msg); + } + + ivpu_ipc_rx_msg_del(vdev, rx_msg); + spin_unlock_irq(&cons->rx_lock); + return ret; +} + +int +ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req, + enum vpu_ipc_msg_type expected_resp_type, + struct vpu_jsm_msg *resp, u32 channel, unsigned long timeout_ms) +{ + struct ivpu_ipc_consumer cons; + int ret; + + drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev) && + pm_runtime_enabled(vdev->drm.dev)); + + ivpu_ipc_consumer_add(vdev, &cons, channel, NULL); + + ret = ivpu_ipc_send(vdev, &cons, req); + if (ret) { + ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret); + goto consumer_del; + } + + ret = ivpu_ipc_receive(vdev, &cons, NULL, resp, timeout_ms); + if (ret) { + ivpu_warn_ratelimited(vdev, "IPC receive failed: type %s, ret %d\n", + ivpu_jsm_msg_type_to_str(req->type), ret); + goto consumer_del; + } + + if (resp->type != expected_resp_type) { + ivpu_warn_ratelimited(vdev, "Invalid JSM response type: 0x%x\n", resp->type); + ret = -EBADE; + } + +consumer_del: + ivpu_ipc_consumer_del(vdev, &cons); + return ret; +} + +int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req, + enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp, + u32 channel, unsigned long timeout_ms) +{ + struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB }; + struct vpu_jsm_msg hb_resp; + int ret, hb_ret; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms); + if (ret != -ETIMEDOUT) + goto rpm_put; + + hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE, + &hb_resp, VPU_IPC_CHAN_ASYNC_CMD, + vdev->timeout.jsm); + if (hb_ret == -ETIMEDOUT) + ivpu_pm_trigger_recovery(vdev, "IPC timeout"); + +rpm_put: + ivpu_rpm_put(vdev); + return ret; +} + +int ivpu_ipc_send_and_wait(struct ivpu_device *vdev, struct vpu_jsm_msg *req, + u32 channel, unsigned long timeout_ms) +{ + struct ivpu_ipc_consumer cons; + int ret; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + ivpu_ipc_consumer_add(vdev, &cons, channel, NULL); + + ret = ivpu_ipc_send(vdev, &cons, req); + if (ret) { + ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret); + goto consumer_del; + } + + msleep(timeout_ms); + +consumer_del: + ivpu_ipc_consumer_del(vdev, &cons); + ivpu_rpm_put(vdev); + return ret; +} + +static bool +ivpu_ipc_match_consumer(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg) +{ + if (cons->channel != ipc_hdr->channel) + return false; + + if (!jsm_msg || jsm_msg->request_id == cons->request_id) + return true; + + return false; +} + +void ivpu_ipc_irq_handler(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + struct ivpu_ipc_consumer *cons; + struct ivpu_ipc_hdr *ipc_hdr; + struct vpu_jsm_msg *jsm_msg; + unsigned long flags; + bool dispatched; + u32 vpu_addr; + + /* + * Driver needs to purge all messages from IPC FIFO to clear IPC interrupt. + * Without purge IPC FIFO to 0 next IPC interrupts won't be generated. + */ + while (ivpu_hw_ipc_rx_count_get(vdev)) { + vpu_addr = ivpu_hw_ipc_rx_addr_get(vdev); + if (vpu_addr == REG_IO_ERROR) { + ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n"); + return; + } + + ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr); + if (!ipc_hdr) { + ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr); + continue; + } + ivpu_ipc_msg_dump(vdev, "RX", ipc_hdr, vpu_addr); + + jsm_msg = NULL; + if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) { + jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr); + if (!jsm_msg) { + ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n", + ipc_hdr->data_addr); + ivpu_ipc_rx_mark_free(vdev, ipc_hdr, NULL); + continue; + } + ivpu_jsm_msg_dump(vdev, "RX", jsm_msg, ipc_hdr->data_addr); + } + + if (atomic_read(&ipc->rx_msg_count) > IPC_MAX_RX_MSG) { + ivpu_warn_ratelimited(vdev, "IPC RX msg dropped, msg count %d\n", + IPC_MAX_RX_MSG); + ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); + continue; + } + + dispatched = false; + spin_lock_irqsave(&ipc->cons_lock, flags); + list_for_each_entry(cons, &ipc->cons_list, link) { + if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) { + ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg); + dispatched = true; + break; + } + } + spin_unlock_irqrestore(&ipc->cons_lock, flags); + + if (!dispatched) { + ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr); + ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); + } + } + + queue_work(system_percpu_wq, &vdev->irq_ipc_work); +} + +void ivpu_ipc_irq_work_fn(struct work_struct *work) +{ + struct ivpu_device *vdev = container_of(work, struct ivpu_device, irq_ipc_work); + struct ivpu_ipc_info *ipc = vdev->ipc; + struct ivpu_ipc_rx_msg *rx_msg, *r; + struct list_head cb_msg_list; + + INIT_LIST_HEAD(&cb_msg_list); + + spin_lock_irq(&ipc->cons_lock); + list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list); + spin_unlock_irq(&ipc->cons_lock); + + list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) { + rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg); + ivpu_ipc_rx_msg_del(vdev, rx_msg); + } +} + +int ivpu_ipc_init(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + int ret; + + ipc->mem_tx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); + if (!ipc->mem_tx) { + ivpu_err(vdev, "Failed to allocate mem_tx\n"); + return -ENOMEM; + } + + ipc->mem_rx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); + if (!ipc->mem_rx) { + ivpu_err(vdev, "Failed to allocate mem_rx\n"); + ret = -ENOMEM; + goto err_free_tx; + } + + ipc->mm_tx = devm_gen_pool_create(vdev->drm.dev, __ffs(IVPU_IPC_ALIGNMENT), + -1, "TX_IPC_JSM"); + if (IS_ERR(ipc->mm_tx)) { + ret = PTR_ERR(ipc->mm_tx); + ivpu_err(vdev, "Failed to create gen pool, %pe\n", ipc->mm_tx); + goto err_free_rx; + } + + ret = gen_pool_add(ipc->mm_tx, ipc->mem_tx->vpu_addr, ivpu_bo_size(ipc->mem_tx), -1); + if (ret) { + ivpu_err(vdev, "gen_pool_add failed, ret %d\n", ret); + goto err_free_rx; + } + + spin_lock_init(&ipc->cons_lock); + INIT_LIST_HEAD(&ipc->cons_list); + INIT_LIST_HEAD(&ipc->cb_msg_list); + ret = drmm_mutex_init(&vdev->drm, &ipc->lock); + if (ret) { + ivpu_err(vdev, "Failed to initialize ipc->lock, ret %d\n", ret); + goto err_free_rx; + } + ivpu_ipc_reset(vdev); + return 0; + +err_free_rx: + ivpu_bo_free(ipc->mem_rx); +err_free_tx: + ivpu_bo_free(ipc->mem_tx); + return ret; +} + +void ivpu_ipc_fini(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + + drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list)); + drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list)); + drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0); + + ivpu_ipc_mem_fini(vdev); +} + +void ivpu_ipc_enable(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + + mutex_lock(&ipc->lock); + ipc->on = true; + mutex_unlock(&ipc->lock); +} + +void ivpu_ipc_disable(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + struct ivpu_ipc_consumer *cons, *c; + struct ivpu_ipc_rx_msg *rx_msg, *r; + + drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list)); + + mutex_lock(&ipc->lock); + ipc->on = false; + mutex_unlock(&ipc->lock); + + spin_lock_irq(&ipc->cons_lock); + list_for_each_entry_safe(cons, c, &ipc->cons_list, link) { + spin_lock(&cons->rx_lock); + if (!cons->rx_callback) + cons->aborted = true; + list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link) + ivpu_ipc_rx_msg_del(vdev, rx_msg); + spin_unlock(&cons->rx_lock); + wake_up(&cons->rx_msg_wq); + } + spin_unlock_irq(&ipc->cons_lock); + + drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0); +} + +void ivpu_ipc_reset(struct ivpu_device *vdev) +{ + struct ivpu_ipc_info *ipc = vdev->ipc; + + mutex_lock(&ipc->lock); + drm_WARN_ON(&vdev->drm, ipc->on); + + memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx)); + memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx)); + wmb(); /* Flush WC buffers for TX and RX rings */ + + mutex_unlock(&ipc->lock); +} diff --git a/drivers/accel/ivpu/ivpu_ipc.h b/drivers/accel/ivpu/ivpu_ipc.h new file mode 100644 index 000000000..b524a1985 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_ipc.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __IVPU_IPC_H__ +#define __IVPU_IPC_H__ + +#include +#include + +#include "vpu_jsm_api.h" + +struct ivpu_bo; + +/* VPU FW boot notification */ +#define IVPU_IPC_CHAN_BOOT_MSG 0x3ff +#define IVPU_IPC_BOOT_MSG_DATA_ADDR 0x424f4f54 + +/* The alignment to be used for IPC Buffers and IPC Data. */ +#define IVPU_IPC_ALIGNMENT 64 + +#define IVPU_IPC_HDR_FREE 0 +#define IVPU_IPC_HDR_ALLOCATED 1 + +/** + * struct ivpu_ipc_hdr - The IPC message header structure, exchanged + * with the VPU device firmware. + * @data_addr: The VPU address of the payload (JSM message) + * @data_size: The size of the payload. + * @channel: The channel used. + * @src_node: The Node ID of the sender. + * @dst_node: The Node ID of the intended receiver. + * @status: IPC buffer usage status + */ +struct ivpu_ipc_hdr { + u32 data_addr; + u32 data_size; + u16 channel; + u8 src_node; + u8 dst_node; + u8 status; +} __packed __aligned(IVPU_IPC_ALIGNMENT); + +typedef void (*ivpu_ipc_rx_callback_t)(struct ivpu_device *vdev, + struct ivpu_ipc_hdr *ipc_hdr, + struct vpu_jsm_msg *jsm_msg); + +struct ivpu_ipc_rx_msg { + struct list_head link; + struct ivpu_ipc_hdr *ipc_hdr; + struct vpu_jsm_msg *jsm_msg; + ivpu_ipc_rx_callback_t callback; +}; + +struct ivpu_ipc_consumer { + struct list_head link; + u32 channel; + u32 tx_vpu_addr; + u32 request_id; + bool aborted; + ivpu_ipc_rx_callback_t rx_callback; + + spinlock_t rx_lock; /* Protects rx_msg_list and aborted */ + struct list_head rx_msg_list; + wait_queue_head_t rx_msg_wq; +}; + +struct ivpu_ipc_info { + struct gen_pool *mm_tx; + struct ivpu_bo *mem_tx; + struct ivpu_bo *mem_rx; + + atomic_t rx_msg_count; + + spinlock_t cons_lock; /* Protects cons_list and cb_msg_list */ + struct list_head cons_list; + struct list_head cb_msg_list; + + atomic_t request_id; + struct mutex lock; /* Lock on status */ + bool on; +}; + +int ivpu_ipc_init(struct ivpu_device *vdev); +void ivpu_ipc_fini(struct ivpu_device *vdev); + +void ivpu_ipc_enable(struct ivpu_device *vdev); +void ivpu_ipc_disable(struct ivpu_device *vdev); +void ivpu_ipc_reset(struct ivpu_device *vdev); + +void ivpu_ipc_irq_handler(struct ivpu_device *vdev); +void ivpu_ipc_irq_work_fn(struct work_struct *work); + +void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + u32 channel, ivpu_ipc_rx_callback_t callback); +void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons); + +int ivpu_ipc_send(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + struct vpu_jsm_msg *req); +int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, + struct ivpu_ipc_hdr *ipc_buf, struct vpu_jsm_msg *jsm_msg, + unsigned long timeout_ms); +int ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req, + enum vpu_ipc_msg_type expected_resp_type, + struct vpu_jsm_msg *resp, u32 channel, unsigned long timeout_ms); +int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req, + enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp, + u32 channel, unsigned long timeout_ms); +int ivpu_ipc_send_and_wait(struct ivpu_device *vdev, struct vpu_jsm_msg *req, + u32 channel, unsigned long timeout_ms); + +#endif /* __IVPU_IPC_H__ */ diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c new file mode 100644 index 000000000..b24f31a8b --- /dev/null +++ b/drivers/accel/ivpu/ivpu_job.c @@ -0,0 +1,1215 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_hw.h" +#include "ivpu_ipc.h" +#include "ivpu_job.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_mmu.h" +#include "ivpu_pm.h" +#include "ivpu_trace.h" +#include "vpu_boot_api.h" + +#define CMD_BUF_IDX 0 +#define JOB_MAX_BUFFER_COUNT 65535 + +static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq) +{ + ivpu_hw_db_set(vdev, cmdq->db_id); +} + +static int ivpu_preemption_buffers_create(struct ivpu_device *vdev, + struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + if (ivpu_fw_preempt_buf_size(vdev) == 0) + return 0; + + cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user, + vdev->fw->primary_preempt_buf_size, + DRM_IVPU_BO_WC); + if (!cmdq->primary_preempt_buf) { + ivpu_err(vdev, "Failed to create primary preemption buffer\n"); + return -ENOMEM; + } + + cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma, + vdev->fw->secondary_preempt_buf_size, + DRM_IVPU_BO_WC); + if (!cmdq->secondary_preempt_buf) { + ivpu_err(vdev, "Failed to create secondary preemption buffer\n"); + goto err_free_primary; + } + + return 0; + +err_free_primary: + ivpu_bo_free(cmdq->primary_preempt_buf); + cmdq->primary_preempt_buf = NULL; + return -ENOMEM; +} + +static void ivpu_preemption_buffers_free(struct ivpu_device *vdev, + struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + if (cmdq->primary_preempt_buf) + ivpu_bo_free(cmdq->primary_preempt_buf); + if (cmdq->secondary_preempt_buf) + ivpu_bo_free(cmdq->secondary_preempt_buf); +} + +static int ivpu_preemption_job_init(struct ivpu_device *vdev, struct ivpu_file_priv *file_priv, + struct ivpu_cmdq *cmdq, struct ivpu_job *job) +{ + int ret; + + /* Use preemption buffer provided by the user space */ + if (job->primary_preempt_buf) + return 0; + + if (!cmdq->primary_preempt_buf) { + /* Allocate per command queue preemption buffers */ + ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq); + if (ret) + return ret; + } + + /* Use preemption buffers allocated by the kernel */ + job->primary_preempt_buf = cmdq->primary_preempt_buf; + job->secondary_preempt_buf = cmdq->secondary_preempt_buf; + + return 0; +} + +static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv) +{ + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_cmdq *cmdq; + + cmdq = kzalloc_obj(*cmdq); + if (!cmdq) + return NULL; + + cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); + if (!cmdq->mem) + goto err_free_cmdq; + + return cmdq; + +err_free_cmdq: + kfree(cmdq); + return NULL; +} + +/** + * ivpu_cmdq_get_entry_count - Calculate the number of entries in the command queue. + * @cmdq: Pointer to the command queue structure. + * + * Returns the number of entries that can fit in the command queue memory. + */ +static inline u32 ivpu_cmdq_get_entry_count(struct ivpu_cmdq *cmdq) +{ + size_t size = ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header); + + return size / sizeof(struct vpu_job_queue_entry); +} + +/** + * ivpu_cmdq_get_flags - Get command queue flags based on input flags and test mode. + * @vdev: Pointer to the ivpu device structure. + * @flags: Input flags to determine the command queue flags. + * + * Returns the calculated command queue flags, considering both the input flags + * and the current test mode settings. + */ +static u32 ivpu_cmdq_get_flags(struct ivpu_device *vdev, u32 flags) +{ + u32 cmdq_flags = 0; + + if ((flags & DRM_IVPU_CMDQ_FLAG_TURBO) && (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)) + cmdq_flags |= VPU_JOB_QUEUE_FLAGS_TURBO_MODE; + + /* Test mode can override the TURBO flag coming from the application */ + if (ivpu_test_mode & IVPU_TEST_MODE_TURBO_ENABLE) + cmdq_flags |= VPU_JOB_QUEUE_FLAGS_TURBO_MODE; + if (ivpu_test_mode & IVPU_TEST_MODE_TURBO_DISABLE) + cmdq_flags &= ~VPU_JOB_QUEUE_FLAGS_TURBO_MODE; + + return cmdq_flags; +} + +static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq); + ivpu_bo_free(cmdq->mem); + kfree(cmdq); +} + +static struct ivpu_cmdq *ivpu_cmdq_create(struct ivpu_file_priv *file_priv, u8 priority, u32 flags) +{ + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_cmdq *cmdq = NULL; + int ret; + + lockdep_assert_held(&file_priv->lock); + + cmdq = ivpu_cmdq_alloc(file_priv); + if (!cmdq) { + ivpu_err(vdev, "Failed to allocate command queue\n"); + return NULL; + } + ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit, + &file_priv->cmdq_id_next, GFP_KERNEL); + if (ret < 0) { + ivpu_dbg(vdev, IOCTL, "Failed to allocate command queue ID: %d\n", ret); + goto err_free_cmdq; + } + + cmdq->entry_count = ivpu_cmdq_get_entry_count(cmdq); + cmdq->priority = priority; + + cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem); + cmdq->jobq->header.engine_idx = VPU_ENGINE_COMPUTE; + cmdq->jobq->header.flags = ivpu_cmdq_get_flags(vdev, flags); + + ivpu_dbg(vdev, JOB, "Command queue %d created, ctx %d, flags 0x%08x\n", + cmdq->id, file_priv->ctx.id, cmdq->jobq->header.flags); + return cmdq; + +err_free_cmdq: + ivpu_cmdq_free(file_priv, cmdq); + return NULL; +} + +static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine, + u8 priority) +{ + struct ivpu_device *vdev = file_priv->vdev; + int ret; + + ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id, + task_pid_nr(current), engine, + cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); + if (ret) + return ret; + + ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id, + priority); + if (ret) + ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); + + return ret; +} + +static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + struct ivpu_user_limits *limits = file_priv->user_limits; + struct ivpu_device *vdev = file_priv->vdev; + int ret; + + if (atomic_inc_return(&limits->db_count) > limits->max_db_count) { + ivpu_dbg(vdev, IOCTL, "Maximum number of %u doorbells for uid %u reached\n", + limits->max_db_count, limits->uid); + ret = -EBUSY; + goto err_dec_db_count; + } + + ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next, + GFP_KERNEL); + if (ret < 0) { + ivpu_dbg(vdev, IOCTL, "Failed to allocate doorbell ID: %d\n", ret); + goto err_dec_db_count; + } + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) + ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id, + cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); + else + ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id, + cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem)); + if (ret) { + xa_erase(&vdev->db_xa, cmdq->db_id); + cmdq->db_id = 0; + goto err_dec_db_count; + } + + ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d priority %d\n", + cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority); + return 0; + +err_dec_db_count: + atomic_dec(&limits->db_count); + return ret; +} + +static void ivpu_cmdq_jobq_reset(struct ivpu_device *vdev, struct vpu_job_queue *jobq) +{ + jobq->header.head = 0; + jobq->header.tail = 0; + + wmb(); /* Flush WC buffer for jobq->header */ +} + +static int ivpu_cmdq_register(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + struct ivpu_device *vdev = file_priv->vdev; + int ret; + + lockdep_assert_held(&file_priv->lock); + + if (cmdq->db_id) + return 0; + + ivpu_cmdq_jobq_reset(vdev, cmdq->jobq); + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { + ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, cmdq->priority); + if (ret) + return ret; + } + + ret = ivpu_register_db(file_priv, cmdq); + if (ret && vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) + ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); + + return ret; +} + +static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + struct ivpu_device *vdev = file_priv->vdev; + int ret; + + lockdep_assert_held(&file_priv->lock); + + if (!cmdq->db_id) + return 0; + + ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id); + if (!ret) + ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id); + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) { + ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id); + if (!ret) + ivpu_dbg(vdev, JOB, "Command queue %d destroyed, ctx %d\n", + cmdq->id, file_priv->ctx.id); + } + + xa_erase(&file_priv->vdev->db_xa, cmdq->db_id); + atomic_dec(&file_priv->user_limits->db_count); + cmdq->db_id = 0; + + return 0; +} + +static inline u8 ivpu_job_to_jsm_priority(u8 priority) +{ + if (priority == DRM_IVPU_JOB_PRIORITY_DEFAULT) + return VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL; + + return priority - 1; +} + +static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq) +{ + lockdep_assert_held(&file_priv->lock); + ivpu_cmdq_unregister(file_priv, cmdq); + xa_erase(&file_priv->cmdq_xa, cmdq->id); + ivpu_cmdq_free(file_priv, cmdq); +} + +static struct ivpu_cmdq *ivpu_cmdq_acquire_legacy(struct ivpu_file_priv *file_priv, u8 priority) +{ + struct ivpu_cmdq *cmdq; + unsigned long id; + + lockdep_assert_held(&file_priv->lock); + + xa_for_each(&file_priv->cmdq_xa, id, cmdq) + if (cmdq->is_legacy && cmdq->priority == priority) + break; + + if (!cmdq) { + cmdq = ivpu_cmdq_create(file_priv, priority, 0); + if (!cmdq) + return NULL; + cmdq->is_legacy = true; + } + + return cmdq; +} + +static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u32 cmdq_id) +{ + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_cmdq *cmdq; + + lockdep_assert_held(&file_priv->lock); + + cmdq = xa_load(&file_priv->cmdq_xa, cmdq_id); + if (!cmdq) { + ivpu_dbg(vdev, IOCTL, "Failed to find command queue with ID: %u\n", cmdq_id); + return NULL; + } + + return cmdq; +} + +void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv) +{ + struct ivpu_cmdq *cmdq; + unsigned long cmdq_id; + + lockdep_assert_held(&file_priv->lock); + + xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) + ivpu_cmdq_destroy(file_priv, cmdq); +} + +/* + * Mark the doorbell as unregistered + * This function needs to be called when the VPU hardware is restarted + * and FW loses job queue state. The next time job queue is used it + * will be registered again. + */ +static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv) +{ + struct ivpu_cmdq *cmdq; + unsigned long cmdq_id; + + mutex_lock(&file_priv->lock); + + xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) { + if (cmdq->db_id) { + xa_erase(&file_priv->vdev->db_xa, cmdq->db_id); + atomic_dec(&file_priv->user_limits->db_count); + cmdq->db_id = 0; + } + } + + mutex_unlock(&file_priv->lock); +} + +void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev) +{ + struct ivpu_file_priv *file_priv; + unsigned long ctx_id; + + mutex_lock(&vdev->context_list_lock); + + xa_for_each(&vdev->context_xa, ctx_id, file_priv) + ivpu_cmdq_reset(file_priv); + + mutex_unlock(&vdev->context_list_lock); +} + +void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv) +{ + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_cmdq *cmdq; + unsigned long cmdq_id; + + lockdep_assert_held(&file_priv->lock); + ivpu_dbg(vdev, JOB, "Context ID: %u abort\n", file_priv->ctx.id); + + xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) + ivpu_cmdq_unregister(file_priv, cmdq); + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_OS) + ivpu_jsm_context_release(vdev, file_priv->ctx.id); + + ivpu_mmu_disable_ssid_events(vdev, file_priv->ctx.id); + + file_priv->aborted = true; +} + +static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job) +{ + struct ivpu_device *vdev = job->vdev; + struct vpu_job_queue_header *header = &cmdq->jobq->header; + struct vpu_job_queue_entry *entry; + u32 tail = READ_ONCE(header->tail); + u32 next_entry = (tail + 1) % cmdq->entry_count; + + /* Check if there is space left in job queue */ + if (next_entry == header->head) { + ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n", + job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail); + return -EBUSY; + } + + entry = &cmdq->jobq->slot[tail].job; + entry->batch_buf_addr = job->cmd_buf_vpu_addr; + entry->job_id = job->job_id; + entry->flags = 0; + if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION)) + entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK; + + if (job->primary_preempt_buf) { + entry->primary_preempt_buf_addr = job->primary_preempt_buf->vpu_addr; + entry->primary_preempt_buf_size = ivpu_bo_size(job->primary_preempt_buf); + } + + if (job->secondary_preempt_buf) { + entry->secondary_preempt_buf_addr = job->secondary_preempt_buf->vpu_addr; + entry->secondary_preempt_buf_size = ivpu_bo_size(job->secondary_preempt_buf); + } + + wmb(); /* Ensure that tail is updated after filling entry */ + header->tail = next_entry; + wmb(); /* Flush WC buffer for jobq header */ + + return 0; +} + +struct ivpu_fence { + struct dma_fence base; + spinlock_t lock; /* protects base */ + struct ivpu_device *vdev; +}; + +static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence) +{ + return container_of(fence, struct ivpu_fence, base); +} + +static const char *ivpu_fence_get_driver_name(struct dma_fence *fence) +{ + return DRIVER_NAME; +} + +static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence) +{ + struct ivpu_fence *ivpu_fence = to_vpu_fence(fence); + + return dev_name(ivpu_fence->vdev->drm.dev); +} + +static const struct dma_fence_ops ivpu_fence_ops = { + .get_driver_name = ivpu_fence_get_driver_name, + .get_timeline_name = ivpu_fence_get_timeline_name, +}; + +static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev) +{ + struct ivpu_fence *fence; + + fence = kzalloc_obj(*fence); + if (!fence) + return NULL; + + fence->vdev = vdev; + spin_lock_init(&fence->lock); + dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1); + + return &fence->base; +} + +static void ivpu_job_destroy(struct ivpu_job *job) +{ + struct ivpu_device *vdev = job->vdev; + u32 i; + + ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d cmdq_id %u engine %d", + job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx); + + for (i = 0; i < job->bo_count; i++) + if (job->bos[i]) + drm_gem_object_put(&job->bos[i]->base.base); + + dma_fence_put(job->done_fence); + ivpu_file_priv_put(&job->file_priv); + kfree(job); +} + +static struct ivpu_job * +ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count) +{ + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_job *job; + + job = kzalloc_flex(*job, bos, bo_count); + if (!job) + return NULL; + + job->vdev = vdev; + job->engine_idx = engine_idx; + job->bo_count = bo_count; + job->done_fence = ivpu_fence_create(vdev); + if (!job->done_fence) { + ivpu_err(vdev, "Failed to create a fence\n"); + goto err_free_job; + } + + job->file_priv = ivpu_file_priv_get(file_priv); + + trace_job("create", job); + ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx); + return job; + +err_free_job: + kfree(job); + return NULL; +} + +static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id) +{ + struct ivpu_job *job; + + lockdep_assert_held(&vdev->submitted_jobs_lock); + + job = xa_erase(&vdev->submitted_jobs_xa, job_id); + if (xa_empty(&vdev->submitted_jobs_xa) && job) { + vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts), + vdev->busy_time); + } + + return job; +} + +bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_status) +{ + lockdep_assert_held(&vdev->submitted_jobs_lock); + + switch (job_status) { + case VPU_JSM_STATUS_PROCESSING_ERR: + case VPU_JSM_STATUS_ENGINE_RESET_REQUIRED_MIN ... VPU_JSM_STATUS_ENGINE_RESET_REQUIRED_MAX: + { + struct ivpu_job *job = xa_load(&vdev->submitted_jobs_xa, job_id); + + if (!job) + return false; + + /* Trigger an engine reset */ + guard(mutex)(&job->file_priv->lock); + + job->job_status = job_status; + + if (job->file_priv->has_mmu_faults) + return false; + + /* + * Mark context as faulty and defer destruction of the job to jobs abort thread + * handler to synchronize between both faults and jobs returning context violation + * status and ensure both are handled in the same way + */ + job->file_priv->has_mmu_faults = true; + atomic_set(&vdev->faults_detected, 1); + queue_work(system_percpu_wq, &vdev->context_abort_work); + return true; + } + default: + /* Complete job with error status, engine reset not required */ + break; + } + + return false; +} + +static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) +{ + struct ivpu_job *job; + + lockdep_assert_held(&vdev->submitted_jobs_lock); + + job = xa_load(&vdev->submitted_jobs_xa, job_id); + if (!job) + return -ENOENT; + + ivpu_job_remove_from_submitted_jobs(vdev, job_id); + + if (job->job_status == VPU_JSM_STATUS_SUCCESS) { + if (job->file_priv->has_mmu_faults) + job->job_status = DRM_IVPU_JOB_STATUS_ABORTED; + else + job->job_status = job_status; + } + + job->bos[CMD_BUF_IDX]->job_status = job->job_status; + dma_fence_signal(job->done_fence); + + trace_job("done", job); + ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d cmdq_id %u engine %d status 0x%x\n", + job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx, + job->job_status); + + ivpu_job_destroy(job); + ivpu_stop_job_timeout_detection(vdev); + + ivpu_rpm_put(vdev); + + if (!xa_empty(&vdev->submitted_jobs_xa)) + ivpu_start_job_timeout_detection(vdev); + + return 0; +} + +void ivpu_jobs_abort_all(struct ivpu_device *vdev) +{ + struct ivpu_job *job; + unsigned long id; + + mutex_lock(&vdev->submitted_jobs_lock); + + xa_for_each(&vdev->submitted_jobs_xa, id, job) + ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED); + + mutex_unlock(&vdev->submitted_jobs_lock); +} + +void ivpu_cmdq_abort_all_jobs(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id) +{ + struct ivpu_job *job; + unsigned long id; + + mutex_lock(&vdev->submitted_jobs_lock); + + xa_for_each(&vdev->submitted_jobs_xa, id, job) + if (job->file_priv->ctx.id == ctx_id && job->cmdq_id == cmdq_id) + ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED); + + mutex_unlock(&vdev->submitted_jobs_lock); +} + +static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id) +{ + struct ivpu_file_priv *file_priv = job->file_priv; + struct ivpu_device *vdev = job->vdev; + struct ivpu_cmdq *cmdq; + bool is_first_job; + int ret; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + mutex_lock(&vdev->submitted_jobs_lock); + mutex_lock(&file_priv->lock); + + if (cmdq_id == 0) + cmdq = ivpu_cmdq_acquire_legacy(file_priv, priority); + else + cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id); + if (!cmdq) { + ret = -EINVAL; + goto err_unlock; + } + + ret = ivpu_cmdq_register(file_priv, cmdq); + if (ret) { + ivpu_err(vdev, "Failed to register command queue: %d\n", ret); + goto err_unlock; + } + + ret = ivpu_preemption_job_init(vdev, file_priv, cmdq, job); + if (ret) { + ivpu_err(vdev, "Failed to initialize preemption buffers for job %d: %d\n", + job->job_id, ret); + goto err_unlock; + } + + job->cmdq_id = cmdq->id; + + is_first_job = xa_empty(&vdev->submitted_jobs_xa); + ret = xa_alloc_cyclic(&vdev->submitted_jobs_xa, &job->job_id, job, file_priv->job_limit, + &file_priv->job_id_next, GFP_KERNEL); + if (ret < 0) { + ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n", + file_priv->ctx.id); + ret = -EBUSY; + goto err_unlock; + } + + ret = ivpu_cmdq_push_job(cmdq, job); + if (ret) + goto err_erase_xa; + + ivpu_start_job_timeout_detection(vdev); + + if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) { + cmdq->jobq->header.head = cmdq->jobq->header.tail; + wmb(); /* Flush WC buffer for jobq header */ + } else { + ivpu_cmdq_ring_db(vdev, cmdq); + if (is_first_job) + vdev->busy_start_ts = ktime_get(); + } + + trace_job("submit", job); + ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d cmdq_id %u engine %d prio %d addr 0x%llx next %d\n", + job->job_id, file_priv->ctx.id, cmdq->id, job->engine_idx, cmdq->priority, + job->cmd_buf_vpu_addr, cmdq->jobq->header.tail); + + mutex_unlock(&file_priv->lock); + + if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) { + ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS); + } + + mutex_unlock(&vdev->submitted_jobs_lock); + + return 0; + +err_erase_xa: + xa_erase(&vdev->submitted_jobs_xa, job->job_id); +err_unlock: + mutex_unlock(&file_priv->lock); + mutex_unlock(&vdev->submitted_jobs_lock); + ivpu_rpm_put(vdev); + return ret; +} + +static int +ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles, + u32 buf_count, u32 commands_offset, u32 preempt_buffer_index) +{ + struct ivpu_file_priv *file_priv = job->file_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct ww_acquire_ctx acquire_ctx; + enum dma_resv_usage usage; + struct ivpu_bo *bo; + int ret; + u32 i; + + for (i = 0; i < buf_count; i++) { + struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]); + + if (!obj) { + ivpu_dbg(vdev, IOCTL, "Failed to lookup GEM object with handle %u\n", + buf_handles[i]); + return -ENOENT; + } + + job->bos[i] = to_ivpu_bo(obj); + + ret = ivpu_bo_bind(job->bos[i]); + if (ret) + return ret; + } + + bo = job->bos[CMD_BUF_IDX]; + if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) { + ivpu_dbg(vdev, IOCTL, "Buffer is already in use by another job\n"); + return -EBUSY; + } + + if (commands_offset >= ivpu_bo_size(bo)) { + ivpu_dbg(vdev, IOCTL, "Invalid commands offset %u for buffer size %zu\n", + commands_offset, ivpu_bo_size(bo)); + return -EINVAL; + } + + job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset; + + if (preempt_buffer_index) { + struct ivpu_bo *preempt_bo = job->bos[preempt_buffer_index]; + + if (ivpu_bo_size(preempt_bo) < ivpu_fw_preempt_buf_size(vdev)) { + ivpu_dbg(vdev, IOCTL, "Preemption buffer is too small\n"); + return -EINVAL; + } + if (ivpu_bo_is_mappable(preempt_bo)) { + ivpu_dbg(vdev, IOCTL, "Preemption buffer cannot be mappable\n"); + return -EINVAL; + } + job->primary_preempt_buf = preempt_bo; + } + + ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count, + &acquire_ctx); + if (ret) { + ivpu_warn_ratelimited(vdev, "Failed to lock reservations: %d\n", ret); + return ret; + } + + for (i = 0; i < buf_count; i++) { + ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1); + if (ret) { + ivpu_warn_ratelimited(vdev, "Failed to reserve fences: %d\n", ret); + goto unlock_reservations; + } + } + + for (i = 0; i < buf_count; i++) { + usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP; + dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage); + } + +unlock_reservations: + drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx); + + wmb(); /* Flush write combining buffers */ + + return ret; +} + +static int ivpu_submit(struct drm_file *file, struct ivpu_file_priv *file_priv, u32 cmdq_id, + u32 buffer_count, u32 engine, void __user *buffers_ptr, u32 cmds_offset, + u32 preempt_buffer_index, u8 priority) +{ + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_job *job; + u32 *buf_handles; + int idx, ret; + + buf_handles = kcalloc(buffer_count, sizeof(u32), GFP_KERNEL); + if (!buf_handles) + return -ENOMEM; + + ret = copy_from_user(buf_handles, buffers_ptr, buffer_count * sizeof(u32)); + if (ret) { + ret = -EFAULT; + goto err_free_handles; + } + + if (!drm_dev_enter(&vdev->drm, &idx)) { + ret = -ENODEV; + goto err_free_handles; + } + + ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u cmdq_id %u buf_count %u\n", + file_priv->ctx.id, cmdq_id, buffer_count); + + job = ivpu_job_create(file_priv, engine, buffer_count); + if (!job) { + ret = -ENOMEM; + goto err_exit_dev; + } + + ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, buffer_count, cmds_offset, + preempt_buffer_index); + if (ret) + goto err_destroy_job; + + down_read(&vdev->pm->reset_lock); + ret = ivpu_job_submit(job, priority, cmdq_id); + up_read(&vdev->pm->reset_lock); + if (ret) + goto err_signal_fence; + + drm_dev_exit(idx); + kfree(buf_handles); + return ret; + +err_signal_fence: + dma_fence_signal(job->done_fence); +err_destroy_job: + ivpu_job_destroy(job); +err_exit_dev: + drm_dev_exit(idx); +err_free_handles: + kfree(buf_handles); + return ret; +} + +int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct drm_ivpu_submit *args = data; + u8 priority; + + if (args->engine != DRM_IVPU_ENGINE_COMPUTE) { + ivpu_dbg(vdev, IOCTL, "Invalid engine %d\n", args->engine); + return -EINVAL; + } + + if (args->priority > DRM_IVPU_JOB_PRIORITY_REALTIME) { + ivpu_dbg(vdev, IOCTL, "Invalid priority %d\n", args->priority); + return -EINVAL; + } + + if (args->buffer_count == 0 || args->buffer_count > JOB_MAX_BUFFER_COUNT) { + ivpu_dbg(vdev, IOCTL, "Invalid buffer count %u\n", args->buffer_count); + return -EINVAL; + } + + if (!IS_ALIGNED(args->commands_offset, 8)) { + ivpu_dbg(vdev, IOCTL, "Invalid commands offset %u\n", args->commands_offset); + return -EINVAL; + } + + if (!file_priv->ctx.id) { + ivpu_dbg(vdev, IOCTL, "Context not initialized\n"); + return -EINVAL; + } + + if (file_priv->has_mmu_faults) { + ivpu_dbg(vdev, IOCTL, "Context %u has MMU faults\n", file_priv->ctx.id); + return -EBADFD; + } + + priority = ivpu_job_to_jsm_priority(args->priority); + + return ivpu_submit(file, file_priv, 0, args->buffer_count, args->engine, + (void __user *)args->buffers_ptr, args->commands_offset, 0, priority); +} + +int ivpu_cmdq_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct drm_ivpu_cmdq_submit *args = data; + + if (!ivpu_is_capable(file_priv->vdev, DRM_IVPU_CAP_MANAGE_CMDQ)) { + ivpu_dbg(vdev, IOCTL, "Command queue management not supported\n"); + return -ENODEV; + } + + if (args->cmdq_id < IVPU_CMDQ_MIN_ID || args->cmdq_id > IVPU_CMDQ_MAX_ID) { + ivpu_dbg(vdev, IOCTL, "Invalid command queue ID %u\n", args->cmdq_id); + return -EINVAL; + } + + if (args->buffer_count == 0 || args->buffer_count > JOB_MAX_BUFFER_COUNT) { + ivpu_dbg(vdev, IOCTL, "Invalid buffer count %u\n", args->buffer_count); + return -EINVAL; + } + + if (args->preempt_buffer_index >= args->buffer_count) { + ivpu_dbg(vdev, IOCTL, "Invalid preemption buffer index %u\n", + args->preempt_buffer_index); + return -EINVAL; + } + + if (!IS_ALIGNED(args->commands_offset, 8)) { + ivpu_dbg(vdev, IOCTL, "Invalid commands offset %u\n", args->commands_offset); + return -EINVAL; + } + + if (!file_priv->ctx.id) { + ivpu_dbg(vdev, IOCTL, "Context not initialized\n"); + return -EINVAL; + } + + if (file_priv->has_mmu_faults) { + ivpu_dbg(vdev, IOCTL, "Context %u has MMU faults\n", file_priv->ctx.id); + return -EBADFD; + } + + return ivpu_submit(file, file_priv, args->cmdq_id, args->buffer_count, VPU_ENGINE_COMPUTE, + (void __user *)args->buffers_ptr, args->commands_offset, + args->preempt_buffer_index, 0); +} + +int ivpu_cmdq_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct drm_ivpu_cmdq_create *args = data; + struct ivpu_cmdq *cmdq; + int ret; + + if (!ivpu_is_capable(vdev, DRM_IVPU_CAP_MANAGE_CMDQ)) { + ivpu_dbg(vdev, IOCTL, "Command queue management not supported\n"); + return -ENODEV; + } + + if (args->priority > DRM_IVPU_JOB_PRIORITY_REALTIME) { + ivpu_dbg(vdev, IOCTL, "Invalid priority %d\n", args->priority); + return -EINVAL; + } + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + mutex_lock(&file_priv->lock); + + cmdq = ivpu_cmdq_create(file_priv, ivpu_job_to_jsm_priority(args->priority), args->flags); + if (cmdq) + args->cmdq_id = cmdq->id; + + mutex_unlock(&file_priv->lock); + + ivpu_rpm_put(vdev); + + return cmdq ? 0 : -ENOMEM; +} + +int ivpu_cmdq_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct drm_ivpu_cmdq_destroy *args = data; + struct ivpu_cmdq *cmdq; + u32 cmdq_id = 0; + int ret; + + if (!ivpu_is_capable(vdev, DRM_IVPU_CAP_MANAGE_CMDQ)) { + ivpu_dbg(vdev, IOCTL, "Command queue management not supported\n"); + return -ENODEV; + } + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + mutex_lock(&file_priv->lock); + + cmdq = xa_load(&file_priv->cmdq_xa, args->cmdq_id); + if (!cmdq || cmdq->is_legacy) { + ret = -ENOENT; + } else { + cmdq_id = cmdq->id; + ivpu_cmdq_destroy(file_priv, cmdq); + ret = 0; + } + + mutex_unlock(&file_priv->lock); + + /* Abort any pending jobs only if cmdq was destroyed */ + if (!ret) + ivpu_cmdq_abort_all_jobs(vdev, file_priv->ctx.id, cmdq_id); + + ivpu_rpm_put(vdev); + + return ret; +} + +static void +ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, + struct vpu_jsm_msg *jsm_msg) +{ + struct vpu_ipc_msg_payload_job_done *payload; + + if (!jsm_msg) { + ivpu_err(vdev, "IPC message has no JSM payload\n"); + return; + } + + if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) { + ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result); + return; + } + + payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload; + + mutex_lock(&vdev->submitted_jobs_lock); + if (!ivpu_job_handle_engine_error(vdev, payload->job_id, payload->job_status)) + /* No engine error, complete the job normally */ + ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status); + mutex_unlock(&vdev->submitted_jobs_lock); +} + +void ivpu_job_done_consumer_init(struct ivpu_device *vdev) +{ + ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer, + VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback); +} + +void ivpu_job_done_consumer_fini(struct ivpu_device *vdev) +{ + ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer); +} + +static int reset_engine_and_mark_faulty_contexts(struct ivpu_device *vdev) +{ + u32 num_impacted_contexts; + struct vpu_jsm_msg resp; + int ret; + u32 i; + + ret = ivpu_jsm_reset_engine(vdev, 0, &resp); + if (ret) + return ret; + + /* + * If faults are detected, ignore guilty contexts from engine reset as NPU may not be stuck + * and could return currently running good context and faulty contexts are already marked + */ + if (atomic_cmpxchg(&vdev->faults_detected, 1, 0) == 1) + return 0; + + num_impacted_contexts = resp.payload.engine_reset_done.num_impacted_contexts; + + ivpu_warn_ratelimited(vdev, "Engine reset performed, impacted contexts: %u\n", + num_impacted_contexts); + + if (!in_range(num_impacted_contexts, 1, VPU_MAX_ENGINE_RESET_IMPACTED_CONTEXTS - 1)) { + ivpu_pm_trigger_recovery(vdev, "Cannot determine guilty contexts"); + return -EIO; + } + + /* No faults detected, NPU likely got stuck. Mark returned contexts as guilty */ + guard(mutex)(&vdev->context_list_lock); + + for (i = 0; i < num_impacted_contexts; i++) { + u32 ssid = resp.payload.engine_reset_done.impacted_contexts[i].host_ssid; + struct ivpu_file_priv *file_priv = xa_load(&vdev->context_xa, ssid); + + if (file_priv) { + mutex_lock(&file_priv->lock); + file_priv->has_mmu_faults = true; + mutex_unlock(&file_priv->lock); + } + } + + return 0; +} + +void ivpu_context_abort_work_fn(struct work_struct *work) +{ + struct ivpu_device *vdev = container_of(work, struct ivpu_device, context_abort_work); + struct ivpu_file_priv *file_priv; + struct ivpu_job *job; + unsigned long ctx_id; + unsigned long id; + + if (drm_WARN_ON(&vdev->drm, pm_runtime_get_if_active(vdev->drm.dev) <= 0)) + return; + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) + if (reset_engine_and_mark_faulty_contexts(vdev)) + goto runtime_put; + + mutex_lock(&vdev->context_list_lock); + xa_for_each(&vdev->context_xa, ctx_id, file_priv) { + if (!file_priv->has_mmu_faults || file_priv->aborted) + continue; + + mutex_lock(&file_priv->lock); + ivpu_context_abort_locked(file_priv); + mutex_unlock(&file_priv->lock); + } + mutex_unlock(&vdev->context_list_lock); + + /* + * We will not receive new MMU event interrupts until existing events are discarded + * however, we want to discard these events only after aborting the faulty context + * to avoid generating new faults from that context + */ + ivpu_mmu_discard_events(vdev); + + if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW) + goto runtime_put; + + if (ivpu_jsm_hws_resume_engine(vdev, 0)) + goto runtime_put; + /* + * In hardware scheduling mode NPU already has stopped processing jobs + * and won't send us any further notifications, thus we have to free job related resources + * and notify userspace + */ + mutex_lock(&vdev->submitted_jobs_lock); + xa_for_each(&vdev->submitted_jobs_xa, id, job) + if (job->file_priv->aborted) + ivpu_job_signal_and_destroy(vdev, job->job_id, DRM_IVPU_JOB_STATUS_ABORTED); + mutex_unlock(&vdev->submitted_jobs_lock); + +runtime_put: + pm_runtime_put_autosuspend(vdev->drm.dev); +} diff --git a/drivers/accel/ivpu/ivpu_job.h b/drivers/accel/ivpu/ivpu_job.h new file mode 100644 index 000000000..3ab61e6a5 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_job.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2025 Intel Corporation + */ + +#ifndef __IVPU_JOB_H__ +#define __IVPU_JOB_H__ + +#include +#include + +#include "ivpu_gem.h" + +struct ivpu_device; +struct ivpu_file_priv; + +/** + * struct ivpu_cmdq - Represents a command queue for submitting jobs to the VPU. + * Tracks queue memory, preemption buffers, and metadata for job management. + * @jobq: Pointer to job queue memory shared with the device + * @primary_preempt_buf: Primary preemption buffer for this queue (optional) + * @secondary_preempt_buf: Secondary preemption buffer for this queue (optional) + * @mem: Memory allocated for the job queue, shared with device + * @entry_count: Number of job entries in the queue + * @id: Unique command queue ID + * @db_id: Doorbell ID assigned to this job queue + * @priority: Priority level of the command queue + * @is_legacy: True if this is a legacy command queue + */ +struct ivpu_cmdq { + struct vpu_job_queue *jobq; + struct ivpu_bo *primary_preempt_buf; + struct ivpu_bo *secondary_preempt_buf; + struct ivpu_bo *mem; + u32 entry_count; + u32 id; + u32 db_id; + u8 priority; + bool is_legacy; +}; + +/** + * struct ivpu_job - Representing a batch or DMA buffer submitted to the VPU. + * Each job is a unit of execution, tracked by job_id for status reporting from VPU FW. + * The structure holds all resources and metadata needed for job submission, execution, + * and completion handling. + * @vdev: Pointer to the VPU device + * @file_priv: The client context that submitted this job + * @done_fence: Fence signaled when job completes + * @cmd_buf_vpu_addr: VPU address of the command buffer for this job + * @cmdq_id: Command queue ID used for submission + * @job_id: Unique job ID for tracking and status reporting + * @engine_idx: Engine index for job execution + * @job_status: Status reported by firmware for this job + * @primary_preempt_buf: Primary preemption buffer for job + * @secondary_preempt_buf: Secondary preemption buffer for job (optional) + * @bo_count: Number of buffer objects associated with this job + * @bos: Array of buffer objects used by the job (batch buffer is at index 0) + */ +struct ivpu_job { + struct ivpu_device *vdev; + struct ivpu_file_priv *file_priv; + struct dma_fence *done_fence; + u64 cmd_buf_vpu_addr; + u32 cmdq_id; + u32 job_id; + u32 engine_idx; + u32 job_status; + struct ivpu_bo *primary_preempt_buf; + struct ivpu_bo *secondary_preempt_buf; + size_t bo_count; + struct ivpu_bo *bos[] __counted_by(bo_count); +}; + +int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_cmdq_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_cmdq_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_cmdq_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file); + +void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv); + +void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv); +void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev); +void ivpu_cmdq_abort_all_jobs(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id); + +void ivpu_job_done_consumer_init(struct ivpu_device *vdev); +void ivpu_job_done_consumer_fini(struct ivpu_device *vdev); +bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_status); +void ivpu_context_abort_work_fn(struct work_struct *work); + +void ivpu_jobs_abort_all(struct ivpu_device *vdev); + +#endif /* __IVPU_JOB_H__ */ diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.c b/drivers/accel/ivpu/ivpu_jsm_msg.c new file mode 100644 index 000000000..6361fe505 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_jsm_msg.c @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include "ivpu_drv.h" +#include "ivpu_hw.h" +#include "ivpu_ipc.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_pm.h" +#include "vpu_jsm_api.h" + +const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type) +{ + #define IVPU_CASE_TO_STR(x) case x: return #x + switch (type) { + IVPU_CASE_TO_STR(VPU_JSM_MSG_UNKNOWN); + IVPU_CASE_TO_STR(VPU_JSM_MSG_ENGINE_RESET); + IVPU_CASE_TO_STR(VPU_JSM_MSG_ENGINE_PREEMPT); + IVPU_CASE_TO_STR(VPU_JSM_MSG_REGISTER_DB); + IVPU_CASE_TO_STR(VPU_JSM_MSG_UNREGISTER_DB); + IVPU_CASE_TO_STR(VPU_JSM_MSG_QUERY_ENGINE_HB); + IVPU_CASE_TO_STR(VPU_JSM_MSG_GET_POWER_LEVEL_COUNT); + IVPU_CASE_TO_STR(VPU_JSM_MSG_GET_POWER_LEVEL); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_POWER_LEVEL); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_OPEN); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_CLOSE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_SET_CONFIG); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_GET_CONFIG); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_GET_CAPABILITY); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_GET_NAME); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SSID_RELEASE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_START); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_STOP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_UPDATE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_INFO); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_CREATE_CMD_QUEUE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DESTROY_CMD_QUEUE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_REGISTER_DB); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_CMDQ); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SUSPEND_CMDQ); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_ENGINE_RESUME); + IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_STATE_DUMP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_STATE_DUMP_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_BLOB_DEINIT_DEPRECATED); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DYNDBG_CONTROL); + IVPU_CASE_TO_STR(VPU_JSM_MSG_JOB_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_NATIVE_FENCE_SIGNALLED); + IVPU_CASE_TO_STR(VPU_JSM_MSG_ENGINE_RESET_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_ENGINE_PREEMPT_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_REGISTER_DB_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_UNREGISTER_DB_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_QUERY_ENGINE_HB_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_GET_POWER_LEVEL_COUNT_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_GET_POWER_LEVEL_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_POWER_LEVEL_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_OPEN_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_CLOSE_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_SET_CONFIG_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_GET_CONFIG_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_GET_CAPABILITY_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_TRACE_GET_NAME_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SSID_RELEASE_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_START_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_STOP_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_UPDATE_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_INFO_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_METRIC_STREAMER_NOTIFICATION); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DESTROY_CMD_QUEUE_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_BLOB_DEINIT_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DYNDBG_CONTROL_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_PWR_D0I3_ENTER); + IVPU_CASE_TO_STR(VPU_JSM_MSG_PWR_D0I3_ENTER_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE_DONE); + IVPU_CASE_TO_STR(VPU_JSM_MSG_FREQ_CONFIG); + IVPU_CASE_TO_STR(VPU_JSM_MSG_FREQ_CONFIG_RSP); + IVPU_CASE_TO_STR(VPU_JSM_MSG_RESERVED_111E); + } + #undef IVPU_CASE_TO_STR + + return "Unknown JSM message type"; +} + +int ivpu_jsm_register_db(struct ivpu_device *vdev, u32 ctx_id, u32 db_id, + u64 jobq_base, u32 jobq_size) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_REGISTER_DB }; + struct vpu_jsm_msg resp; + int ret = 0; + + req.payload.register_db.db_idx = db_id; + req.payload.register_db.jobq_base = jobq_base; + req.payload.register_db.jobq_size = jobq_size; + req.payload.register_db.host_ssid = ctx_id; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_REGISTER_DB_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_err_ratelimited(vdev, "Failed to register doorbell %u: %d\n", db_id, ret); + + return ret; +} + +int ivpu_jsm_unregister_db(struct ivpu_device *vdev, u32 db_id) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_UNREGISTER_DB }; + struct vpu_jsm_msg resp; + int ret = 0; + + req.payload.unregister_db.db_idx = db_id; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_UNREGISTER_DB_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to unregister doorbell %u: %d\n", db_id, ret); + + return ret; +} + +int ivpu_jsm_get_heartbeat(struct ivpu_device *vdev, u32 engine, u64 *heartbeat) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB }; + struct vpu_jsm_msg resp; + int ret; + + if (engine != VPU_ENGINE_COMPUTE) + return -EINVAL; + + req.payload.query_engine_hb.engine_idx = engine; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_err_ratelimited(vdev, "Failed to get heartbeat from engine %d: %d\n", + engine, ret); + return ret; + } + + *heartbeat = resp.payload.query_engine_hb_done.heartbeat; + return ret; +} + +int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine, struct vpu_jsm_msg *resp) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_ENGINE_RESET }; + int ret; + + if (engine != VPU_ENGINE_COMPUTE) + return -EINVAL; + + req.payload.engine_reset.engine_idx = engine; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_ENGINE_RESET_DONE, resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_err_ratelimited(vdev, "Failed to reset engine %d: %d\n", engine, ret); + ivpu_pm_trigger_recovery(vdev, "Engine reset failed"); + return ret; + } + + atomic_inc(&vdev->pm->engine_reset_counter); + + return 0; +} + +int ivpu_jsm_preempt_engine(struct ivpu_device *vdev, u32 engine, u32 preempt_id) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_ENGINE_PREEMPT }; + struct vpu_jsm_msg resp; + int ret; + + if (engine != VPU_ENGINE_COMPUTE) + return -EINVAL; + + req.payload.engine_preempt.engine_idx = engine; + req.payload.engine_preempt.preempt_id = preempt_id; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_ENGINE_PREEMPT_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_err_ratelimited(vdev, "Failed to preempt engine %d: %d\n", engine, ret); + + return ret; +} + +int ivpu_jsm_dyndbg_control(struct ivpu_device *vdev, char *command, size_t size) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DYNDBG_CONTROL }; + struct vpu_jsm_msg resp; + int ret; + + strscpy(req.payload.dyndbg_control.dyndbg_cmd, command, VPU_DYNDBG_CMD_MAX_LEN); + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_DYNDBG_CONTROL_RSP, &resp, + VPU_IPC_CHAN_GEN_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to send command \"%s\": ret %d\n", + command, ret); + + return ret; +} + +int ivpu_jsm_trace_get_capability(struct ivpu_device *vdev, u32 *trace_destination_mask, + u64 *trace_hw_component_mask) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_TRACE_GET_CAPABILITY }; + struct vpu_jsm_msg resp; + int ret; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_TRACE_GET_CAPABILITY_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_warn_ratelimited(vdev, "Failed to get trace capability: %d\n", ret); + return ret; + } + + *trace_destination_mask = resp.payload.trace_capability.trace_destination_mask; + *trace_hw_component_mask = resp.payload.trace_capability.trace_hw_component_mask; + + return ret; +} + +int ivpu_jsm_trace_set_config(struct ivpu_device *vdev, u32 trace_level, u32 trace_destination_mask, + u64 trace_hw_component_mask) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_TRACE_SET_CONFIG }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.trace_config.trace_level = trace_level; + req.payload.trace_config.trace_destination_mask = trace_destination_mask; + req.payload.trace_config.trace_hw_component_mask = trace_hw_component_mask; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_TRACE_SET_CONFIG_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to set config: %d\n", ret); + + return ret; +} + +int ivpu_jsm_context_release(struct ivpu_device *vdev, u32 host_ssid) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_SSID_RELEASE }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.ssid_release.host_ssid = host_ssid; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_SSID_RELEASE_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to release context: %d\n", ret); + + return ret; +} + +int ivpu_jsm_pwr_d0i3_enter(struct ivpu_device *vdev) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_PWR_D0I3_ENTER }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.pwr_d0i3_enter.send_response = 1; + + ret = ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_PWR_D0I3_ENTER_DONE, &resp, + VPU_IPC_CHAN_GEN_CMD, vdev->timeout.d0i3_entry_msg); + if (ret) + return ret; + + return ivpu_hw_wait_for_idle(vdev); +} + +int ivpu_jsm_hws_create_cmdq(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_group, u32 cmdq_id, + u32 pid, u32 engine, u64 cmdq_base, u32 cmdq_size) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_CREATE_CMD_QUEUE }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.hws_create_cmdq.host_ssid = ctx_id; + req.payload.hws_create_cmdq.process_id = pid; + req.payload.hws_create_cmdq.engine_idx = engine; + req.payload.hws_create_cmdq.cmdq_group = cmdq_group; + req.payload.hws_create_cmdq.cmdq_id = cmdq_id; + req.payload.hws_create_cmdq.cmdq_base = cmdq_base; + req.payload.hws_create_cmdq.cmdq_size = cmdq_size; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to create command queue: %d\n", ret); + + return ret; +} + +int ivpu_jsm_hws_destroy_cmdq(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DESTROY_CMD_QUEUE }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.hws_destroy_cmdq.host_ssid = ctx_id; + req.payload.hws_destroy_cmdq.cmdq_id = cmdq_id; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_DESTROY_CMD_QUEUE_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to destroy command queue: %d\n", ret); + + return ret; +} + +int ivpu_jsm_hws_register_db(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id, u32 db_id, + u64 cmdq_base, u32 cmdq_size) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_HWS_REGISTER_DB }; + struct vpu_jsm_msg resp; + int ret = 0; + + req.payload.hws_register_db.db_id = db_id; + req.payload.hws_register_db.host_ssid = ctx_id; + req.payload.hws_register_db.cmdq_id = cmdq_id; + req.payload.hws_register_db.cmdq_base = cmdq_base; + req.payload.hws_register_db.cmdq_size = cmdq_size; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_REGISTER_DB_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_err_ratelimited(vdev, "Failed to register doorbell %u: %d\n", db_id, ret); + + return ret; +} + +int ivpu_jsm_hws_resume_engine(struct ivpu_device *vdev, u32 engine) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_HWS_ENGINE_RESUME }; + struct vpu_jsm_msg resp; + int ret; + + if (engine != VPU_ENGINE_COMPUTE) + return -EINVAL; + + req.payload.hws_resume_engine.engine_idx = engine; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_err_ratelimited(vdev, "Failed to resume engine %d: %d\n", engine, ret); + ivpu_pm_trigger_recovery(vdev, "Engine resume failed"); + } + + return ret; +} + +int ivpu_jsm_hws_set_context_sched_properties(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id, + u32 priority) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.hws_set_context_sched_properties.host_ssid = ctx_id; + req.payload.hws_set_context_sched_properties.cmdq_id = cmdq_id; + req.payload.hws_set_context_sched_properties.priority_band = priority; + req.payload.hws_set_context_sched_properties.realtime_priority_level = 0; + req.payload.hws_set_context_sched_properties.in_process_priority = 0; + req.payload.hws_set_context_sched_properties.context_quantum = 20000; + req.payload.hws_set_context_sched_properties.grace_period_same_priority = 10000; + req.payload.hws_set_context_sched_properties.grace_period_lower_priority = 0; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to set context sched properties: %d\n", ret); + + return ret; +} + +int ivpu_jsm_hws_set_scheduling_log(struct ivpu_device *vdev, u32 engine_idx, u32 host_ssid, + u64 vpu_log_buffer_va) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.hws_set_scheduling_log.engine_idx = engine_idx; + req.payload.hws_set_scheduling_log.host_ssid = host_ssid; + req.payload.hws_set_scheduling_log.vpu_log_buffer_va = vpu_log_buffer_va; + req.payload.hws_set_scheduling_log.notify_index = 0; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to set scheduling log: %d\n", ret); + + return ret; +} + +int ivpu_jsm_hws_setup_priority_bands(struct ivpu_device *vdev) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP }; + struct vpu_jsm_msg resp; + struct ivpu_hw_info *hw = vdev->hw; + struct vpu_ipc_msg_payload_hws_priority_band_setup *setup = + &req.payload.hws_priority_band_setup; + int ret; + + for (int band = VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE; + band < VPU_JOB_SCHEDULING_PRIORITY_BAND_COUNT; band++) { + setup->grace_period[band] = hw->hws.grace_period[band]; + setup->process_grace_period[band] = hw->hws.process_grace_period[band]; + setup->process_quantum[band] = hw->hws.process_quantum[band]; + } + setup->normal_band_percentage = 10; + + ret = ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP_RSP, + &resp, VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to set priority bands: %d\n", ret); + + return ret; +} + +int ivpu_jsm_metric_streamer_start(struct ivpu_device *vdev, u64 metric_group_mask, + u64 sampling_rate, u64 buffer_addr, u64 buffer_size) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_METRIC_STREAMER_START }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.metric_streamer_start.metric_group_mask = metric_group_mask; + req.payload.metric_streamer_start.sampling_rate = sampling_rate; + req.payload.metric_streamer_start.buffer_addr = buffer_addr; + req.payload.metric_streamer_start.buffer_size = buffer_size; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_METRIC_STREAMER_START_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_warn_ratelimited(vdev, "Failed to start metric streamer: ret %d\n", ret); + return ret; + } + + return ret; +} + +int ivpu_jsm_metric_streamer_stop(struct ivpu_device *vdev, u64 metric_group_mask) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_METRIC_STREAMER_STOP }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.metric_streamer_stop.metric_group_mask = metric_group_mask; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_METRIC_STREAMER_STOP_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to stop metric streamer: ret %d\n", ret); + + return ret; +} + +int ivpu_jsm_metric_streamer_update(struct ivpu_device *vdev, u64 metric_group_mask, + u64 buffer_addr, u64 buffer_size, u64 *bytes_written) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_METRIC_STREAMER_UPDATE }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.metric_streamer_update.metric_group_mask = metric_group_mask; + req.payload.metric_streamer_update.buffer_addr = buffer_addr; + req.payload.metric_streamer_update.buffer_size = buffer_size; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_METRIC_STREAMER_UPDATE_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_warn_ratelimited(vdev, "Failed to update metric streamer: ret %d\n", ret); + return ret; + } + + if (buffer_size && resp.payload.metric_streamer_done.bytes_written > buffer_size) { + ivpu_warn_ratelimited(vdev, "MS buffer overflow: bytes_written %#llx > buffer_size %#llx\n", + resp.payload.metric_streamer_done.bytes_written, buffer_size); + return -EOVERFLOW; + } + + *bytes_written = resp.payload.metric_streamer_done.bytes_written; + + return ret; +} + +int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mask, u64 buffer_addr, + u64 buffer_size, u32 *sample_size, u64 *info_size) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_METRIC_STREAMER_INFO }; + struct vpu_jsm_msg resp; + int ret; + + req.payload.metric_streamer_start.metric_group_mask = metric_group_mask; + req.payload.metric_streamer_start.buffer_addr = buffer_addr; + req.payload.metric_streamer_start.buffer_size = buffer_size; + + ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_METRIC_STREAMER_INFO_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); + if (ret) { + ivpu_warn_ratelimited(vdev, "Failed to get metric streamer info: ret %d\n", ret); + return ret; + } + + if (!resp.payload.metric_streamer_done.sample_size) { + ivpu_warn_ratelimited(vdev, "Invalid sample size\n"); + return -EBADMSG; + } + + if (sample_size) + *sample_size = resp.payload.metric_streamer_done.sample_size; + if (info_size) + *info_size = resp.payload.metric_streamer_done.bytes_written; + + return ret; +} + +int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DCT_ENABLE }; + struct vpu_jsm_msg resp; + + req.payload.pwr_dct_control.dct_active_us = active_us; + req.payload.pwr_dct_control.dct_inactive_us = inactive_us; + + return ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_DCT_ENABLE_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); +} + +int ivpu_jsm_dct_disable(struct ivpu_device *vdev) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_DCT_DISABLE }; + struct vpu_jsm_msg resp; + + return ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_DCT_DISABLE_DONE, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); +} + +int ivpu_jsm_state_dump(struct ivpu_device *vdev) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_STATE_DUMP }; + struct vpu_jsm_msg resp; + + return ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_STATE_DUMP_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); +} + +int ivpu_jsm_state_dump_no_reply(struct ivpu_device *vdev) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_STATE_DUMP }; + + return ivpu_ipc_send_and_wait(vdev, &req, VPU_IPC_CHAN_ASYNC_CMD, + vdev->timeout.state_dump_msg); +} + +int ivpu_jsm_msg_freq_config(struct ivpu_device *vdev, u16 min_ratio, u16 pn_ratio, u16 max_ratio) +{ + struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_FREQ_CONFIG}; + struct vpu_jsm_msg resp; + + req.payload.freq_config.min_freq_pll_ratio = min_ratio; + req.payload.freq_config.pn_freq_pll_ratio = pn_ratio; + req.payload.freq_config.max_freq_pll_ratio = max_ratio; + + return ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_FREQ_CONFIG_RSP, &resp, + VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm); +} diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.h b/drivers/accel/ivpu/ivpu_jsm_msg.h new file mode 100644 index 000000000..28ffe0742 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_jsm_msg.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#ifndef __IVPU_JSM_MSG_H__ +#define __IVPU_JSM_MSG_H__ + +#include "vpu_jsm_api.h" + +const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type); + +int ivpu_jsm_register_db(struct ivpu_device *vdev, u32 ctx_id, u32 db_id, + u64 jobq_base, u32 jobq_size); +int ivpu_jsm_unregister_db(struct ivpu_device *vdev, u32 db_id); +int ivpu_jsm_get_heartbeat(struct ivpu_device *vdev, u32 engine, u64 *heartbeat); +int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine, struct vpu_jsm_msg *response); +int ivpu_jsm_preempt_engine(struct ivpu_device *vdev, u32 engine, u32 preempt_id); +int ivpu_jsm_dyndbg_control(struct ivpu_device *vdev, char *command, size_t size); +int ivpu_jsm_trace_get_capability(struct ivpu_device *vdev, u32 *trace_destination_mask, + u64 *trace_hw_component_mask); +int ivpu_jsm_trace_set_config(struct ivpu_device *vdev, u32 trace_level, u32 trace_destination_mask, + u64 trace_hw_component_mask); +int ivpu_jsm_context_release(struct ivpu_device *vdev, u32 host_ssid); +int ivpu_jsm_pwr_d0i3_enter(struct ivpu_device *vdev); +int ivpu_jsm_hws_create_cmdq(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_group, u32 cmdq_id, + u32 pid, u32 engine, u64 cmdq_base, u32 cmdq_size); +int ivpu_jsm_hws_destroy_cmdq(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id); +int ivpu_jsm_hws_register_db(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id, u32 db_id, + u64 cmdq_base, u32 cmdq_size); +int ivpu_jsm_hws_resume_engine(struct ivpu_device *vdev, u32 engine); +int ivpu_jsm_hws_set_context_sched_properties(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id, + u32 priority); +int ivpu_jsm_hws_set_scheduling_log(struct ivpu_device *vdev, u32 engine_idx, u32 host_ssid, + u64 vpu_log_buffer_va); +int ivpu_jsm_hws_setup_priority_bands(struct ivpu_device *vdev); +int ivpu_jsm_metric_streamer_start(struct ivpu_device *vdev, u64 metric_group_mask, + u64 sampling_rate, u64 buffer_addr, u64 buffer_size); +int ivpu_jsm_metric_streamer_stop(struct ivpu_device *vdev, u64 metric_group_mask); +int ivpu_jsm_metric_streamer_update(struct ivpu_device *vdev, u64 metric_group_mask, + u64 buffer_addr, u64 buffer_size, u64 *bytes_written); +int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mask, u64 buffer_addr, + u64 buffer_size, u32 *sample_size, u64 *info_size); +int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us); +int ivpu_jsm_dct_disable(struct ivpu_device *vdev); +int ivpu_jsm_state_dump(struct ivpu_device *vdev); +int ivpu_jsm_state_dump_no_reply(struct ivpu_device *vdev); +int ivpu_jsm_msg_freq_config(struct ivpu_device *vdev, u16 min_ratio, u16 pn_ratio, u16 max_ratio); + +#endif diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c new file mode 100644 index 000000000..41efd8985 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_mmu.c @@ -0,0 +1,1030 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_hw.h" +#include "ivpu_hw_reg_io.h" +#include "ivpu_mmu.h" +#include "ivpu_mmu_context.h" +#include "ivpu_pm.h" + +#define IVPU_MMU_REG_IDR0 0x00200000u +#define IVPU_MMU_REG_IDR1 0x00200004u +#define IVPU_MMU_REG_IDR3 0x0020000cu +#define IVPU_MMU_REG_IDR5 0x00200014u +#define IVPU_MMU_REG_CR0 0x00200020u +#define IVPU_MMU_REG_CR0ACK 0x00200024u +#define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0) +#define IVPU_MMU_REG_CR0_ATSCHK_MASK BIT(4) +#define IVPU_MMU_REG_CR0_CMDQEN_MASK BIT(3) +#define IVPU_MMU_REG_CR0_EVTQEN_MASK BIT(2) +#define IVPU_MMU_REG_CR0_PRIQEN_MASK BIT(1) +#define IVPU_MMU_REG_CR0_SMMUEN_MASK BIT(0) + +#define IVPU_MMU_REG_CR1 0x00200028u +#define IVPU_MMU_REG_CR2 0x0020002cu +#define IVPU_MMU_REG_IRQ_CTRL 0x00200050u +#define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u +#define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0) + +#define IVPU_MMU_REG_GERROR 0x00200060u +#define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0) +#define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2) +#define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3) +#define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4) +#define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5) +#define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6) +#define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7) + +#define IVPU_MMU_REG_GERRORN 0x00200064u + +#define IVPU_MMU_REG_STRTAB_BASE 0x00200080u +#define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u +#define IVPU_MMU_REG_CMDQ_BASE 0x00200090u +#define IVPU_MMU_REG_CMDQ_PROD 0x00200098u +#define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu +#define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0) +#define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24) +#define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u +#define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u +#define IVPU_MMU_REG_EVTQ_CONS 0x002000acu +#define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K) +#define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K) + +#define IVPU_MMU_IDR0_REF 0x080f3e0f +#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f +#define IVPU_MMU_IDR1_REF 0x0e739d18 +#define IVPU_MMU_IDR3_REF 0x0000003c +#define IVPU_MMU_IDR5_REF 0x00040070 +#define IVPU_MMU_IDR5_REF_SIMICS 0x00000075 +#define IVPU_MMU_IDR5_REF_FPGA 0x00800075 + +#define IVPU_MMU_CDTAB_ENT_SIZE 64 +#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */ +#define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2) + +#define IVPU_MMU_STREAM_ID0 0 +#define IVPU_MMU_STREAM_ID3 3 + +#define IVPU_MMU_STRTAB_ENT_SIZE 64 +#define IVPU_MMU_STRTAB_ENT_COUNT 4 +#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2 +#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE + +#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */ +#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2) +#define IVPU_MMU_Q_WRAP_MASK GENMASK(IVPU_MMU_Q_COUNT_LOG2, 0) +#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1) +#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK) +#define IVPU_MMU_Q_WRP(val) ((val) & IVPU_MMU_Q_COUNT) + +#define IVPU_MMU_CMDQ_CMD_SIZE 16 +#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE) + +#define IVPU_MMU_EVTQ_CMD_SIZE 32 +#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE) + +#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0) + +#define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12) +#define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22) +#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) +#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) +#define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32) + +#define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10) +#define IVPU_MMU_CMD_CFGI_0_SSV BIT(11) +#define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12) +#define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32) +#define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0) + +#define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48) +#define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32) + +#define CMD_PREFETCH_CFG 0x1 +#define CMD_CFGI_STE 0x3 +#define CMD_CFGI_ALL 0x4 +#define CMD_CFGI_CD 0x5 +#define CMD_CFGI_CD_ALL 0x6 +#define CMD_TLBI_NH_ASID 0x11 +#define CMD_TLBI_EL2_ALL 0x20 +#define CMD_TLBI_NSNH_ALL 0x30 +#define CMD_SYNC 0x46 + +#define IVPU_MMU_EVT_F_UUT 0x01 +#define IVPU_MMU_EVT_C_BAD_STREAMID 0x02 +#define IVPU_MMU_EVT_F_STE_FETCH 0x03 +#define IVPU_MMU_EVT_C_BAD_STE 0x04 +#define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05 +#define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06 +#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07 +#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08 +#define IVPU_MMU_EVT_F_CD_FETCH 0x09 +#define IVPU_MMU_EVT_C_BAD_CD 0x0a +#define IVPU_MMU_EVT_F_WALK_EABT 0x0b +#define IVPU_MMU_EVT_F_TRANSLATION 0x10 +#define IVPU_MMU_EVT_F_ADDR_SIZE 0x11 +#define IVPU_MMU_EVT_F_ACCESS 0x12 +#define IVPU_MMU_EVT_F_PERMISSION 0x13 +#define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20 +#define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21 +#define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24 +#define IVPU_MMU_EVT_F_VMS_FETCH 0x25 + +#define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0) +#define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12) + +#define IVPU_MMU_Q_BASE_RWA BIT(62) +#define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) +#define IVPU_MMU_STRTAB_BASE_RA BIT(62) +#define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) + +#define IVPU_MMU_IRQ_EVTQ_EN BIT(2) +#define IVPU_MMU_IRQ_GERROR_EN BIT(0) + +#define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10) +#define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8) +#define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6) +#define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4) +#define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2) +#define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0) +#define IVPU_MMU_CACHE_NC 0 +#define IVPU_MMU_CACHE_WB 1 +#define IVPU_MMU_CACHE_WT 2 +#define IVPU_MMU_SH_NSH 0 +#define IVPU_MMU_SH_OSH 2 +#define IVPU_MMU_SH_ISH 3 + +#define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0) + +#define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) +#define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6) +#define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) +#define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) +#define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12) +#define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14) +#define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30) +#define IVPU_MMU_CD_0_ENDI BIT(15) +#define IVPU_MMU_CD_0_V BIT(31) +#define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32) +#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38) +#define IVPU_MMU_CD_0_AA64 BIT(41) +#define IVPU_MMU_CD_0_S BIT(44) +#define IVPU_MMU_CD_0_R BIT(45) +#define IVPU_MMU_CD_0_A BIT(46) +#define IVPU_MMU_CD_0_ASET BIT(47) +#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) + +#define IVPU_MMU_T0SZ_48BIT 16 +#define IVPU_MMU_T0SZ_38BIT 26 + +#define IVPU_MMU_IPS_48BIT 5 +#define IVPU_MMU_IPS_44BIT 4 +#define IVPU_MMU_IPS_42BIT 3 +#define IVPU_MMU_IPS_40BIT 2 +#define IVPU_MMU_IPS_36BIT 1 +#define IVPU_MMU_IPS_32BIT 0 + +#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) + +#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) +#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4) +#define IVPU_MMU_STE_0_S1FMT_LINEAR 0 +#define IVPU_MMU_STE_DWORDS 8 +#define IVPU_MMU_STE_0_CFG_S1_TRANS 5 +#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1) +#define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) +#define IVPU_MMU_STE_0_V BIT(0) + +#define IVPU_MMU_STE_1_STRW_NSEL1 0ul +#define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13) +#define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30) +#define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48) +#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul +#define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50) +#define IVPU_MMU_STE_1_INSTCFG_DATA 2ul +#define IVPU_MMU_STE_1_MEV BIT(19) +#define IVPU_MMU_STE_1_S1STALLD BIT(27) +#define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul +#define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul +#define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul +#define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul +#define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2) +#define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4) +#define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6) +#define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0) +#define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0 + +#define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC) +#define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC) + +#define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \ + (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \ + (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \ + (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \ + (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \ + (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \ + (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT))) + +#define IVPU_MMU_CERROR_NONE 0x0 +#define IVPU_MMU_CERROR_ILL 0x1 +#define IVPU_MMU_CERROR_ABT 0x2 +#define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3 + +static const char *ivpu_mmu_event_to_str(u32 cmd) +{ + switch (cmd) { + case IVPU_MMU_EVT_F_UUT: + return "Unsupported Upstream Transaction"; + case IVPU_MMU_EVT_C_BAD_STREAMID: + return "Transaction StreamID out of range"; + case IVPU_MMU_EVT_F_STE_FETCH: + return "Fetch of STE caused external abort"; + case IVPU_MMU_EVT_C_BAD_STE: + return "Used STE invalid"; + case IVPU_MMU_EVT_F_BAD_ATS_TREQ: + return "Address Request disallowed for a StreamID"; + case IVPU_MMU_EVT_F_STREAM_DISABLED: + return "Transaction marks non-substream disabled"; + case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN: + return "MMU bypass is disallowed for this StreamID"; + case IVPU_MMU_EVT_C_BAD_SUBSTREAMID: + return "Invalid StreamID"; + case IVPU_MMU_EVT_F_CD_FETCH: + return "Fetch of CD caused external abort"; + case IVPU_MMU_EVT_C_BAD_CD: + return "Fetched CD invalid"; + case IVPU_MMU_EVT_F_WALK_EABT: + return " An external abort occurred fetching a TLB"; + case IVPU_MMU_EVT_F_TRANSLATION: + return "Translation fault"; + case IVPU_MMU_EVT_F_ADDR_SIZE: + return " Output address caused address size fault"; + case IVPU_MMU_EVT_F_ACCESS: + return "Access flag fault"; + case IVPU_MMU_EVT_F_PERMISSION: + return "Permission fault occurred on page access"; + case IVPU_MMU_EVT_F_TLB_CONFLICT: + return "A TLB conflict"; + case IVPU_MMU_EVT_F_CFG_CONFLICT: + return "A configuration cache conflict"; + case IVPU_MMU_EVT_E_PAGE_REQUEST: + return "Page request hint from a client device"; + case IVPU_MMU_EVT_F_VMS_FETCH: + return "Fetch of VMS caused external abort"; + default: + return "Unknown event"; + } +} + +static const char *ivpu_mmu_cmdq_err_to_str(u32 err) +{ + switch (err) { + case IVPU_MMU_CERROR_NONE: + return "No error"; + case IVPU_MMU_CERROR_ILL: + return "Illegal command"; + case IVPU_MMU_CERROR_ABT: + return "External abort on command queue read"; + case IVPU_MMU_CERROR_ATC_INV_SYNC: + return "Sync failed to complete ATS invalidation"; + default: + return "Unknown error"; + } +} + +static void ivpu_mmu_config_check(struct ivpu_device *vdev) +{ + u32 val_ref; + u32 val; + + if (ivpu_is_simics(vdev)) + val_ref = IVPU_MMU_IDR0_REF_SIMICS; + else + val_ref = IVPU_MMU_IDR0_REF; + + val = REGV_RD32(IVPU_MMU_REG_IDR0); + if (val != val_ref) + ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref); + + val = REGV_RD32(IVPU_MMU_REG_IDR1); + if (val != IVPU_MMU_IDR1_REF) + ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF); + + val = REGV_RD32(IVPU_MMU_REG_IDR3); + if (val != IVPU_MMU_IDR3_REF) + ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF); + + if (ivpu_is_simics(vdev)) + val_ref = IVPU_MMU_IDR5_REF_SIMICS; + else if (ivpu_is_fpga(vdev)) + val_ref = IVPU_MMU_IDR5_REF_FPGA; + else + val_ref = IVPU_MMU_IDR5_REF; + + val = REGV_RD32(IVPU_MMU_REG_IDR5); + if (val != val_ref) + ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref); +} + +static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; + size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE; + + cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL); + if (!cdtab->base) + return -ENOMEM; + + ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size); + + return 0; +} + +static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_strtab *strtab = &mmu->strtab; + size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE; + + strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL); + if (!strtab->base) + return -ENOMEM; + + strtab->base_cfg = IVPU_MMU_STRTAB_CFG; + strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA; + strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK; + + ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n", + &strtab->dma, &strtab->dma_q, size); + + return 0; +} + +static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_queue *q = &mmu->cmdq; + + q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL); + if (!q->base) + return -ENOMEM; + + q->dma_q = IVPU_MMU_Q_BASE_RWA; + q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; + q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; + + ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n", + &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE); + + return 0; +} + +static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_queue *q = &mmu->evtq; + + q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL); + if (!q->base) + return -ENOMEM; + + q->dma_q = IVPU_MMU_Q_BASE_RWA; + q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; + q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; + + ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n", + &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE); + + return 0; +} + +static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) +{ + int ret; + + ret = ivpu_mmu_cdtab_alloc(vdev); + if (ret) { + ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret); + return ret; + } + + ret = ivpu_mmu_strtab_alloc(vdev); + if (ret) { + ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret); + return ret; + } + + ret = ivpu_mmu_cmdq_alloc(vdev); + if (ret) { + ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret); + return ret; + } + + ret = ivpu_mmu_evtq_alloc(vdev); + if (ret) + ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret); + + return ret; +} + +static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val) +{ + REGV_WR32(IVPU_MMU_REG_CR0, val); + + return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); +} + +static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val) +{ + REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val); + + return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); +} + +static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) +{ + u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN; + int ret; + + ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0); + if (ret) + return ret; + + return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl); +} + +static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) +{ + struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; + int ret; + + ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod, + IVPU_MMU_QUEUE_TIMEOUT_US); + if (ret) + return ret; + + cmdq->cons = cmdq->prod; + + return 0; +} + +static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q) +{ + return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) && + (IVPU_MMU_Q_WRP(q->prod) != IVPU_MMU_Q_WRP(q->cons))); +} + +static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q) +{ + return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) && + (IVPU_MMU_Q_WRP(q->prod) == IVPU_MMU_Q_WRP(q->cons))); +} + +static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) +{ + struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; + u64 *queue_buffer = cmdq->base; + int idx = IVPU_MMU_Q_IDX(cmdq->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer)); + + if (ivpu_mmu_queue_is_full(cmdq)) { + ivpu_err(vdev, "Failed to write MMU CMD %s\n", name); + return -EBUSY; + } + + queue_buffer[idx] = data0; + queue_buffer[idx + 1] = data1; + cmdq->prod = (cmdq->prod + 1) & IVPU_MMU_Q_WRAP_MASK; + + ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1); + + return 0; +} + +static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) +{ + struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; + u64 val; + int ret; + + val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC); + + ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0); + if (ret) + return ret; + + if (!ivpu_is_force_snoop_enabled(vdev)) + clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE); + REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod); + + ret = ivpu_mmu_cmdq_wait_for_cons(vdev); + if (ret) { + u32 err; + + val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS); + err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val); + + ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret, + ivpu_mmu_cmdq_err_to_str(err)); + ivpu_hw_diagnose_failure(vdev); + } + + return ret; +} + +static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) +{ + u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL); + u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f); + + return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1); +} + +static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) +{ + u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) | + FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid); + + return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0); +} + +static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) +{ + u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL); + + return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0); +} + +static int ivpu_mmu_reset(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + u32 val; + int ret; + + memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE); + if (!ivpu_is_force_snoop_enabled(vdev)) + clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE); + mmu->cmdq.prod = 0; + mmu->cmdq.cons = 0; + + memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE); + mmu->evtq.prod = 0; + mmu->evtq.cons = 0; + + ret = ivpu_mmu_reg_write_cr0(vdev, 0); + if (ret) + return ret; + + val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) | + FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) | + FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) | + FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) | + FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) | + FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB); + REGV_WR32(IVPU_MMU_REG_CR1, val); + + REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q); + REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg); + + REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q); + REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0); + REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0); + + val = REG_SET_FLD(IVPU_MMU_REG_CR0, CMDQEN, 0); + ret = ivpu_mmu_reg_write_cr0(vdev, val); + if (ret) + return ret; + + ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); + if (ret) + return ret; + + ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); + if (ret) + return ret; + + ret = ivpu_mmu_cmdq_sync(vdev); + if (ret) + return ret; + + REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q); + REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0); + REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0); + + val = REG_SET_FLD(IVPU_MMU_REG_CR0, EVTQEN, val); + ret = ivpu_mmu_reg_write_cr0(vdev, val); + if (ret) + return ret; + + val = REG_SET_FLD(IVPU_MMU_REG_CR0, ATSCHK, val); + ret = ivpu_mmu_reg_write_cr0(vdev, val); + if (ret) + return ret; + + ret = ivpu_mmu_irqs_setup(vdev); + if (ret) + return ret; + + val = REG_SET_FLD(IVPU_MMU_REG_CR0, SMMUEN, val); + return ivpu_mmu_reg_write_cr0(vdev, val); +} + +static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_strtab *strtab = &mmu->strtab; + struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; + u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE); + u64 str[2]; + + str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) | + FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) | + FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) | + IVPU_MMU_STE_0_V | + (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK); + + str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) | + FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) | + FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) | + FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) | + FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) | + FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) | + FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) | + FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) | + IVPU_MMU_STE_1_MEV | + IVPU_MMU_STE_1_S1STALLD; + + WRITE_ONCE(entry[1], str[1]); + WRITE_ONCE(entry[0], str[0]); + + if (!ivpu_is_force_snoop_enabled(vdev)) + clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE); + + ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]); +} + +static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) +{ + ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); + ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); + + return 0; +} + +int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + int ret = 0; + + mutex_lock(&mmu->lock); + if (!mmu->on) + goto unlock; + + ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); + if (ret) + goto unlock; + + ret = ivpu_mmu_cmdq_sync(vdev); +unlock: + mutex_unlock(&mmu->lock); + return ret; +} + +static int ivpu_mmu_cdtab_entry_set(struct ivpu_device *vdev, u32 ssid, u64 cd_dma, bool valid) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; + u64 *entry; + u64 cd[4]; + int ret = 0; + + if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) + return -EINVAL; + + entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); + drm_WARN_ON(&vdev->drm, (entry[0] & IVPU_MMU_CD_0_V) == valid); + + cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) | + FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | + FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | + FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | + FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | + FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) | + FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | + IVPU_MMU_CD_0_TCR_EPD1 | + IVPU_MMU_CD_0_AA64 | + IVPU_MMU_CD_0_R | + IVPU_MMU_CD_0_ASET; + cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK; + cd[2] = 0; + cd[3] = 0x0000000000007444; + + /* For global and reserved contexts generate memory fault on VPU */ + if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID || ssid == IVPU_RESERVED_CONTEXT_MMU_SSID) + cd[0] |= IVPU_MMU_CD_0_A; + + if (valid) + cd[0] |= IVPU_MMU_CD_0_V; + + WRITE_ONCE(entry[1], cd[1]); + WRITE_ONCE(entry[2], cd[2]); + WRITE_ONCE(entry[3], cd[3]); + WRITE_ONCE(entry[0], cd[0]); + + if (!ivpu_is_force_snoop_enabled(vdev)) + clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE); + + ivpu_dbg(vdev, MMU, "CDTAB set %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n", + valid ? "valid" : "invalid", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]); + + mutex_lock(&mmu->lock); + if (!mmu->on) + goto unlock; + + ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); + if (ret) + goto err_invalidate; + + ret = ivpu_mmu_cmdq_sync(vdev); + if (ret) + goto err_invalidate; +unlock: + mutex_unlock(&mmu->lock); + return 0; + +err_invalidate: + WRITE_ONCE(entry[0], 0); + mutex_unlock(&mmu->lock); + return ret; +} + +int ivpu_mmu_init(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + int ret; + + ivpu_dbg(vdev, MMU, "Init..\n"); + + ivpu_mmu_config_check(vdev); + + ret = drmm_mutex_init(&vdev->drm, &mmu->lock); + if (ret) + return ret; + + ret = ivpu_mmu_structs_alloc(vdev); + if (ret) + return ret; + + ret = ivpu_mmu_strtab_init(vdev); + if (ret) { + ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); + return ret; + } + + ret = ivpu_mmu_enable(vdev); + if (ret) { + ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); + return ret; + } + + ivpu_dbg(vdev, MMU, "Init done\n"); + + return 0; +} + +int ivpu_mmu_enable(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + int ret; + + mutex_lock(&mmu->lock); + + mmu->on = true; + + ret = ivpu_mmu_reset(vdev); + if (ret) { + ivpu_err(vdev, "Failed to reset MMU: %d\n", ret); + goto err; + } + + ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); + if (ret) + goto err; + + ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); + if (ret) + goto err; + + ret = ivpu_mmu_cmdq_sync(vdev); + if (ret) + goto err; + + mutex_unlock(&mmu->lock); + + return 0; +err: + mmu->on = false; + mutex_unlock(&mmu->lock); + return ret; +} + +void ivpu_mmu_disable(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + + mutex_lock(&mmu->lock); + mmu->on = false; + mutex_unlock(&mmu->lock); +} + +static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) +{ + u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); + u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]); + u64 fetch_addr = ((u64)event[7]) << 32 | event[6]; + u64 in_addr = ((u64)event[5]) << 32 | event[4]; + u32 sid = event[1]; + + ivpu_err_ratelimited(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n", + op, ivpu_mmu_event_to_str(op), ssid, sid, + event[2], event[3], in_addr, fetch_addr); +} + +static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) +{ + struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; + u32 idx = IVPU_MMU_Q_IDX(evtq->cons); + u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE); + + evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); + if (ivpu_mmu_queue_is_empty(evtq)) + return NULL; + + evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK; + return evt; +} + +static int ivpu_mmu_evtq_set(struct ivpu_device *vdev, bool enable) +{ + u32 val = REGV_RD32(IVPU_MMU_REG_CR0); + + if (enable) + val = REG_SET_FLD(IVPU_MMU_REG_CR0, EVTQEN, val); + else + val = REG_CLR_FLD(IVPU_MMU_REG_CR0, EVTQEN, val); + REGV_WR32(IVPU_MMU_REG_CR0, val); + + return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); +} + +static int ivpu_mmu_evtq_enable(struct ivpu_device *vdev) +{ + return ivpu_mmu_evtq_set(vdev, true); +} + +static int ivpu_mmu_evtq_disable(struct ivpu_device *vdev) +{ + return ivpu_mmu_evtq_set(vdev, false); +} + +void ivpu_mmu_discard_events(struct ivpu_device *vdev) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + + mutex_lock(&mmu->lock); + /* + * Disable event queue (stop MMU from updating the producer) + * to allow synchronization of consumer and producer indexes + */ + ivpu_mmu_evtq_disable(vdev); + + vdev->mmu->evtq.cons = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); + REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons); + vdev->mmu->evtq.prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); + + ivpu_mmu_evtq_enable(vdev); + + drm_WARN_ON_ONCE(&vdev->drm, vdev->mmu->evtq.cons != vdev->mmu->evtq.prod); + + mutex_unlock(&mmu->lock); +} + +int ivpu_mmu_disable_ssid_events(struct ivpu_device *vdev, u32 ssid) +{ + struct ivpu_mmu_info *mmu = vdev->mmu; + struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; + u64 *entry; + u64 val; + + if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) + return -EINVAL; + + mutex_lock(&mmu->lock); + + entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); + + val = READ_ONCE(entry[0]); + val &= ~IVPU_MMU_CD_0_R; + WRITE_ONCE(entry[0], val); + + if (!ivpu_is_force_snoop_enabled(vdev)) + clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE); + + ivpu_mmu_cmdq_write_cfgi_all(vdev); + ivpu_mmu_cmdq_sync(vdev); + + mutex_unlock(&mmu->lock); + + return 0; +} + +void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) +{ + struct ivpu_file_priv *file_priv; + u32 *event; + u32 ssid; + + ivpu_dbg(vdev, IRQ, "MMU event queue\n"); + + while ((event = ivpu_mmu_get_event(vdev))) { + ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, *event); + if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID || + ssid == IVPU_RESERVED_CONTEXT_MMU_SSID) { + ivpu_mmu_dump_event(vdev, event); + ivpu_pm_trigger_recovery(vdev, "MMU event"); + return; + } + + file_priv = xa_load(&vdev->context_xa, ssid); + if (file_priv) { + if (!READ_ONCE(file_priv->has_mmu_faults)) { + atomic_set(&vdev->faults_detected, 1); + ivpu_mmu_dump_event(vdev, event); + WRITE_ONCE(file_priv->has_mmu_faults, true); + } + } + } + + queue_work(system_percpu_wq, &vdev->context_abort_work); +} + +void ivpu_mmu_evtq_dump(struct ivpu_device *vdev) +{ + u32 *event; + + while ((event = ivpu_mmu_get_event(vdev)) != NULL) + ivpu_mmu_dump_event(vdev, event); +} + +void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) +{ + u32 gerror_val, gerrorn_val, active; + + ivpu_dbg(vdev, IRQ, "MMU error\n"); + + gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR); + gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN); + + active = gerror_val ^ gerrorn_val; + if (!(active & IVPU_MMU_GERROR_ERR_MASK)) + return; + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active)) + ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n"); + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active)) + ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n"); + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active)) + ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n"); + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active)) + ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n"); + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active)) + ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n"); + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active)) + ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n"); + + if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active)) + ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n"); + + REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val); +} + +int ivpu_mmu_cd_set(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) +{ + return ivpu_mmu_cdtab_entry_set(vdev, ssid, pgtable->pgd_dma, true); +} + +void ivpu_mmu_cd_clear(struct ivpu_device *vdev, int ssid) +{ + ivpu_mmu_cdtab_entry_set(vdev, ssid, 0, false); +} diff --git a/drivers/accel/ivpu/ivpu_mmu.h b/drivers/accel/ivpu/ivpu_mmu.h new file mode 100644 index 000000000..1ce752974 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_mmu.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_MMU_H__ +#define __IVPU_MMU_H__ + +struct ivpu_device; + +struct ivpu_mmu_cdtab { + void *base; + dma_addr_t dma; +}; + +struct ivpu_mmu_strtab { + void *base; + dma_addr_t dma; + u64 dma_q; + u32 base_cfg; +}; + +struct ivpu_mmu_queue { + void *base; + dma_addr_t dma; + u64 dma_q; + u32 prod; + u32 cons; +}; + +struct ivpu_mmu_info { + struct mutex lock; /* Protects cdtab, strtab, cmdq, on */ + struct ivpu_mmu_cdtab cdtab; + struct ivpu_mmu_strtab strtab; + struct ivpu_mmu_queue cmdq; + struct ivpu_mmu_queue evtq; + bool on; +}; + +int ivpu_mmu_init(struct ivpu_device *vdev); +void ivpu_mmu_disable(struct ivpu_device *vdev); +int ivpu_mmu_enable(struct ivpu_device *vdev); +int ivpu_mmu_cd_set(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable); +void ivpu_mmu_cd_clear(struct ivpu_device *vdev, int ssid); +int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid); + +void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev); +void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev); +void ivpu_mmu_evtq_dump(struct ivpu_device *vdev); +void ivpu_mmu_discard_events(struct ivpu_device *vdev); +int ivpu_mmu_disable_ssid_events(struct ivpu_device *vdev, u32 ssid); + +#endif /* __IVPU_MMU_H__ */ diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c new file mode 100644 index 000000000..c4014c83e --- /dev/null +++ b/drivers/accel/ivpu/ivpu_mmu_context.c @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#include +#include +#include +#include + +#include + +#include "ivpu_drv.h" +#include "ivpu_hw.h" +#include "ivpu_mmu.h" +#include "ivpu_mmu_context.h" + +#define IVPU_MMU_VPU_ADDRESS_MASK GENMASK(47, 12) +#define IVPU_MMU_PGD_INDEX_MASK GENMASK(47, 39) +#define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30) +#define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21) +#define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12) +#define IVPU_MMU_ENTRY_FLAGS_MASK (BIT(52) | GENMASK(11, 0)) +#define IVPU_MMU_ENTRY_FLAG_CONT BIT(52) +#define IVPU_MMU_ENTRY_FLAG_NG BIT(11) +#define IVPU_MMU_ENTRY_FLAG_AF BIT(10) +#define IVPU_MMU_ENTRY_FLAG_RO BIT(7) +#define IVPU_MMU_ENTRY_FLAG_USER BIT(6) +#define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2) +#define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1) +#define IVPU_MMU_ENTRY_FLAG_VALID BIT(0) + +#define IVPU_MMU_PAGE_SIZE SZ_4K +#define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16) +#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE) +#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE) +#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE) +#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE) +#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64)) + +#define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000 +#define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID) +#define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK) +#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \ + IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID) + +static void *ivpu_pgtable_alloc_page(struct ivpu_device *vdev, dma_addr_t *dma) +{ + dma_addr_t dma_addr; + struct page *page; + void *cpu; + + page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); + if (!page) + return NULL; + + set_pages_array_wc(&page, 1); + + dma_addr = dma_map_page(vdev->drm.dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(vdev->drm.dev, dma_addr)) + goto err_free_page; + + cpu = vmap(&page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL)); + if (!cpu) + goto err_dma_unmap_page; + + + *dma = dma_addr; + return cpu; + +err_dma_unmap_page: + dma_unmap_page(vdev->drm.dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + +err_free_page: + put_page(page); + return NULL; +} + +static void ivpu_pgtable_free_page(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr) +{ + struct page *page; + + if (cpu_addr) { + page = vmalloc_to_page(cpu_addr); + vunmap(cpu_addr); + dma_unmap_page(vdev->drm.dev, dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK, PAGE_SIZE, + DMA_BIDIRECTIONAL); + set_pages_array_wb(&page, 1); + put_page(page); + } +} + +static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) +{ + int pgd_idx, pud_idx, pmd_idx; + dma_addr_t pud_dma, pmd_dma, pte_dma; + u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr; + + for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) { + pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; + pud_dma = pgtable->pgd_dma_ptr[pgd_idx]; + + if (!pud_dma_ptr) + continue; + + for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) { + pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; + pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx]; + + if (!pmd_dma_ptr) + continue; + + for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) { + pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx]; + pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx]; + + ivpu_pgtable_free_page(vdev, pte_dma_ptr, pte_dma); + } + + kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]); + ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma); + } + + kfree(pgtable->pmd_ptrs[pgd_idx]); + kfree(pgtable->pte_ptrs[pgd_idx]); + ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma); + } + + ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma); + pgtable->pgd_dma_ptr = NULL; + pgtable->pgd_dma = 0; +} + +static u64* +ivpu_mmu_ensure_pgd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) +{ + u64 *pgd_dma_ptr = pgtable->pgd_dma_ptr; + dma_addr_t pgd_dma; + + if (pgd_dma_ptr) + return pgd_dma_ptr; + + pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pgd_dma); + if (!pgd_dma_ptr) + return NULL; + + pgtable->pgd_dma_ptr = pgd_dma_ptr; + pgtable->pgd_dma = pgd_dma; + + return pgd_dma_ptr; +} + +static u64* +ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx) +{ + u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; + dma_addr_t pud_dma; + + if (pud_dma_ptr) + return pud_dma_ptr; + + pud_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pud_dma); + if (!pud_dma_ptr) + return NULL; + + drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]); + pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pgtable->pmd_ptrs[pgd_idx]) + goto err_free_pud_dma_ptr; + + drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]); + pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pgtable->pte_ptrs[pgd_idx]) + goto err_free_pmd_ptrs; + + pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr; + pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID; + + return pud_dma_ptr; + +err_free_pmd_ptrs: + kfree(pgtable->pmd_ptrs[pgd_idx]); + +err_free_pud_dma_ptr: + ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma); + return NULL; +} + +static u64* +ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx, + int pud_idx) +{ + u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; + dma_addr_t pmd_dma; + + if (pmd_dma_ptr) + return pmd_dma_ptr; + + pmd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pmd_dma); + if (!pmd_dma_ptr) + return NULL; + + drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]); + pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); + if (!pgtable->pte_ptrs[pgd_idx][pud_idx]) + goto err_free_pmd_dma_ptr; + + pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr; + pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID; + + return pmd_dma_ptr; + +err_free_pmd_dma_ptr: + ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma); + return NULL; +} + +static u64* +ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, + int pgd_idx, int pud_idx, int pmd_idx) +{ + u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx]; + dma_addr_t pte_dma; + + if (pte_dma_ptr) + return pte_dma_ptr; + + pte_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pte_dma); + if (!pte_dma_ptr) + return NULL; + + pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr; + pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID; + + return pte_dma_ptr; +} + +static int +ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr, dma_addr_t dma_addr, u64 prot) +{ + u64 *pte; + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + + drm_WARN_ON(&vdev->drm, ctx->id == IVPU_RESERVED_CONTEXT_MMU_SSID); + + /* Allocate PGD - first level page table if needed */ + if (!ivpu_mmu_ensure_pgd(vdev, &ctx->pgtable)) + return -ENOMEM; + + /* Allocate PUD - second level page table if needed */ + if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx)) + return -ENOMEM; + + /* Allocate PMD - third level page table if needed */ + if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx)) + return -ENOMEM; + + /* Allocate PTE - fourth level page table if needed */ + pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx); + if (!pte) + return -ENOMEM; + + /* Update PTE */ + pte[pte_idx] = dma_addr | prot; + + return 0; +} + +static int +ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, + dma_addr_t dma_addr, u64 prot) +{ + size_t size = IVPU_MMU_CONT_PAGES_SIZE; + + drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size)); + drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size)); + + prot |= IVPU_MMU_ENTRY_FLAG_CONT; + + while (size) { + int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); + + if (ret) + return ret; + + size -= IVPU_MMU_PAGE_SIZE; + vpu_addr += IVPU_MMU_PAGE_SIZE; + dma_addr += IVPU_MMU_PAGE_SIZE; + } + + return 0; +} + +static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr) +{ + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + + /* Update PTE with dummy physical address and clear flags */ + ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID; +} + +static int +ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot) +{ + int map_size; + int ret; + + while (size) { + if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE && + IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) { + ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot); + map_size = IVPU_MMU_CONT_PAGES_SIZE; + } else { + ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); + map_size = IVPU_MMU_PAGE_SIZE; + } + + if (ret) + return ret; + + vpu_addr += map_size; + dma_addr += map_size; + size -= map_size; + } + + return 0; +} + +static void ivpu_mmu_context_set_page_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr) +{ + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + + ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] |= IVPU_MMU_ENTRY_FLAG_RO; +} + +static void ivpu_mmu_context_split_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr) +{ + int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); + int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); + int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); + int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); + + ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] &= ~IVPU_MMU_ENTRY_FLAG_CONT; +} + +static void ivpu_mmu_context_split_64k_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr) +{ + u64 start = ALIGN_DOWN(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE); + u64 end = ALIGN(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE); + u64 offset = 0; + + ivpu_dbg(vdev, MMU_MAP, "Split 64K page ctx: %u vpu_addr: 0x%llx\n", ctx->id, vpu_addr); + + while (start + offset < end) { + ivpu_mmu_context_split_page(vdev, ctx, start + offset); + offset += IVPU_MMU_PAGE_SIZE; + } +} + +int +ivpu_mmu_context_set_pages_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, + size_t size) +{ + u64 end = vpu_addr + size; + size_t size_left = size; + int ret; + + if (size == 0) + return 0; + + if (drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr | size, IVPU_MMU_PAGE_SIZE))) + return -EINVAL; + + mutex_lock(&ctx->lock); + + ivpu_dbg(vdev, MMU_MAP, "Set read-only pages ctx: %u vpu_addr: 0x%llx size: %lu\n", + ctx->id, vpu_addr, size); + + if (!ivpu_disable_mmu_cont_pages) { + /* Split 64K contiguous page at the beginning if needed */ + if (!IS_ALIGNED(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE)) + ivpu_mmu_context_split_64k_page(vdev, ctx, vpu_addr); + + /* Split 64K contiguous page at the end if needed */ + if (!IS_ALIGNED(vpu_addr + size, IVPU_MMU_CONT_PAGES_SIZE)) + ivpu_mmu_context_split_64k_page(vdev, ctx, vpu_addr + size); + } + + while (size_left) { + if (vpu_addr < end) + ivpu_mmu_context_set_page_ro(vdev, ctx, vpu_addr); + + vpu_addr += IVPU_MMU_PAGE_SIZE; + size_left -= IVPU_MMU_PAGE_SIZE; + } + + /* Ensure page table modifications are flushed from wc buffers to memory */ + wmb(); + + mutex_unlock(&ctx->lock); + ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); + if (ret) + ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret); + + return 0; +} + +static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size) +{ + while (size) { + ivpu_mmu_context_unmap_page(ctx, vpu_addr); + vpu_addr += IVPU_MMU_PAGE_SIZE; + size -= IVPU_MMU_PAGE_SIZE; + } +} + +int +ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, + struct sg_table *sgt, size_t bo_size, bool llc_coherent, bool read_only) +{ + size_t start_vpu_addr = vpu_addr; + struct scatterlist *sg; + size_t sgt_size = 0; + int ret; + u64 prot; + u64 i; + + if (drm_WARN_ON(&vdev->drm, !ctx)) + return -EINVAL; + + if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE)) + return -EINVAL; + + if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK) + return -EINVAL; + + prot = IVPU_MMU_ENTRY_MAPPED; + if (llc_coherent) + prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT; + if (read_only) + prot |= IVPU_MMU_ENTRY_FLAG_RO; + + mutex_lock(&ctx->lock); + + for_each_sgtable_dma_sg(sgt, sg, i) { + dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset; + size_t size = sg_dma_len(sg) + sg->offset; + + ivpu_dbg(vdev, MMU_MAP, "Map ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n", + ctx->id, dma_addr, vpu_addr, size); + + if (sgt_size + size > bo_size) { + ivpu_err(vdev, "Scatter-gather table size exceeds buffer object size\n"); + ret = -EINVAL; + goto err_unmap_pages; + } + + ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot); + if (ret) { + ivpu_err(vdev, "Failed to map context pages\n"); + goto err_unmap_pages; + } + vpu_addr += size; + sgt_size += size; + } + + if (sgt_size < bo_size) { + ivpu_err(vdev, "Scatter-gather table size too small to cover buffer object size\n"); + ret = -EINVAL; + goto err_unmap_pages; + } + + if (!ctx->is_cd_valid) { + ret = ivpu_mmu_cd_set(vdev, ctx->id, &ctx->pgtable); + if (ret) { + ivpu_err(vdev, "Failed to set context descriptor for context %u: %d\n", + ctx->id, ret); + goto err_unmap_pages; + } + ctx->is_cd_valid = true; + } + + /* Ensure page table modifications are flushed from wc buffers to memory */ + wmb(); + + ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); + if (ret) { + ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret); + goto err_unmap_pages; + } + + mutex_unlock(&ctx->lock); + return 0; + +err_unmap_pages: + ivpu_mmu_context_unmap_pages(ctx, start_vpu_addr, sgt_size); + mutex_unlock(&ctx->lock); + return ret; +} + +void +ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr, struct sg_table *sgt) +{ + struct scatterlist *sg; + int ret; + u64 i; + + if (drm_WARN_ON(&vdev->drm, !ctx)) + return; + + mutex_lock(&ctx->lock); + + for_each_sgtable_dma_sg(sgt, sg, i) { + dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset; + size_t size = sg_dma_len(sg) + sg->offset; + + ivpu_dbg(vdev, MMU_MAP, "Unmap ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n", + ctx->id, dma_addr, vpu_addr, size); + + ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size); + vpu_addr += size; + } + + /* Ensure page table modifications are flushed from wc buffers to memory */ + wmb(); + + mutex_unlock(&ctx->lock); + + ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); + if (ret) + ivpu_warn_ratelimited(vdev, "Failed to invalidate TLB for ctx %u: %d\n", + ctx->id, ret); +} + +int +ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range, + u64 size, struct drm_mm_node *node) +{ + int ret; + + WARN_ON(!range); + + mutex_lock(&ctx->lock); + if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) { + ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0, + range->start, range->end, DRM_MM_INSERT_BEST); + if (!ret) + goto unlock; + } + + ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0, + range->start, range->end, DRM_MM_INSERT_BEST); +unlock: + mutex_unlock(&ctx->lock); + return ret; +} + +void +ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node) +{ + mutex_lock(&ctx->lock); + drm_mm_remove_node(node); + mutex_unlock(&ctx->lock); +} + +void ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id) +{ + u64 start, end; + + mutex_init(&ctx->lock); + + if (!context_id) { + start = vdev->hw->ranges.runtime.start; + end = vdev->hw->ranges.shave.end; + } else { + start = min_t(u64, vdev->hw->ranges.user.start, vdev->hw->ranges.shave.start); + end = max_t(u64, vdev->hw->ranges.user.end, vdev->hw->ranges.dma.end); + } + + drm_mm_init(&ctx->mm, start, end - start); + ctx->id = context_id; +} + +void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) +{ + if (ctx->is_cd_valid) { + ivpu_mmu_cd_clear(vdev, ctx->id); + ctx->is_cd_valid = false; + } + + mutex_destroy(&ctx->lock); + ivpu_mmu_pgtables_free(vdev, &ctx->pgtable); + drm_mm_takedown(&ctx->mm); +} + +void ivpu_mmu_global_context_init(struct ivpu_device *vdev) +{ + ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID); +} + +void ivpu_mmu_global_context_fini(struct ivpu_device *vdev) +{ + ivpu_mmu_context_fini(vdev, &vdev->gctx); +} + +int ivpu_mmu_reserved_context_init(struct ivpu_device *vdev) +{ + int ret; + + ivpu_mmu_context_init(vdev, &vdev->rctx, IVPU_RESERVED_CONTEXT_MMU_SSID); + + mutex_lock(&vdev->rctx.lock); + + if (!ivpu_mmu_ensure_pgd(vdev, &vdev->rctx.pgtable)) { + ivpu_err(vdev, "Failed to allocate root page table for reserved context\n"); + ret = -ENOMEM; + goto err_ctx_fini; + } + + ret = ivpu_mmu_cd_set(vdev, vdev->rctx.id, &vdev->rctx.pgtable); + if (ret) { + ivpu_err(vdev, "Failed to set context descriptor for reserved context\n"); + goto err_ctx_fini; + } + + mutex_unlock(&vdev->rctx.lock); + return ret; + +err_ctx_fini: + mutex_unlock(&vdev->rctx.lock); + ivpu_mmu_context_fini(vdev, &vdev->rctx); + return ret; +} + +void ivpu_mmu_reserved_context_fini(struct ivpu_device *vdev) +{ + ivpu_mmu_cd_clear(vdev, vdev->rctx.id); + ivpu_mmu_context_fini(vdev, &vdev->rctx); +} diff --git a/drivers/accel/ivpu/ivpu_mmu_context.h b/drivers/accel/ivpu/ivpu_mmu_context.h new file mode 100644 index 000000000..cc02e7bab --- /dev/null +++ b/drivers/accel/ivpu/ivpu_mmu_context.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2023 Intel Corporation + */ + +#ifndef __IVPU_MMU_CONTEXT_H__ +#define __IVPU_MMU_CONTEXT_H__ + +#include + +struct ivpu_device; +struct ivpu_file_priv; +struct ivpu_addr_range; + +#define IVPU_MMU_PGTABLE_ENTRIES 512ull + +struct ivpu_mmu_pgtable { + u64 ***pte_ptrs[IVPU_MMU_PGTABLE_ENTRIES]; + u64 **pmd_ptrs[IVPU_MMU_PGTABLE_ENTRIES]; + u64 *pud_ptrs[IVPU_MMU_PGTABLE_ENTRIES]; + u64 *pgd_dma_ptr; + dma_addr_t pgd_dma; +}; + +struct ivpu_mmu_context { + struct mutex lock; /* Protects: mm, pgtable, is_cd_valid */ + struct drm_mm mm; + struct ivpu_mmu_pgtable pgtable; + bool is_cd_valid; + u32 id; +}; + +void ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id); +void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx); +void ivpu_mmu_global_context_init(struct ivpu_device *vdev); +void ivpu_mmu_global_context_fini(struct ivpu_device *vdev); +int ivpu_mmu_reserved_context_init(struct ivpu_device *vdev); +void ivpu_mmu_reserved_context_fini(struct ivpu_device *vdev); + +int ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range, + u64 size, struct drm_mm_node *node); +void ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node); + +int +ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, + struct sg_table *sgt, size_t bo_size, bool llc_coherent, bool read_only); +void ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr, struct sg_table *sgt); +int ivpu_mmu_context_set_pages_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, + u64 vpu_addr, size_t size); + +#endif /* __IVPU_MMU_CONTEXT_H__ */ diff --git a/drivers/accel/ivpu/ivpu_ms.c b/drivers/accel/ivpu/ivpu_ms.c new file mode 100644 index 000000000..cd176e77b --- /dev/null +++ b/drivers/accel/ivpu/ivpu_ms.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_gem.h" +#include "ivpu_hw.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_ms.h" +#include "ivpu_pm.h" + +#define MS_INFO_BUFFER_SIZE SZ_64K +#define MS_NUM_BUFFERS 2 +#define MS_READ_PERIOD_MULTIPLIER 2 +#define MS_MIN_SAMPLE_PERIOD_NS 1000000 + +static struct ivpu_ms_instance * +get_instance_by_mask(struct ivpu_file_priv *file_priv, u64 metric_mask) +{ + struct ivpu_ms_instance *ms; + + lockdep_assert_held(&file_priv->ms_lock); + + list_for_each_entry(ms, &file_priv->ms_instance_list, ms_instance_node) + if (ms->mask == metric_mask) + return ms; + + return NULL; +} + +int ivpu_ms_start_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct drm_ivpu_metric_streamer_start *args = data; + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_ms_instance *ms; + u32 sample_size; + u64 buf_size; + int ret; + + if (!args->metric_group_mask || !args->read_period_samples || + args->sampling_period_ns < MS_MIN_SAMPLE_PERIOD_NS) + return -EINVAL; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + mutex_lock(&file_priv->ms_lock); + + if (get_instance_by_mask(file_priv, args->metric_group_mask)) { + ivpu_dbg(vdev, IOCTL, "Instance already exists (mask %#llx)\n", + args->metric_group_mask); + ret = -EALREADY; + goto unlock; + } + + ms = kzalloc_obj(*ms); + if (!ms) { + ret = -ENOMEM; + goto unlock; + } + + ms->mask = args->metric_group_mask; + + ret = ivpu_jsm_metric_streamer_info(vdev, ms->mask, 0, 0, &sample_size, NULL); + if (ret) + goto err_free_ms; + + buf_size = PAGE_ALIGN((u64)args->read_period_samples * sample_size * + MS_READ_PERIOD_MULTIPLIER * MS_NUM_BUFFERS); + if (buf_size > ivpu_hw_range_size(&vdev->hw->ranges.global)) { + ivpu_dbg(vdev, IOCTL, "Requested MS buffer size %llu exceeds range size %llu\n", + buf_size, ivpu_hw_range_size(&vdev->hw->ranges.global)); + ret = -EINVAL; + goto err_free_ms; + } + + ms->bo = ivpu_bo_create_global(vdev, buf_size, DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE); + if (!ms->bo) { + ivpu_dbg(vdev, IOCTL, "Failed to allocate MS buffer (size %llu)\n", buf_size); + ret = -ENOMEM; + goto err_free_ms; + } + + ms->buff_size = ivpu_bo_size(ms->bo) / MS_NUM_BUFFERS; + ms->active_buff_vpu_addr = ms->bo->vpu_addr; + ms->inactive_buff_vpu_addr = ms->bo->vpu_addr + ms->buff_size; + ms->active_buff_ptr = ivpu_bo_vaddr(ms->bo); + ms->inactive_buff_ptr = ivpu_bo_vaddr(ms->bo) + ms->buff_size; + + ret = ivpu_jsm_metric_streamer_start(vdev, ms->mask, args->sampling_period_ns, + ms->active_buff_vpu_addr, ms->buff_size); + if (ret) + goto err_free_bo; + + args->sample_size = sample_size; + args->max_data_size = ivpu_bo_size(ms->bo); + list_add_tail(&ms->ms_instance_node, &file_priv->ms_instance_list); + goto unlock; + +err_free_bo: + ivpu_bo_free(ms->bo); +err_free_ms: + kfree(ms); +unlock: + mutex_unlock(&file_priv->ms_lock); + + ivpu_rpm_put(vdev); + return ret; +} + +static int +copy_leftover_bytes(struct ivpu_ms_instance *ms, + void __user *user_ptr, u64 user_size, u64 *user_bytes_copied) +{ + u64 copy_bytes; + + if (ms->leftover_bytes) { + copy_bytes = min(user_size - *user_bytes_copied, ms->leftover_bytes); + if (copy_to_user(user_ptr + *user_bytes_copied, ms->leftover_addr, copy_bytes)) + return -EFAULT; + + ms->leftover_bytes -= copy_bytes; + ms->leftover_addr += copy_bytes; + *user_bytes_copied += copy_bytes; + } + + return 0; +} + +static int +copy_samples_to_user(struct ivpu_device *vdev, struct ivpu_ms_instance *ms, + void __user *user_ptr, u64 user_size, u64 *user_bytes_copied) +{ + u64 bytes_written; + int ret; + + *user_bytes_copied = 0; + + ret = copy_leftover_bytes(ms, user_ptr, user_size, user_bytes_copied); + if (ret) + return ret; + + if (*user_bytes_copied == user_size) + return 0; + + ret = ivpu_jsm_metric_streamer_update(vdev, ms->mask, ms->inactive_buff_vpu_addr, + ms->buff_size, &bytes_written); + if (ret) + return ret; + + swap(ms->active_buff_vpu_addr, ms->inactive_buff_vpu_addr); + swap(ms->active_buff_ptr, ms->inactive_buff_ptr); + + ms->leftover_bytes = bytes_written; + ms->leftover_addr = ms->inactive_buff_ptr; + + return copy_leftover_bytes(ms, user_ptr, user_size, user_bytes_copied); +} + +int ivpu_ms_get_data_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_ivpu_metric_streamer_get_data *args = data; + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_ms_instance *ms; + u64 bytes_written; + int ret; + + if (!args->metric_group_mask) + return -EINVAL; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + mutex_lock(&file_priv->ms_lock); + + ms = get_instance_by_mask(file_priv, args->metric_group_mask); + if (!ms) { + ivpu_dbg(vdev, IOCTL, "Instance doesn't exist for mask: %#llx\n", + args->metric_group_mask); + ret = -EINVAL; + goto unlock; + } + + if (!args->buffer_size) { + ret = ivpu_jsm_metric_streamer_update(vdev, ms->mask, 0, 0, &bytes_written); + if (ret) + goto unlock; + args->data_size = bytes_written + ms->leftover_bytes; + goto unlock; + } + + if (!args->buffer_ptr) { + ret = -EINVAL; + goto unlock; + } + + ret = copy_samples_to_user(vdev, ms, u64_to_user_ptr(args->buffer_ptr), + args->buffer_size, &args->data_size); +unlock: + mutex_unlock(&file_priv->ms_lock); + + ivpu_rpm_put(vdev); + return ret; +} + +static void free_instance(struct ivpu_file_priv *file_priv, struct ivpu_ms_instance *ms) +{ + lockdep_assert_held(&file_priv->ms_lock); + + list_del(&ms->ms_instance_node); + ivpu_jsm_metric_streamer_stop(file_priv->vdev, ms->mask); + ivpu_bo_free(ms->bo); + kfree(ms); +} + +int ivpu_ms_stop_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct ivpu_file_priv *file_priv = file->driver_priv; + struct drm_ivpu_metric_streamer_stop *args = data; + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_ms_instance *ms; + int ret; + + if (!args->metric_group_mask) + return -EINVAL; + + ret = ivpu_rpm_get(vdev); + if (ret < 0) + return ret; + + mutex_lock(&file_priv->ms_lock); + + ms = get_instance_by_mask(file_priv, args->metric_group_mask); + if (ms) + free_instance(file_priv, ms); + + mutex_unlock(&file_priv->ms_lock); + + ivpu_rpm_put(vdev); + return ms ? 0 : -EINVAL; +} + +static inline struct ivpu_bo *get_ms_info_bo(struct ivpu_file_priv *file_priv) +{ + lockdep_assert_held(&file_priv->ms_lock); + + if (file_priv->ms_info_bo) + return file_priv->ms_info_bo; + + file_priv->ms_info_bo = ivpu_bo_create_global(file_priv->vdev, MS_INFO_BUFFER_SIZE, + DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE); + return file_priv->ms_info_bo; +} + +int ivpu_ms_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_ivpu_metric_streamer_get_data *args = data; + struct ivpu_file_priv *file_priv = file->driver_priv; + struct ivpu_device *vdev = file_priv->vdev; + struct ivpu_bo *bo; + u64 info_size; + int ret; + + if (!args->metric_group_mask) + return -EINVAL; + + if (!args->buffer_size) + return ivpu_jsm_metric_streamer_info(vdev, args->metric_group_mask, + 0, 0, NULL, &args->data_size); + if (!args->buffer_ptr) + return -EINVAL; + + mutex_lock(&file_priv->ms_lock); + + bo = get_ms_info_bo(file_priv); + if (!bo) { + ret = -ENOMEM; + goto unlock; + } + + ret = ivpu_jsm_metric_streamer_info(vdev, args->metric_group_mask, bo->vpu_addr, + ivpu_bo_size(bo), NULL, &info_size); + if (ret) + goto unlock; + + if (info_size > ivpu_bo_size(bo)) { + ivpu_warn_ratelimited(vdev, "MS info overflow: %#llx > %#zx\n", + info_size, ivpu_bo_size(bo)); + ret = -EOVERFLOW; + goto unlock; + } + + if (args->buffer_size < info_size) { + ret = -ENOSPC; + goto unlock; + } + + if (copy_to_user(u64_to_user_ptr(args->buffer_ptr), ivpu_bo_vaddr(bo), info_size)) + ret = -EFAULT; + + args->data_size = info_size; +unlock: + mutex_unlock(&file_priv->ms_lock); + + return ret; +} + +void ivpu_ms_cleanup(struct ivpu_file_priv *file_priv) +{ + struct ivpu_ms_instance *ms, *tmp; + struct ivpu_device *vdev = file_priv->vdev; + + pm_runtime_get_sync(vdev->drm.dev); + + mutex_lock(&file_priv->ms_lock); + + if (file_priv->ms_info_bo) { + ivpu_bo_free(file_priv->ms_info_bo); + file_priv->ms_info_bo = NULL; + } + + list_for_each_entry_safe(ms, tmp, &file_priv->ms_instance_list, ms_instance_node) + free_instance(file_priv, ms); + + mutex_unlock(&file_priv->ms_lock); + + pm_runtime_put_autosuspend(vdev->drm.dev); +} + +void ivpu_ms_cleanup_all(struct ivpu_device *vdev) +{ + struct ivpu_file_priv *file_priv; + unsigned long ctx_id; + + mutex_lock(&vdev->context_list_lock); + + xa_for_each(&vdev->context_xa, ctx_id, file_priv) + ivpu_ms_cleanup(file_priv); + + mutex_unlock(&vdev->context_list_lock); +} diff --git a/drivers/accel/ivpu/ivpu_ms.h b/drivers/accel/ivpu/ivpu_ms.h new file mode 100644 index 000000000..fbd5ebebc --- /dev/null +++ b/drivers/accel/ivpu/ivpu_ms.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ +#ifndef __IVPU_MS_H__ +#define __IVPU_MS_H__ + +#include + +struct drm_device; +struct drm_file; +struct ivpu_bo; +struct ivpu_device; +struct ivpu_file_priv; + +struct ivpu_ms_instance { + struct ivpu_bo *bo; + struct list_head ms_instance_node; + u64 mask; + u64 buff_size; + u64 active_buff_vpu_addr; + u64 inactive_buff_vpu_addr; + void *active_buff_ptr; + void *inactive_buff_ptr; + u64 leftover_bytes; + void *leftover_addr; +}; + +int ivpu_ms_start_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_ms_stop_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_ms_get_data_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int ivpu_ms_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +void ivpu_ms_cleanup(struct ivpu_file_priv *file_priv); +void ivpu_ms_cleanup_all(struct ivpu_device *vdev); + +#endif /* __IVPU_MS_H__ */ diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c new file mode 100644 index 000000000..c1ce83297 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2026 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include "ivpu_coredump.h" +#include "ivpu_drv.h" +#include "ivpu_fw.h" +#include "ivpu_fw_log.h" +#include "ivpu_hw.h" +#include "ivpu_ipc.h" +#include "ivpu_job.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_mmu.h" +#include "ivpu_ms.h" +#include "ivpu_pm.h" +#include "ivpu_trace.h" +#include "vpu_boot_api.h" + +static bool ivpu_disable_recovery; +#if IS_ENABLED(CONFIG_DRM_ACCEL_IVPU_DEBUG) +module_param_named_unsafe(disable_recovery, ivpu_disable_recovery, bool, 0644); +MODULE_PARM_DESC(disable_recovery, "Disables recovery when NPU hang is detected"); +#endif + +static unsigned long ivpu_tdr_timeout_ms; +module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, ulong, 0644); +MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default"); + +static unsigned long ivpu_inference_timeout_ms; +module_param_named(inference_timeout_ms, ivpu_inference_timeout_ms, ulong, 0644); +MODULE_PARM_DESC(inference_timeout_ms, "Inference maximum duration, in milliseconds, 0 - default"); + +#define PM_RESCHEDULE_LIMIT 5 + +static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev) +{ + struct ivpu_fw_info *fw = vdev->fw; + + ivpu_cmdq_reset_all_contexts(vdev); + ivpu_ipc_reset(vdev); + ivpu_fw_log_reset(vdev); + ivpu_fw_load(vdev); + fw->last_heartbeat = 0; + + ivpu_dbg(vdev, FW_BOOT, "Cold boot entry point 0x%llx", vdev->fw->cold_boot_entry_point); + fw->next_boot_mode = VPU_BOOT_TYPE_COLDBOOT; +} + +static void ivpu_pm_prepare_warm_boot(struct ivpu_device *vdev) +{ + struct ivpu_fw_info *fw = vdev->fw; + struct vpu_boot_params *bp = ivpu_bo_vaddr(fw->mem_bp); + + fw->warm_boot_entry_point = bp->save_restore_ret_address; + if (!fw->warm_boot_entry_point) { + ivpu_pm_prepare_cold_boot(vdev); + return; + } + + ivpu_dbg(vdev, FW_BOOT, "Warm boot entry point 0x%llx", fw->warm_boot_entry_point); + fw->next_boot_mode = VPU_BOOT_TYPE_WARMBOOT; +} + +static int ivpu_suspend(struct ivpu_device *vdev) +{ + int ret; + + ivpu_prepare_for_reset(vdev); + + ret = ivpu_shutdown(vdev); + if (ret) + ivpu_err(vdev, "Failed to shutdown NPU: %d\n", ret); + + return ret; +} + +static int ivpu_resume(struct ivpu_device *vdev) +{ + int ret; + +retry: + pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D0); + pci_restore_state(to_pci_dev(vdev->drm.dev)); + + ret = ivpu_hw_power_up(vdev); + if (ret) { + ivpu_err(vdev, "Failed to power up HW: %d\n", ret); + goto err_power_down; + } + + ret = ivpu_mmu_enable(vdev); + if (ret) { + ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); + goto err_power_down; + } + + ret = ivpu_boot(vdev); + if (ret) + goto err_mmu_disable; + + return 0; + +err_mmu_disable: + ivpu_mmu_disable(vdev); +err_power_down: + ivpu_hw_power_down(vdev); + pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot); + + if (ivpu_fw_is_warm_boot(vdev)) { + ivpu_pm_prepare_cold_boot(vdev); + goto retry; + } else { + ivpu_err(vdev, "Failed to resume the FW: %d\n", ret); + } + + return ret; +} + +static void ivpu_pm_reset_begin(struct ivpu_device *vdev) +{ + pm_runtime_disable(vdev->drm.dev); + + atomic_inc(&vdev->pm->reset_counter); + atomic_set(&vdev->pm->reset_pending, 1); + down_write(&vdev->pm->reset_lock); +} + +static void ivpu_pm_reset_complete(struct ivpu_device *vdev) +{ + int ret; + + ivpu_pm_prepare_cold_boot(vdev); + ivpu_jobs_abort_all(vdev); + ivpu_ms_cleanup_all(vdev); + + ret = ivpu_resume(vdev); + if (ret) { + ivpu_err(vdev, "Failed to resume NPU: %d\n", ret); + pm_runtime_set_suspended(vdev->drm.dev); + } else { + pm_runtime_set_active(vdev->drm.dev); + } + + up_write(&vdev->pm->reset_lock); + atomic_set(&vdev->pm->reset_pending, 0); + + pm_runtime_mark_last_busy(vdev->drm.dev); + pm_runtime_enable(vdev->drm.dev); +} + +static void ivpu_pm_recovery_work(struct work_struct *work) +{ + struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, recovery_work); + struct ivpu_device *vdev = pm->vdev; + char *evt[2] = {"IVPU_PM_EVENT=IVPU_RECOVER", NULL}; + + ivpu_err(vdev, "Recovering the NPU (reset #%d)\n", atomic_read(&vdev->pm->reset_counter)); + + ivpu_pm_reset_begin(vdev); + + if (!pm_runtime_status_suspended(vdev->drm.dev)) { + ivpu_jsm_state_dump_no_reply(vdev); + ivpu_dev_coredump(vdev); + ivpu_suspend(vdev); + } + + ivpu_pm_reset_complete(vdev); + + kobject_uevent_env(&vdev->drm.dev->kobj, KOBJ_CHANGE, evt); +} + +void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason) +{ + ivpu_err(vdev, "Recovery triggered by %s\n", reason); + + if (ivpu_disable_recovery) { + ivpu_err(vdev, "Recovery not available when disable_recovery param is set\n"); + return; + } + + /* Trigger recovery if it's not in progress */ + if (atomic_cmpxchg(&vdev->pm->reset_pending, 0, 1) == 0) { + ivpu_hw_diagnose_failure(vdev); + ivpu_hw_irq_disable(vdev); /* Disable IRQ early to protect from IRQ storm */ + queue_work(system_dfl_wq, &vdev->pm->recovery_work); + } +} + +static void ivpu_job_timeout_work(struct work_struct *work) +{ + struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, job_timeout_work.work); + struct ivpu_device *vdev = pm->vdev; + unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr; + unsigned long inference_timeout_ms = ivpu_inference_timeout_ms ? ivpu_inference_timeout_ms : + vdev->timeout.inference; + u64 inference_max_retries; + u64 heartbeat; + + if (ivpu_jsm_get_heartbeat(vdev, 0, &heartbeat) || heartbeat <= vdev->fw->last_heartbeat) { + ivpu_err(vdev, "Job timeout detected, heartbeat not progressed\n"); + goto abort; + } + + inference_max_retries = DIV_ROUND_UP(inference_timeout_ms, timeout_ms); + if (atomic_fetch_inc(&vdev->job_timeout_counter) >= inference_max_retries) { + ivpu_err(vdev, "Job timeout detected, heartbeat limit (%lld) exceeded\n", + inference_max_retries); + goto abort; + } + + vdev->fw->last_heartbeat = heartbeat; + ivpu_start_job_timeout_detection(vdev); + return; + +abort: + atomic_set(&vdev->job_timeout_counter, 0); + + if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_OS) { + ivpu_pm_trigger_recovery(vdev, "Job timeout"); + return; + } + + ivpu_jsm_state_dump(vdev); + ivpu_dev_coredump(vdev); + queue_work(system_percpu_wq, &vdev->context_abort_work); +} + +void ivpu_start_job_timeout_detection(struct ivpu_device *vdev) +{ + unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr; + + /* No-op if already queued */ + queue_delayed_work(system_percpu_wq, &vdev->pm->job_timeout_work, + msecs_to_jiffies(timeout_ms)); +} + +void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev) +{ + cancel_delayed_work_sync(&vdev->pm->job_timeout_work); + atomic_set(&vdev->job_timeout_counter, 0); +} + +int ivpu_pm_suspend_cb(struct device *dev) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + unsigned long timeout; + + trace_pm("suspend"); + ivpu_dbg(vdev, PM, "Suspend..\n"); + + timeout = jiffies + msecs_to_jiffies(vdev->timeout.tdr); + while (!ivpu_hw_is_idle(vdev)) { + cond_resched(); + if (time_after_eq(jiffies, timeout)) { + ivpu_err(vdev, "Failed to enter idle on system suspend\n"); + return -EBUSY; + } + } + + ivpu_jsm_pwr_d0i3_enter(vdev); + + ivpu_suspend(vdev); + ivpu_pm_prepare_warm_boot(vdev); + + ivpu_dbg(vdev, PM, "Suspend done.\n"); + trace_pm("suspend done"); + + return 0; +} + +int ivpu_pm_resume_cb(struct device *dev) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + int ret; + + trace_pm("resume"); + ivpu_dbg(vdev, PM, "Resume..\n"); + + ret = ivpu_resume(vdev); + if (ret) + ivpu_err(vdev, "Failed to resume: %d\n", ret); + + ivpu_dbg(vdev, PM, "Resume done.\n"); + trace_pm("resume done"); + + return ret; +} + +int ivpu_pm_runtime_suspend_cb(struct device *dev) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + int ret, ret_d0i3; + bool is_idle; + + drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa)); + drm_WARN_ON(&vdev->drm, work_pending(&vdev->pm->recovery_work)); + + trace_pm("runtime suspend"); + ivpu_dbg(vdev, PM, "Runtime suspend..\n"); + + ivpu_mmu_disable(vdev); + + is_idle = ivpu_hw_is_idle(vdev) || vdev->pm->dct_active_percent; + if (!is_idle) + ivpu_err(vdev, "NPU is not idle before autosuspend\n"); + + ret_d0i3 = ivpu_jsm_pwr_d0i3_enter(vdev); + if (ret_d0i3) + ivpu_err(vdev, "Failed to prepare for d0i3: %d\n", ret_d0i3); + + ret = ivpu_suspend(vdev); + if (ret) + ivpu_err(vdev, "Failed to suspend NPU: %d\n", ret); + + if (!is_idle || ret_d0i3) { + ivpu_err(vdev, "Forcing cold boot due to previous errors\n"); + atomic_inc(&vdev->pm->reset_counter); + ivpu_dev_coredump(vdev); + ivpu_pm_prepare_cold_boot(vdev); + } else { + ivpu_pm_prepare_warm_boot(vdev); + } + + ivpu_dbg(vdev, PM, "Runtime suspend done.\n"); + trace_pm("runtime suspend done"); + + return 0; +} + +int ivpu_pm_runtime_resume_cb(struct device *dev) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + int ret; + + trace_pm("runtime resume"); + ivpu_dbg(vdev, PM, "Runtime resume..\n"); + + ret = ivpu_resume(vdev); + if (ret) + ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret); + + ivpu_dbg(vdev, PM, "Runtime resume done.\n"); + trace_pm("runtime resume done"); + + return ret; +} + +int ivpu_rpm_get(struct ivpu_device *vdev) +{ + int ret; + + ret = pm_runtime_resume_and_get(vdev->drm.dev); + if (ret < 0) { + ivpu_err(vdev, "Failed to resume NPU: %d\n", ret); + pm_runtime_set_suspended(vdev->drm.dev); + } + + return ret; +} + +void ivpu_rpm_put(struct ivpu_device *vdev) +{ + pm_runtime_put_autosuspend(vdev->drm.dev); +} + +void ivpu_pm_reset_prepare_cb(struct pci_dev *pdev) +{ + struct ivpu_device *vdev = pci_get_drvdata(pdev); + + ivpu_dbg(vdev, PM, "Pre-reset..\n"); + + ivpu_pm_reset_begin(vdev); + + if (!pm_runtime_status_suspended(vdev->drm.dev)) { + ivpu_prepare_for_reset(vdev); + ivpu_hw_reset(vdev); + } + + ivpu_dbg(vdev, PM, "Pre-reset done.\n"); +} + +void ivpu_pm_reset_done_cb(struct pci_dev *pdev) +{ + struct ivpu_device *vdev = pci_get_drvdata(pdev); + + ivpu_dbg(vdev, PM, "Post-reset..\n"); + + ivpu_pm_reset_complete(vdev); + + ivpu_dbg(vdev, PM, "Post-reset done.\n"); +} + +void ivpu_pm_init(struct ivpu_device *vdev) +{ + struct device *dev = vdev->drm.dev; + struct ivpu_pm_info *pm = vdev->pm; + int delay; + + pm->vdev = vdev; + + init_rwsem(&pm->reset_lock); + atomic_set(&pm->reset_pending, 0); + atomic_set(&pm->reset_counter, 0); + atomic_set(&pm->engine_reset_counter, 0); + + INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work); + INIT_DELAYED_WORK(&pm->job_timeout_work, ivpu_job_timeout_work); + + if (ivpu_disable_recovery) + delay = -1; + else + delay = vdev->timeout.autosuspend; + + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, delay); + pm_runtime_set_active(dev); + + ivpu_dbg(vdev, PM, "Autosuspend delay = %d\n", delay); +} + +void ivpu_pm_disable_recovery(struct ivpu_device *vdev) +{ + drm_WARN_ON(&vdev->drm, delayed_work_pending(&vdev->pm->job_timeout_work)); + disable_work_sync(&vdev->pm->recovery_work); +} + +void ivpu_pm_enable(struct ivpu_device *vdev) +{ + struct device *dev = vdev->drm.dev; + + pm_runtime_allow(dev); + pm_runtime_put_autosuspend(dev); +} + +void ivpu_pm_disable(struct ivpu_device *vdev) +{ + pm_runtime_get_noresume(vdev->drm.dev); + pm_runtime_forbid(vdev->drm.dev); +} + +int ivpu_pm_dct_init(struct ivpu_device *vdev) +{ + if (vdev->pm->dct_active_percent) + return ivpu_pm_dct_enable(vdev, vdev->pm->dct_active_percent); + + return 0; +} + +int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent) +{ + u32 active_us, inactive_us; + int ret; + + if (active_percent == 0 || active_percent > 100) + return -EINVAL; + + active_us = (DCT_PERIOD_US * active_percent) / 100; + inactive_us = DCT_PERIOD_US - active_us; + + vdev->pm->dct_active_percent = active_percent; + + ivpu_dbg(vdev, PM, "DCT requested %u%% (D0: %uus, D0i2: %uus)\n", + active_percent, active_us, inactive_us); + + ret = ivpu_jsm_dct_enable(vdev, active_us, inactive_us); + if (ret) { + ivpu_err_ratelimited(vdev, "Failed to enable DCT: %d\n", ret); + return ret; + } + + return 0; +} + +int ivpu_pm_dct_disable(struct ivpu_device *vdev) +{ + int ret; + + vdev->pm->dct_active_percent = 0; + + ivpu_dbg(vdev, PM, "DCT requested to be disabled\n"); + + ret = ivpu_jsm_dct_disable(vdev); + if (ret) { + ivpu_err_ratelimited(vdev, "Failed to disable DCT: %d\n", ret); + return ret; + } + + return 0; +} + +void ivpu_pm_irq_dct_work_fn(struct work_struct *work) +{ + struct ivpu_device *vdev = container_of(work, struct ivpu_device, irq_dct_work); + bool enable; + int ret; + + if (ivpu_hw_btrs_dct_get_request(vdev, &enable)) + return; + + if (enable) + ret = ivpu_pm_dct_enable(vdev, DCT_DEFAULT_ACTIVE_PERCENT); + else + ret = ivpu_pm_dct_disable(vdev); + + if (!ret) { + /* Convert percent to U1.7 format */ + u8 val = DIV_ROUND_CLOSEST(vdev->pm->dct_active_percent * 128, 100); + + ivpu_hw_btrs_dct_set_status(vdev, enable, val); + } + +} diff --git a/drivers/accel/ivpu/ivpu_pm.h b/drivers/accel/ivpu/ivpu_pm.h new file mode 100644 index 000000000..2f07bb0b4 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_pm.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __IVPU_PM_H__ +#define __IVPU_PM_H__ + +#include +#include + +struct ivpu_device; + +struct ivpu_pm_info { + struct ivpu_device *vdev; + struct delayed_work job_timeout_work; + struct work_struct recovery_work; + struct rw_semaphore reset_lock; + atomic_t reset_counter; + atomic_t reset_pending; + atomic_t engine_reset_counter; + u8 dct_active_percent; +}; + +void ivpu_pm_init(struct ivpu_device *vdev); +void ivpu_pm_enable(struct ivpu_device *vdev); +void ivpu_pm_disable(struct ivpu_device *vdev); +void ivpu_pm_disable_recovery(struct ivpu_device *vdev); + +int ivpu_pm_suspend_cb(struct device *dev); +int ivpu_pm_resume_cb(struct device *dev); +int ivpu_pm_runtime_suspend_cb(struct device *dev); +int ivpu_pm_runtime_resume_cb(struct device *dev); + +void ivpu_pm_reset_prepare_cb(struct pci_dev *pdev); +void ivpu_pm_reset_done_cb(struct pci_dev *pdev); + +int __must_check ivpu_rpm_get(struct ivpu_device *vdev); +void ivpu_rpm_put(struct ivpu_device *vdev); + +void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason); +void ivpu_start_job_timeout_detection(struct ivpu_device *vdev); +void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev); + +int ivpu_pm_dct_init(struct ivpu_device *vdev); +int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent); +int ivpu_pm_dct_disable(struct ivpu_device *vdev); +void ivpu_pm_irq_dct_work_fn(struct work_struct *work); + +#endif /* __IVPU_PM_H__ */ diff --git a/drivers/accel/ivpu/ivpu_sysfs.c b/drivers/accel/ivpu/ivpu_sysfs.c new file mode 100644 index 000000000..95d201b3b --- /dev/null +++ b/drivers/accel/ivpu/ivpu_sysfs.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024-2026 Intel Corporation + */ + +#include +#include +#include +#include + +#include "ivpu_drv.h" +#include "ivpu_gem.h" +#include "ivpu_fw.h" +#include "ivpu_hw.h" +#include "ivpu_sysfs.h" + +/** + * DOC: npu_busy_time_us + * + * npu_busy_time_us is the time that the device spent executing jobs. + * The time is counted when and only when there are jobs submitted to firmware. + * + * This time can be used to measure the utilization of NPU, either by calculating + * npu_busy_time_us difference between two timepoints (i.e. measuring the time + * that the NPU was active during some workload) or monitoring utilization percentage + * by reading npu_busy_time_us periodically. + * + * When reading the value periodically, it shouldn't be read too often as it may have + * an impact on job submission performance. Recommended period is 1 second. + */ +static ssize_t +npu_busy_time_us_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + ktime_t total, now = 0; + + mutex_lock(&vdev->submitted_jobs_lock); + + total = vdev->busy_time; + if (!xa_empty(&vdev->submitted_jobs_xa)) + now = ktime_sub(ktime_get(), vdev->busy_start_ts); + mutex_unlock(&vdev->submitted_jobs_lock); + + return sysfs_emit(buf, "%lld\n", ktime_to_us(ktime_add(total, now))); +} + +static DEVICE_ATTR_RO(npu_busy_time_us); + +/** + * DOC: npu_memory_utilization + * + * The npu_memory_utilization is used to report in bytes a current NPU memory utilization. + * + */ +static ssize_t +npu_memory_utilization_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + struct ivpu_bo *bo; + u64 total_npu_memory = 0; + + mutex_lock(&vdev->bo_list_lock); + list_for_each_entry(bo, &vdev->bo_list, bo_list_node) + if (ivpu_bo_is_resident(bo)) + total_npu_memory += ivpu_bo_size(bo); + mutex_unlock(&vdev->bo_list_lock); + + return sysfs_emit(buf, "%lld\n", total_npu_memory); +} + +static DEVICE_ATTR_RO(npu_memory_utilization); + +/** + * DOC: sched_mode + * + * The sched_mode is used to report current NPU scheduling mode. + * + * It returns following strings: + * - "HW" - Hardware Scheduler mode + * - "OS" - Operating System Scheduler mode + * + */ +static ssize_t sched_mode_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + + return sysfs_emit(buf, "%s\n", vdev->fw->sched_mode ? "HW" : "OS"); +} + +static DEVICE_ATTR_RO(sched_mode); + +/** + * DOC: NPU frequency control and information + * + * Hardware frequency capabilities: + * freq/hw_min_freq - Minimum frequency supported by the NPU hardware. + * freq/hw_max_freq - Maximum frequency supported by the NPU hardware. + * freq/hw_efficient_freq - Most efficient operating frequency for the NPU. + * + * Configurable frequency limits (50XX devices and newer): + * freq/set_min_freq - Configured minimum operating frequency. + * freq/set_max_freq - Configured maximum operating frequency. + * + * Clamping behavior: Values written to set_min_freq and set_max_freq are + * clamped to hardware limits. If set_min_freq exceeds set_max_freq, the driver + * clamps set_min_freq to set_max_freq when selecting the operating frequency. + * + * Current operating frequency: + * freq/current_freq - Current frequency in MHz. Valid only when the device is + * active; returns 0 when idle. May be lower than the requested range due to + * power or thermal constraints. + * + * Legacy attributes (backward compatibility): + * npu_max_frequency_mhz - Alias for freq/hw_max_freq. + * npu_current_frequency_mhz - Alias for freq/current_freq. + */ + +static ssize_t hw_min_freq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.min_ratio); + + return sysfs_emit(buf, "%u\n", freq_mhz); +} + +static DEVICE_ATTR_RO(hw_min_freq); + +static ssize_t hw_efficient_freq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.pn_ratio); + + return sysfs_emit(buf, "%u\n", freq_mhz); +} + +static DEVICE_ATTR_RO(hw_efficient_freq); + +static ssize_t hw_max_freq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.max_ratio); + + return sysfs_emit(buf, "%u\n", freq_mhz); +} + +static DEVICE_ATTR_RO(hw_max_freq); + +static ssize_t set_min_freq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.cfg_min_ratio); + + return sysfs_emit(buf, "%u\n", freq_mhz); +} + +static ssize_t +set_min_freq_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz; + int ret; + + ret = kstrtou32(buf, 10, &freq_mhz); + if (ret) + return ret; + + ret = ivpu_hw_btrs_cfg_min_freq_set(vdev, freq_mhz); + if (ret) + return ret; + + return count; +} + +static DEVICE_ATTR_RW(set_min_freq); + +static ssize_t set_max_freq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.cfg_max_ratio); + + return sysfs_emit(buf, "%u\n", freq_mhz); +} + +static ssize_t +set_max_freq_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz; + int ret; + + ret = kstrtou32(buf, 10, &freq_mhz); + if (ret) + return ret; + + /* Convert MHz to Hz and set max frequency */ + ret = ivpu_hw_btrs_cfg_max_freq_set(vdev, freq_mhz); + if (ret) + return ret; + + return count; +} + +static DEVICE_ATTR_RW(set_max_freq); + +static ssize_t current_freq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *drm = dev_get_drvdata(dev); + struct ivpu_device *vdev = to_ivpu_device(drm); + u32 freq_mhz = 0; + + /* Read frequency only if device is active, otherwise frequency is 0 */ + if (pm_runtime_get_if_active(vdev->drm.dev) > 0) { + freq_mhz = ivpu_hw_btrs_current_freq_get(vdev); + + pm_runtime_put_autosuspend(vdev->drm.dev); + } + + return sysfs_emit(buf, "%u\n", freq_mhz); +} + +static DEVICE_ATTR_RO(current_freq); + +/* Alias to current_freq for legacy compat */ +static struct device_attribute dev_attr_npu_max_frequency_mhz = + __ATTR(npu_max_frequency_mhz, 0444, hw_max_freq_show, NULL); + +static struct device_attribute dev_attr_npu_current_frequency_mhz = + __ATTR(npu_current_frequency_mhz, 0444, current_freq_show, NULL); + +static struct attribute *ivpu_freq_attrs[] = { + &dev_attr_hw_min_freq.attr, + &dev_attr_hw_efficient_freq.attr, + &dev_attr_hw_max_freq.attr, + &dev_attr_current_freq.attr, + NULL, +}; + +static struct attribute_group ivpu_freq_attr_group = { + .name = "freq", + .attrs = ivpu_freq_attrs, +}; + +static struct attribute *ivpu_dev_attrs[] = { + &dev_attr_npu_busy_time_us.attr, + &dev_attr_npu_memory_utilization.attr, + &dev_attr_sched_mode.attr, + &dev_attr_npu_max_frequency_mhz.attr, + &dev_attr_npu_current_frequency_mhz.attr, + NULL, +}; + +static struct attribute_group ivpu_dev_attr_group = { + .attrs = ivpu_dev_attrs, +}; + +void ivpu_sysfs_init(struct ivpu_device *vdev) +{ + int ret; + + ret = devm_device_add_group(vdev->drm.dev, &ivpu_dev_attr_group); + if (ret) { + ivpu_warn(vdev, "Failed to add sysfs group to device, ret %d", ret); + return; + } + + ret = devm_device_add_group(vdev->drm.dev, &ivpu_freq_attr_group); + if (ret) { + ivpu_warn(vdev, "Failed to add sysfs freq group, ret %d", ret); + return; + } + + if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_50XX) { + ret = sysfs_add_file_to_group(&vdev->drm.dev->kobj, + &dev_attr_set_min_freq.attr, + "freq"); + if (ret) { + ivpu_warn(vdev, "Failed to add sysfs set_min_freq to device, ret %d", ret); + return; + } + + ret = sysfs_add_file_to_group(&vdev->drm.dev->kobj, + &dev_attr_set_max_freq.attr, + "freq"); + if (ret) { + ivpu_warn(vdev, "Failed to add sysfs set_max_freq to device, ret %d", ret); + return; + } + } +} diff --git a/drivers/accel/ivpu/ivpu_sysfs.h b/drivers/accel/ivpu/ivpu_sysfs.h new file mode 100644 index 000000000..9836f09b3 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_sysfs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Intel Corporation + */ + +#ifndef __IVPU_SYSFS_H__ +#define __IVPU_SYSFS_H__ + +#include "ivpu_drv.h" + +void ivpu_sysfs_init(struct ivpu_device *vdev); + +#endif /* __IVPU_SYSFS_H__ */ diff --git a/drivers/accel/ivpu/ivpu_trace.h b/drivers/accel/ivpu/ivpu_trace.h new file mode 100644 index 000000000..eb792038e --- /dev/null +++ b/drivers/accel/ivpu/ivpu_trace.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#if !defined(__IVPU_TRACE_H__) || defined(TRACE_HEADER_MULTI_READ) +#define __IVPU_TRACE_H__ + +#include +#include "ivpu_drv.h" +#include "ivpu_job.h" +#include "vpu_jsm_api.h" +#include "ivpu_jsm_msg.h" +#include "ivpu_ipc.h" + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM vpu +#define TRACE_INCLUDE_FILE ivpu_trace + +TRACE_EVENT(pm, + TP_PROTO(const char *event), + TP_ARGS(event), + TP_STRUCT__entry(__field(const char *, event)), + TP_fast_assign(__entry->event = event;), + TP_printk("%s", __entry->event) +); + +TRACE_EVENT(job, + TP_PROTO(const char *event, struct ivpu_job *job), + TP_ARGS(event, job), + TP_STRUCT__entry(__field(const char *, event) + __field(u32, ctx_id) + __field(u32, engine_id) + __field(u32, job_id) + ), + TP_fast_assign(__entry->event = event; + __entry->ctx_id = job->file_priv->ctx.id; + __entry->engine_id = job->engine_idx; + __entry->job_id = job->job_id;), + TP_printk("%s context:%d engine:%d job:%d", + __entry->event, + __entry->ctx_id, + __entry->engine_id, + __entry->job_id) +); + +TRACE_EVENT(jsm, + TP_PROTO(const char *event, struct vpu_jsm_msg *msg), + TP_ARGS(event, msg), + TP_STRUCT__entry(__field(const char *, event) + __field(const char *, type) + __field(enum vpu_ipc_msg_status, status) + __field(u32, request_id) + __field(u32, result) + ), + TP_fast_assign(__entry->event = event; + __entry->type = ivpu_jsm_msg_type_to_str(msg->type); + __entry->status = msg->status; + __entry->request_id = msg->request_id; + __entry->result = msg->result;), + TP_printk("%s type:%s, status:%#x, id:%#x, result:%#x", + __entry->event, + __entry->type, + __entry->status, + __entry->request_id, + __entry->result) +); + +#endif /* __IVPU_TRACE_H__ */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#include diff --git a/drivers/accel/ivpu/ivpu_trace_points.c b/drivers/accel/ivpu/ivpu_trace_points.c new file mode 100644 index 000000000..f8fb99de0 --- /dev/null +++ b/drivers/accel/ivpu/ivpu_trace_points.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2024 Intel Corporation + */ + +#ifndef __CHECKER__ +#define CREATE_TRACE_POINTS +#include "ivpu_trace.h" +#endif diff --git a/drivers/accel/ivpu/vpu_boot_api.h b/drivers/accel/ivpu/vpu_boot_api.h new file mode 100644 index 000000000..06c092d5e --- /dev/null +++ b/drivers/accel/ivpu/vpu_boot_api.h @@ -0,0 +1,450 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright (c) 2020-2025, Intel Corporation. + */ + +/** + * @addtogroup Boot + * @{ + */ + +/** + * @file + * @brief Boot API public header file. + */ + +#ifndef VPU_BOOT_API_H +#define VPU_BOOT_API_H + +/** + * The below values will be used to construct the version info this way: + * fw_bin_header->api_version[VPU_BOOT_API_VER_ID] = (VPU_BOOT_API_VER_MAJOR << 16) | + * VPU_BOOT_API_VER_MINOR; + * VPU_BOOT_API_VER_PATCH will be ignored. KMD and compatibility is not affected if this changes + * This information is collected by using vpuip_2/application/vpuFirmware/make_std_fw_image.py + * If a header is missing this info we ignore the header, if a header is missing or contains + * partial info a build error will be generated. + */ + +/** + * Major version changes that break backward compatibility. + * Major version must start from 1 and can only be incremented. + */ +#define VPU_BOOT_API_VER_MAJOR 3 + +/** + * Minor version changes when API backward compatibility is preserved. + * Resets to 0 if Major version is incremented. + */ +#define VPU_BOOT_API_VER_MINOR 29 + +/** + * API header changed (field names, documentation, formatting) but API itself has not been changed + */ +#define VPU_BOOT_API_VER_PATCH 5 + +/** + * Index in the API version table + * Must be unique for each API + */ +#define VPU_BOOT_API_VER_INDEX 0 + +#pragma pack(push, 4) + +/** + * Firmware image header format + */ +#define VPU_FW_HEADER_SIZE 4096 +#define VPU_FW_HEADER_VERSION 0x1 +#define VPU_FW_VERSION_SIZE 32 +#define VPU_FW_API_VER_NUM 16 + +struct vpu_firmware_header { + u32 header_version; + u32 image_format; + u64 image_load_address; + u32 image_size; + u64 entry_point; + u8 vpu_version[VPU_FW_VERSION_SIZE]; + u32 compression_type; + u64 firmware_version_load_address; + u32 firmware_version_size; + u64 boot_params_load_address; + u32 api_version[VPU_FW_API_VER_NUM]; + /** Size of memory require for firmware execution */ + u32 runtime_size; + u32 shave_nn_fw_size; + /** + * Size of primary preemption buffer, assuming a 2-job submission queue. + * NOTE: host driver is expected to adapt size accordingly to actual + * submission queue size and device capabilities. + */ + u32 preemption_buffer_1_size; + /** + * Size of secondary preemption buffer, assuming a 2-job submission queue. + * NOTE: host driver is expected to adapt size accordingly to actual + * submission queue size and device capabilities. + */ + u32 preemption_buffer_2_size; + /** + * Maximum preemption buffer size that the FW can use: no need for the host + * driver to allocate more space than that specified by these fields. + * A value of 0 means no declared limit. + */ + u32 preemption_buffer_1_max_size; + u32 preemption_buffer_2_max_size; + /** Space reserved for future preemption-related fields. */ + u32 preemption_reserved[4]; + /** FW image read only section start address, 4KB aligned */ + u64 ro_section_start_address; + /** FW image read only section size, 4KB aligned */ + u32 ro_section_size; + u32 reserved; +}; + +/** + * Firmware boot parameters format + */ + +/** Values for boot_type field */ +#define VPU_BOOT_TYPE_COLDBOOT 0 +#define VPU_BOOT_TYPE_WARMBOOT 1 + +/** Value for magic filed */ +#define VPU_BOOT_PARAMS_MAGIC 0x10000 + +/** VPU scheduling mode. By default, OS scheduling is used. */ +#define VPU_SCHEDULING_MODE_OS 0 +#define VPU_SCHEDULING_MODE_HW 1 + +enum VPU_BOOT_L2_CACHE_CFG_TYPE { + VPU_BOOT_L2_CACHE_CFG_UPA = 0, + VPU_BOOT_L2_CACHE_CFG_NN = 1, + VPU_BOOT_L2_CACHE_CFG_NUM = 2 +}; + +/** VPU MCA ECC signalling mode. By default, no signalling is used */ +enum VPU_BOOT_MCA_ECC_SIGNAL_TYPE { + VPU_BOOT_MCA_ECC_NONE = 0, + VPU_BOOT_MCA_ECC_CORR = 1, + VPU_BOOT_MCA_ECC_FATAL = 2, + VPU_BOOT_MCA_ECC_BOTH = 3 +}; + +/** + * Logging destinations. + * + * Logging output can be directed to different logging destinations. This enum + * defines the list of logging destinations supported by the VPU firmware (NOTE: + * a specific VPU FW binary may support only a subset of such output + * destinations, depending on the target platform and compile options). + */ +enum vpu_trace_destination { + VPU_TRACE_DESTINATION_PIPEPRINT = 0x1, + VPU_TRACE_DESTINATION_VERBOSE_TRACING = 0x2, + VPU_TRACE_DESTINATION_NORTH_PEAK = 0x4, +}; + +/* + * Processor bit shifts (for loggable HW components). + */ +#define VPU_TRACE_PROC_BIT_RESERVED 0 +#define VPU_TRACE_PROC_BIT_LRT 1 +#define VPU_TRACE_PROC_BIT_LNN 2 +#define VPU_TRACE_PROC_BIT_SHV_0 3 +#define VPU_TRACE_PROC_BIT_SHV_1 4 +#define VPU_TRACE_PROC_BIT_SHV_2 5 +#define VPU_TRACE_PROC_BIT_SHV_3 6 +#define VPU_TRACE_PROC_BIT_SHV_4 7 +#define VPU_TRACE_PROC_BIT_SHV_5 8 +#define VPU_TRACE_PROC_BIT_SHV_6 9 +#define VPU_TRACE_PROC_BIT_SHV_7 10 +#define VPU_TRACE_PROC_BIT_SHV_8 11 +#define VPU_TRACE_PROC_BIT_SHV_9 12 +#define VPU_TRACE_PROC_BIT_SHV_10 13 +#define VPU_TRACE_PROC_BIT_SHV_11 14 +#define VPU_TRACE_PROC_BIT_SHV_12 15 +#define VPU_TRACE_PROC_BIT_SHV_13 16 +#define VPU_TRACE_PROC_BIT_SHV_14 17 +#define VPU_TRACE_PROC_BIT_SHV_15 18 +#define VPU_TRACE_PROC_BIT_ACT_SHV_0 19 +#define VPU_TRACE_PROC_BIT_ACT_SHV_1 20 +#define VPU_TRACE_PROC_BIT_ACT_SHV_2 21 +#define VPU_TRACE_PROC_BIT_ACT_SHV_3 22 +#define VPU_TRACE_PROC_NO_OF_HW_DEVS 23 + +/** VPU 30xx HW component IDs are sequential, so define first and last IDs. */ +#define VPU_TRACE_PROC_BIT_30XX_FIRST VPU_TRACE_PROC_BIT_LRT +#define VPU_TRACE_PROC_BIT_30XX_LAST VPU_TRACE_PROC_BIT_SHV_15 + +struct vpu_boot_l2_cache_config { + u8 use; + u8 cfg; +}; + +/** + * When HW scheduling mode is enabled, a present period is defined. + * It will be used by VPU to swap between normal and focus priorities + * to prevent starving of normal priority band (when implemented). + * Host must provide a valid value at boot time in + * `vpu_focus_present_timer_ms`. If the value provided by the host is not within the + * defined range a default value will be used. Here we define the min. and max. + * allowed values and the and default value of the present period. Units are milliseconds. + */ +#define VPU_PRESENT_CALL_PERIOD_MS_DEFAULT 50 +#define VPU_PRESENT_CALL_PERIOD_MS_MIN 16 +#define VPU_PRESENT_CALL_PERIOD_MS_MAX 10000 + +/** + * Macros to enable various power profiles within the NPU. + * To be defined as part of 32 bit mask. + */ +#define POWER_PROFILE_SURVIVABILITY 0x1 + +/** + * Enum for dvfs_mode boot param. + */ +enum vpu_governor { + VPU_GOV_DEFAULT = 0, /** Default Governor for the system */ + VPU_GOV_MAX_PERFORMANCE = 1, /** Maximum performance governor */ + VPU_GOV_ON_DEMAND = 2, /** On Demand frequency control governor */ + VPU_GOV_POWER_SAVE = 3, /** Power save governor */ + VPU_GOV_ON_DEMAND_PRIORITY_AWARE = 4 /** On Demand priority based governor */ +}; + +struct vpu_boot_params { + u32 magic; + u32 vpu_id; + u32 vpu_count; + u32 reserved_0[5]; + /** Clock frequencies: 0x20 - 0xFF */ + u32 frequency; + u32 reserved_1[12]; + u32 perf_clk_frequency; + u32 reserved_2[42]; + /** Memory regions: 0x100 - 0x1FF */ + u64 ipc_header_area_start; + u32 ipc_header_area_size; + u64 shared_region_base; + u32 shared_region_size; + u64 ipc_payload_area_start; + u32 ipc_payload_area_size; + u64 global_aliased_pio_base; + u32 global_aliased_pio_size; + u32 autoconfig; + struct vpu_boot_l2_cache_config cache_defaults[VPU_BOOT_L2_CACHE_CFG_NUM]; + u32 reserved_3[3]; + /** + * ShaveNN FW section VPU base address + * On VPU2.7 HW this address must be within 2GB range starting from L2C_PAGE_TABLE base + */ + u64 shave_nn_fw_base; + u64 save_restore_ret_address; /** stores the address of FW's restore entry point */ + u32 reserved_4[43]; + /** IRQ re-direct numbers: 0x200 - 0x2FF */ + s32 watchdog_irq_mss; + s32 watchdog_irq_nce; + /** ARM -> VPU doorbell interrupt. ARM is notifying VPU of async command or compute job. */ + u32 host_to_vpu_irq; + /** VPU -> ARM job done interrupt. VPU is notifying ARM of compute job completion. */ + u32 job_done_irq; + /** Padding. */ + u32 reserved_5[60]; + /** Silicon information: 0x300 - 0x3FF */ + u32 host_version_id; + u32 si_stepping; + u64 device_id; + u64 feature_exclusion; + u64 sku; + /** PLL ratio for minimum clock frequency */ + u32 min_freq_pll_ratio; + /** PLL ratio for maximum clock frequency */ + u32 max_freq_pll_ratio; + /** + * Initial log level threshold (messages with log level severity less than + * the threshold will not be logged); applies to every enabled logging + * destination and loggable HW component. See 'mvLog_t' enum for acceptable + * values. + * TODO: EISW-33556: Move log level definition (mvLog_t) to this file. + */ + u32 default_trace_level; + u32 boot_type; + u64 punit_telemetry_sram_base; + u64 punit_telemetry_sram_size; + u32 vpu_telemetry_enable; + u64 crit_tracing_buff_addr; + u32 crit_tracing_buff_size; + u64 verbose_tracing_buff_addr; + u32 verbose_tracing_buff_size; + u64 verbose_tracing_sw_component_mask; /** TO BE REMOVED */ + /** + * Mask of destinations to which logging messages are delivered; bitwise OR + * of values defined in vpu_trace_destination enum. + */ + u32 trace_destination_mask; + /** + * Mask of hardware components for which logging is enabled; bitwise OR of + * bits defined by the VPU_TRACE_PROC_BIT_* macros. + */ + u64 trace_hw_component_mask; + /** Mask of trace message formats supported by the driver */ + u64 tracing_buff_message_format_mask; + u64 trace_reserved_1[2]; + u32 reserved_6; + /** PLL ratio for efficient clock frequency */ + u32 pn_freq_pll_ratio; + /** + * DVFS Mode: + * 0 - Default, DVFS mode selected by the firmware + * 1 - Max Performance + * 2 - On Demand + * 3 - Power Save + * 4 - On Demand Priority Aware + */ + u32 dvfs_mode; + /** + * Depending on DVFS Mode: + * On-demand: Default if 0. + * Bit 0-7 - uint8_t: Highest residency percent + * Bit 8-15 - uint8_t: High residency percent + * Bit 16-23 - uint8_t: Low residency percent + * Bit 24-31 - uint8_t: Lowest residency percent + * Bit 32-35 - unsigned 4b: PLL Ratio increase amount on highest residency + * Bit 36-39 - unsigned 4b: PLL Ratio increase amount on high residency + * Bit 40-43 - unsigned 4b: PLL Ratio decrease amount on low residency + * Bit 44-47 - unsigned 4b: PLL Ratio decrease amount on lowest frequency + * Bit 48-55 - uint8_t: Period (ms) for residency decisions + * Bit 56-63 - uint8_t: Averaging windows (as multiples of period. Max: 30 decimal) + * Power Save/Max Performance: Unused + */ + u64 dvfs_param; + /** + * D0i3 delayed entry + * Bit 0: Disable CPU state save on D0i2 entry flow. + * 0: Every D0i2 entry saves state. Save state IPC message ignored. + * 1: IPC message required to save state on D0i3 entry flow. + * NOTE: This parameter is deprecated starting NPU50xx+. Bit 0 is now hardcoded to 1, + * meaning CPU state save always requires IPC message on D0i3 entry flow. + */ + u32 d0i3_delayed_entry; + /** Time spent by VPU in D0i3 state */ + u64 d0i3_residency_time_us; + /** Value of VPU perf counter at the time of entering D0i3 state . */ + u64 d0i3_entry_vpu_ts; + /** + * The system time of the host operating system in microseconds. + * E.g the number of microseconds since 1st of January 1970, or whatever + * date the host operating system uses to maintain system time. + * This value will be used to track system time on the VPU. + * The KMD is required to update this value on every VPU reset. + */ + u64 system_time_us; + u32 reserved_7[2]; + /** + * The delta between device monotonic time and the current value of the + * HW timestamp register, in ticks. Written by the firmware during boot. + * Can be used by the KMD to calculate device time. + */ + u64 device_time_delta_ticks; + u32 reserved_8[30]; + /** Power States transitions timestamps: 0x440 - 0x46F*/ + struct power_states_timestamps { + /** VPU_IDLE -> VPU_ACTIVE transition initiated timestamp */ + u64 vpu_active_state_requested; + /** VPU_IDLE -> VPU_ACTIVE transition completed timestamp */ + u64 vpu_active_state_achieved; + /** VPU_ACTIVE -> VPU_IDLE transition initiated timestamp */ + u64 vpu_idle_state_requested; + /** VPU_ACTIVE -> VPU_IDLE transition completed timestamp */ + u64 vpu_idle_state_achieved; + /** VPU_IDLE -> VPU_STANDBY transition initiated timestamp */ + u64 vpu_standby_state_requested; + /** VPU_IDLE -> VPU_STANDBY transition completed timestamp */ + u64 vpu_standby_state_achieved; + } power_states_timestamps; + /** VPU scheduling mode. Values defined by VPU_SCHEDULING_MODE_* macros. */ + u32 vpu_scheduling_mode; + /** Present call period in milliseconds. */ + u32 vpu_focus_present_timer_ms; + /** VPU ECC Signaling */ + u32 vpu_uses_ecc_mca_signal; + /** Values defined by POWER_PROFILE* macros */ + u32 power_profile; + /** Microsecond value for DCT active cycle */ + u32 dct_active_us; + /** Microsecond value for DCT inactive cycle */ + u32 dct_inactive_us; + /** Unused/reserved: 0x488 - 0xFFF */ + u32 reserved_9[734]; +}; + +/** Magic numbers set between host and vpu to detect corruption of tracing init */ +#define VPU_TRACING_BUFFER_CANARY (0xCAFECAFE) + +/** Tracing buffer message format definitions */ +#define VPU_TRACING_FORMAT_STRING 0 +#define VPU_TRACING_FORMAT_MIPI 2 +/** + * Header of the tracing buffer. + * The below defined header will be stored at the beginning of + * each allocated tracing buffer, followed by a series of 256b + * of ASCII trace message entries. + */ +struct vpu_tracing_buffer_header { + /** + * Magic number set by host to detect corruption + * @see VPU_TRACING_BUFFER_CANARY + */ + u32 host_canary_start; + /** offset from start of buffer for trace entries */ + u32 read_index; + /** keeps track of wrapping on the reader side */ + u32 read_wrap_count; + u32 pad_to_cache_line_size_0[13]; + /** End of first cache line */ + + /** + * Magic number set by host to detect corruption + * @see VPU_TRACING_BUFFER_CANARY + */ + u32 vpu_canary_start; + /** offset from start of buffer from write start */ + u32 write_index; + /** counter for buffer wrapping */ + u32 wrap_count; + /** legacy field - do not use */ + u32 reserved_0; + /** + * Size of the log buffer including this header (`header_size`) and space + * reserved for all messages. If `alignment` is greater than 0, the `size` + * must be a multiple of `alignment`. + */ + u32 size; + /** Header version */ + u16 header_version; + /** Header size */ + u16 header_size; + /** + * Format of the messages in the trace buffer + * 0 - null terminated string + * 1 - size + null terminated string + * 2 - MIPI-SysT encoding + */ + u32 format; + /** + * Message alignment + * 0 - messages are place 1 after another + * n - every message starts and multiple on offset + */ + u32 alignment; /** 64, 128, 256 */ + /** Name of the logging entity, i.e "LRT", "LNN", "SHV0", etc */ + char name[16]; + u32 pad_to_cache_line_size_1[4]; + /** End of second cache line */ +}; + +#pragma pack(pop) + +#endif + +///@} diff --git a/drivers/accel/ivpu/vpu_jsm_api.h b/drivers/accel/ivpu/vpu_jsm_api.h new file mode 100644 index 000000000..6d58d04cc --- /dev/null +++ b/drivers/accel/ivpu/vpu_jsm_api.h @@ -0,0 +1,1758 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright (c) 2020-2025, Intel Corporation. + */ + +/** + * @addtogroup Jsm + * @{ + */ + +/** + * @file + * @brief JSM shared definitions + */ +#ifndef VPU_JSM_API_H +#define VPU_JSM_API_H + +/* + * Major version changes that break backward compatibility + */ +#define VPU_JSM_API_VER_MAJOR 3 + +/* + * Minor version changes when API backward compatibility is preserved. + */ +#define VPU_JSM_API_VER_MINOR 34 + +/* + * API header changed (field names, documentation, formatting) but API itself has not been changed + */ +#define VPU_JSM_API_VER_PATCH 0 + +/* + * Index in the API version table + */ +#define VPU_JSM_API_VER_INDEX 4 + +/* + * Number of Priority Bands for Hardware Scheduling + * Bands: Idle(0), Normal(1), Focus(2), RealTime(3) + */ +#define VPU_HWS_NUM_PRIORITY_BANDS 4 + +/* Max number of impacted contexts that can be dealt with the engine reset command */ +#define VPU_MAX_ENGINE_RESET_IMPACTED_CONTEXTS 3 + +/* + * Pack the API structures to enforce binary compatibility + * Align to 8 bytes for optimal performance + */ +#pragma pack(push, 8) + +/* + * Engine indexes. + */ +#define VPU_ENGINE_COMPUTE 0 +#define VPU_ENGINE_NB 1 + +/* + * VPU status values. + */ +#define VPU_JSM_STATUS_SUCCESS 0x0U +#define VPU_JSM_STATUS_PARSING_ERR 0x1U +#define VPU_JSM_STATUS_PROCESSING_ERR 0x2U +#define VPU_JSM_STATUS_PREEMPTED 0x3U +#define VPU_JSM_STATUS_ABORTED 0x4U +#define VPU_JSM_STATUS_USER_CTX_VIOL_ERR 0x5U +#define VPU_JSM_STATUS_GLOBAL_CTX_VIOL_ERR 0x6U +#define VPU_JSM_STATUS_MVNCI_WRONG_INPUT_FORMAT 0x7U +#define VPU_JSM_STATUS_MVNCI_UNSUPPORTED_NETWORK_ELEMENT 0x8U +#define VPU_JSM_STATUS_MVNCI_INVALID_HANDLE 0x9U +#define VPU_JSM_STATUS_MVNCI_OUT_OF_RESOURCES 0xAU +#define VPU_JSM_STATUS_MVNCI_NOT_IMPLEMENTED 0xBU +#define VPU_JSM_STATUS_MVNCI_INTERNAL_ERROR 0xCU +/* @deprecated (use VPU_JSM_STATUS_PREEMPTED_MID_COMMAND instead) */ +#define VPU_JSM_STATUS_PREEMPTED_MID_INFERENCE 0xDU +/* Job status returned when the job was preempted mid-command */ +#define VPU_JSM_STATUS_PREEMPTED_MID_COMMAND 0xDU +/* Range of status codes that require engine reset */ +#define VPU_JSM_STATUS_ENGINE_RESET_REQUIRED_MIN 0xEU +#define VPU_JSM_STATUS_MVNCI_CONTEXT_VIOLATION_HW 0xEU +#define VPU_JSM_STATUS_MVNCI_PREEMPTION_TIMED_OUT 0xFU +#define VPU_JSM_STATUS_ENGINE_RESET_REQUIRED_MAX 0x1FU + +/* + * Host <-> VPU IPC channels. + * ASYNC commands use a high priority channel, other messages use low-priority ones. + */ +#define VPU_IPC_CHAN_ASYNC_CMD 0 +#define VPU_IPC_CHAN_GEN_CMD 10 +#define VPU_IPC_CHAN_JOB_RET 11 + +/* + * Job flags bit masks. + */ +enum { + /* + * Null submission mask. + * When set, batch buffer's commands are not processed but returned as + * successful immediately, except fences and timestamps. + * When cleared, batch buffer's commands are processed normally. + * Used for testing and profiling purposes. + */ + VPU_JOB_FLAGS_NULL_SUBMISSION_MASK = (1 << 0U), + /* + * Inline command mask. + * When set, the object in job queue is an inline command (see struct vpu_inline_cmd below). + * When cleared, the object in job queue is a job (see struct vpu_job_queue_entry below). + */ + VPU_JOB_FLAGS_INLINE_CMD_MASK = (1 << 1U), + /* + * VPU private data mask. + * Reserved for the VPU to store private data about the job (or inline command) + * while being processed. + */ + VPU_JOB_FLAGS_PRIVATE_DATA_MASK = 0xFFFF0000U +}; + +/* + * Job queue flags bit masks. + */ +enum { + /* + * No job done notification mask. + * When set, indicates that no job done notification should be sent for any + * job from this queue. When cleared, indicates that job done notification + * should be sent for every job completed from this queue. + */ + VPU_JOB_QUEUE_FLAGS_NO_JOB_DONE_MASK = (1 << 0U), + /* + * Native fence usage mask. + * When set, indicates that job queue uses native fences (as inline commands + * in job queue). Such queues may also use legacy fences (as commands in batch buffers). + * When cleared, indicates the job queue only uses legacy fences. + * NOTES: + * 1. For queues using native fences, VPU expects that all jobs in the queue + * are immediately followed by an inline command object. This object is expected + * to be a fence signal command in most cases, but can also be a NOP in case the host + * does not need per-job fence signalling. Other inline commands objects can be + * inserted between "job and inline command" pairs. + * 2. Native fence queues are only supported on VPU 40xx onwards. + */ + VPU_JOB_QUEUE_FLAGS_USE_NATIVE_FENCE_MASK = (1 << 1U), + /* + * Enable turbo mode for testing NPU performance; not recommended for regular usage. + */ + VPU_JOB_QUEUE_FLAGS_TURBO_MODE = (1 << 2U), + /* + * Queue error detection mode flag + * For 'interactive' queues (this bit not set), the FW will identify queues that have not + * completed a job inside the TDR timeout as in error as part of engine reset sequence. + * For 'non-interactive' queues (this bit set), the FW will identify queues that have not + * progressed the heartbeat inside the non-interactive no-progress timeout as in error as + * part of engine reset sequence. Additionally, there is an upper limit applied to these + * queues: even if they progress the heartbeat, if they run longer than non-interactive + * timeout, then the FW will also identify them as in error. + */ + VPU_JOB_QUEUE_FLAGS_NON_INTERACTIVE = (1 << 3U) +}; + +/* + * Max length (including trailing NULL char) of trace entity name (e.g., the + * name of a logging destination or a loggable HW component). + */ +#define VPU_TRACE_ENTITY_NAME_MAX_LEN 32 + +/* + * Max length (including trailing NULL char) of a dyndbg command. + * + * NOTE: 96 is used so that the size of 'struct vpu_ipc_msg' in the JSM API is + * 128 bytes (multiple of 64 bytes, the cache line size). + */ +#define VPU_DYNDBG_CMD_MAX_LEN 96 + +/* + * For HWS command queue scheduling, we can prioritise command queues inside the + * same process with a relative in-process priority. Valid values for relative + * priority are given below - max and min. + */ +#define VPU_HWS_COMMAND_QUEUE_MAX_IN_PROCESS_PRIORITY 7 +#define VPU_HWS_COMMAND_QUEUE_MIN_IN_PROCESS_PRIORITY -7 + +/* + * For HWS priority scheduling, we can have multiple realtime priority bands. + * They are numbered 0 to a MAX. + */ +#define VPU_HWS_MAX_REALTIME_PRIORITY_LEVEL 31U + +/* + * vpu_jsm_engine_reset_context flag definitions + */ +#define VPU_ENGINE_RESET_CONTEXT_FLAG_COLLATERAL_DAMAGE_MASK BIT(0) +#define VPU_ENGINE_RESET_CONTEXT_HANG_PRIMARY_CAUSE 0 +#define VPU_ENGINE_RESET_CONTEXT_COLLATERAL_DAMAGE 1 + +/* + * Invalid command queue handle identifier. Applies to cmdq_id and cmdq_group + * in this API. + */ +#define VPU_HWS_INVALID_CMDQ_HANDLE 0ULL + +/* + * Inline commands types. + */ +/* + * NOP. + * VPU does nothing other than consuming the inline command object. + */ +#define VPU_INLINE_CMD_TYPE_NOP 0x0 +/* + * Fence wait. + * VPU waits for the fence current value to reach monitored value. + * Fence wait operations are executed upon job dispatching. While waiting for + * the fence to be satisfied, VPU blocks fetching of the next objects in the queue. + * Jobs present in the queue prior to the fence wait object may be processed + * concurrently. + */ +#define VPU_INLINE_CMD_TYPE_FENCE_WAIT 0x1 +/* + * Fence signal. + * VPU sets the fence current value to the provided value. If new current value + * is equal to or higher than monitored value, VPU sends fence signalled notification + * to the host. Fence signal operations are executed upon completion of all the jobs + * present in the queue prior to them, and in-order relative to each other in the queue. + * But jobs in-between them may be processed concurrently and may complete out-of-order. + */ +#define VPU_INLINE_CMD_TYPE_FENCE_SIGNAL 0x2 + +/** + * Job scheduling priority bands for both hardware scheduling and OS scheduling. + */ +enum vpu_job_scheduling_priority_band { + VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE = 0, + VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL = 1, + VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS = 2, + VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME = 3, + VPU_JOB_SCHEDULING_PRIORITY_BAND_COUNT = 4, +}; + +/** + * Job format. + * Jobs defines the actual workloads to be executed by a given engine. + */ +struct vpu_job_queue_entry { + /** Address of VPU commands batch buffer */ + u64 batch_buf_addr; + /** Job ID */ + u32 job_id; + /** Flags bit field, see VPU_JOB_FLAGS_* above */ + u32 flags; + /** + * Doorbell ring timestamp taken by KMD from SoC's global system clock, in + * microseconds. NPU can convert this value to its own fixed clock's timebase, + * to match other profiling timestamps. + */ + u64 doorbell_timestamp; + /** Extra id for job tracking, used only in the firmware perf traces */ + u64 host_tracking_id; + /** Address of the primary preemption buffer to use for this job */ + u64 primary_preempt_buf_addr; + /** Size of the primary preemption buffer to use for this job */ + u32 primary_preempt_buf_size; + /** Size of secondary preemption buffer to use for this job */ + u32 secondary_preempt_buf_size; + /** Address of secondary preemption buffer to use for this job */ + u64 secondary_preempt_buf_addr; + u64 reserved_0; +}; + +/** + * Inline command format. + * Inline commands are the commands executed at scheduler level (typically, + * synchronization directives). Inline command and job objects must be of + * the same size and have flags field at same offset. + */ +struct vpu_inline_cmd { + u64 reserved_0; + /** Inline command type, see VPU_INLINE_CMD_TYPE_* defines. */ + u32 type; + /** Flags bit field, see VPU_JOB_FLAGS_* above. */ + u32 flags; + /** Inline command payload. Depends on inline command type. */ + union payload { + /** Fence (wait and signal) commands' payload. */ + struct fence { + /** Fence object handle. */ + u64 fence_handle; + /** User VA of the current fence value. */ + u64 current_value_va; + /** User VA of the monitored fence value (read-only). */ + u64 monitored_value_va; + /** Value to wait for or write in fence location. */ + u64 value; + /** User VA of the log buffer in which to add log entry on completion. */ + u64 log_buffer_va; + /** NPU private data. */ + u64 npu_private_data; + } fence; + /** + * Other commands do not have a payload: + * Payload definition for future inline commands can be inserted here. + */ + u64 reserved_1[6]; + } payload; +}; + +/** + * Job queue slots can be populated either with job objects or inline command objects. + */ +union vpu_jobq_slot { + struct vpu_job_queue_entry job; + struct vpu_inline_cmd inline_cmd; +}; + +/** + * Job queue control registers. + */ +struct vpu_job_queue_header { + u32 engine_idx; + u32 head; + u32 tail; + u32 flags; + /** Set to 1 to indicate priority_band field is valid */ + u32 priority_band_valid; + /** + * Priority for the work of this job queue, valid only if the HWS is NOT used + * and the @ref priority_band_valid is set to 1. It is applied only during + * the @ref VPU_JSM_MSG_REGISTER_DB message processing. + * The device firmware might use the priority_band to optimize the power + * management logic, but it will not affect the order of jobs. + * Available priority bands: @see enum vpu_job_scheduling_priority_band + */ + u32 priority_band; + /** Inside realtime band assigns a further priority, limited to 0..31 range */ + u32 realtime_priority_level; + u32 reserved_0[9]; +}; + +/* + * Job queue format. + */ +struct vpu_job_queue { + struct vpu_job_queue_header header; + union vpu_jobq_slot slot[]; +}; + +/** + * Logging entity types. + * + * This enum defines the different types of entities involved in logging. + */ +enum vpu_trace_entity_type { + /** Logging destination (entity where logs can be stored / printed). */ + VPU_TRACE_ENTITY_TYPE_DESTINATION = 1, + /** Loggable HW component (HW entity that can be logged). */ + VPU_TRACE_ENTITY_TYPE_HW_COMPONENT = 2, +}; + +/** + * HWS specific log buffer header details. + * Total size is 32 bytes. + */ +struct vpu_hws_log_buffer_header { + /** Written by VPU after adding a log entry. Initialised by host to 0. */ + u32 first_free_entry_index; + /** Incremented by VPU every time the VPU writes the 0th entry; initialised by host to 0. */ + u32 wraparound_count; + /** + * This is the number of buffers that can be stored in the log buffer provided by the host. + * It is written by host before passing buffer to VPU. VPU should consider it read-only. + */ + u64 num_of_entries; + u64 reserved[2]; +}; + +/** + * HWS specific log buffer entry details. + * Total size is 32 bytes. + */ +struct vpu_hws_log_buffer_entry { + /** VPU timestamp must be an invariant timer tick (not impacted by DVFS) */ + u64 vpu_timestamp; + /** + * Operation type: + * 0 - context state change + * 1 - queue new work + * 2 - queue unwait sync object + * 3 - queue no more work + * 4 - queue wait sync object + */ + u32 operation_type; + u32 reserved; + /** Operation data depends on operation type */ + u64 operation_data[2]; +}; + +/* Native fence log buffer types. */ +enum vpu_hws_native_fence_log_type { + VPU_HWS_NATIVE_FENCE_LOG_TYPE_WAITS = 1, + VPU_HWS_NATIVE_FENCE_LOG_TYPE_SIGNALS = 2 +}; + +/** HWS native fence log buffer header. */ +struct vpu_hws_native_fence_log_header { + union { + struct { + /** Index of the first free entry in buffer. */ + u32 first_free_entry_idx; + /** + * Incremented whenever the NPU wraps around the buffer and writes + * to the first entry again. + */ + u32 wraparound_count; + }; + /** Field allowing atomic update of both fields above. */ + u64 atomic_wraparound_and_entry_idx; + }; + /** Log buffer type, see enum vpu_hws_native_fence_log_type. */ + u64 type; + /** Allocated number of entries in the log buffer. */ + u64 entry_nb; + u64 reserved[2]; +}; + +/** Native fence log operation types. */ +enum vpu_hws_native_fence_log_op { + VPU_HWS_NATIVE_FENCE_LOG_OP_SIGNAL_EXECUTED = 0, + VPU_HWS_NATIVE_FENCE_LOG_OP_WAIT_UNBLOCKED = 1 +}; + +/** HWS native fence log entry. */ +struct vpu_hws_native_fence_log_entry { + /** Newly signaled/unblocked fence value. */ + u64 fence_value; + /** Native fence object handle to which this operation belongs. */ + u64 fence_handle; + /** Operation type, see enum vpu_hws_native_fence_log_op. */ + u64 op_type; + u64 reserved_0; + /** + * VPU_HWS_NATIVE_FENCE_LOG_OP_WAIT_UNBLOCKED only: Timestamp at which fence + * wait was started (in NPU SysTime). + */ + u64 fence_wait_start_ts; + u64 reserved_1; + /** Timestamp at which fence operation was completed (in NPU SysTime). */ + u64 fence_end_ts; +}; + +/** Native fence log buffer. */ +struct vpu_hws_native_fence_log_buffer { + struct vpu_hws_native_fence_log_header header; + struct vpu_hws_native_fence_log_entry entry[]; +}; + +/* + * Host <-> VPU IPC messages types. + */ +enum vpu_ipc_msg_type { + /** Unsupported command */ + VPU_JSM_MSG_UNKNOWN = 0xFFFFFFFF, + + /** IPC Host -> Device, base id for async commands */ + VPU_JSM_MSG_ASYNC_CMD = 0x1100, + /** + * Reset engine. The NPU cancels all the jobs currently executing on the target + * engine making the engine become idle and then does a HW reset, before returning + * to the host. + * @see struct vpu_ipc_msg_payload_engine_reset + */ + VPU_JSM_MSG_ENGINE_RESET = VPU_JSM_MSG_ASYNC_CMD, + /** + * Preempt engine. The NPU stops (preempts) all the jobs currently + * executing on the target engine making the engine become idle and ready to + * execute new jobs. + * NOTE: The NPU does not remove unstarted jobs (if any) from job queues of + * the target engine, but it stops processing them (until the queue doorbell + * is rung again); the host is responsible to reset the job queue, either + * after preemption or when resubmitting jobs to the queue. + * @see vpu_ipc_msg_payload_engine_preempt + */ + VPU_JSM_MSG_ENGINE_PREEMPT = 0x1101, + /** + * OS scheduling doorbell register command + * @see vpu_ipc_msg_payload_register_db + */ + VPU_JSM_MSG_REGISTER_DB = 0x1102, + /** + * OS scheduling doorbell unregister command + * @see vpu_ipc_msg_payload_unregister_db + */ + VPU_JSM_MSG_UNREGISTER_DB = 0x1103, + /** + * Query engine heartbeat. Heartbeat is expected to increase monotonically + * and increase while work is being progressed by NPU. + * @see vpu_ipc_msg_payload_query_engine_hb + */ + VPU_JSM_MSG_QUERY_ENGINE_HB = 0x1104, + VPU_JSM_MSG_GET_POWER_LEVEL_COUNT = 0x1105, + VPU_JSM_MSG_GET_POWER_LEVEL = 0x1106, + VPU_JSM_MSG_SET_POWER_LEVEL = 0x1107, + /* @deprecated */ + VPU_JSM_MSG_METRIC_STREAMER_OPEN = 0x1108, + /* @deprecated */ + VPU_JSM_MSG_METRIC_STREAMER_CLOSE = 0x1109, + /** Configure logging (used to modify configuration passed in boot params). */ + VPU_JSM_MSG_TRACE_SET_CONFIG = 0x110a, + /** Return current logging configuration. */ + VPU_JSM_MSG_TRACE_GET_CONFIG = 0x110b, + /** + * Get masks of destinations and HW components supported by the firmware + * (may vary between HW generations and FW compile + * time configurations) + */ + VPU_JSM_MSG_TRACE_GET_CAPABILITY = 0x110c, + /** Get the name of a destination or HW component. */ + VPU_JSM_MSG_TRACE_GET_NAME = 0x110d, + /** + * Release resource associated with host ssid . All jobs that belong to the host_ssid + * aborted and removed from internal scheduling queues. All doorbells assigned + * to the host_ssid are unregistered and any internal FW resources belonging to + * the host_ssid are released. + * @see vpu_ipc_msg_payload_ssid_release + */ + VPU_JSM_MSG_SSID_RELEASE = 0x110e, + /** + * Start collecting metric data. + * @see vpu_jsm_metric_streamer_start + */ + VPU_JSM_MSG_METRIC_STREAMER_START = 0x110f, + /** + * Stop collecting metric data. This command will return success if it is called + * for a metric stream that has already been stopped or was never started. + * @see vpu_jsm_metric_streamer_stop + */ + VPU_JSM_MSG_METRIC_STREAMER_STOP = 0x1110, + /** + * Update current and next buffer for metric data collection. This command can + * also be used to request information about the number of collected samples + * and the amount of data written to the buffer. + * @see vpu_jsm_metric_streamer_update + */ + VPU_JSM_MSG_METRIC_STREAMER_UPDATE = 0x1111, + /** + * Request description of selected metric groups and metric counters within + * each group. The VPU will write the description of groups and counters to + * the buffer specified in the command structure. + * @see vpu_jsm_metric_streamer_start + */ + VPU_JSM_MSG_METRIC_STREAMER_INFO = 0x1112, + /** + * Control command: Priority band setup + * @see vpu_ipc_msg_payload_hws_priority_band_setup + */ + VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP = 0x1113, + /** + * Control command: Create command queue + * @see vpu_ipc_msg_payload_hws_create_cmdq + */ + VPU_JSM_MSG_CREATE_CMD_QUEUE = 0x1114, + /** + * Control command: Destroy command queue + * @see vpu_ipc_msg_payload_hws_destroy_cmdq + */ + VPU_JSM_MSG_DESTROY_CMD_QUEUE = 0x1115, + /** + * Control command: Set context scheduling properties + * @see vpu_ipc_msg_payload_hws_set_context_sched_properties + */ + VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES = 0x1116, + /** + * Register a doorbell to notify VPU of new work. The doorbell may later be + * deallocated or reassigned to another context. + * @see vpu_jsm_hws_register_db + */ + VPU_JSM_MSG_HWS_REGISTER_DB = 0x1117, + /** + * Control command: Log buffer setting + * @see vpu_ipc_msg_payload_hws_set_scheduling_log + */ + VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG = 0x1118, + /** + * Control command: Suspend command queue. + * @see vpu_ipc_msg_payload_hws_suspend_cmdq + */ + VPU_JSM_MSG_HWS_SUSPEND_CMDQ = 0x1119, + /** + * Control command: Resume command queue + * @see vpu_ipc_msg_payload_hws_resume_cmdq + */ + VPU_JSM_MSG_HWS_RESUME_CMDQ = 0x111a, + /** + * Control command: Resume engine after reset + * @see vpu_ipc_msg_payload_hws_resume_engine + */ + VPU_JSM_MSG_HWS_ENGINE_RESUME = 0x111b, + /** + * Control command: Enable survivability/DCT mode + * @see vpu_ipc_msg_payload_pwr_dct_control + */ + VPU_JSM_MSG_DCT_ENABLE = 0x111c, + /** + * Control command: Disable survivability/DCT mode + * This command has no payload + */ + VPU_JSM_MSG_DCT_DISABLE = 0x111d, + /** + * Reserved command ID to ensure that the following command requests / + * responses have the same lower byte value. + */ + VPU_JSM_MSG_RESERVED_111E = 0x111e, + /** + * Control command: Configure VPU frequency scaling parameters. + * @see vpu_ipc_msg_payload_freq_config + */ + VPU_JSM_MSG_FREQ_CONFIG = 0x111f, + /** + * Dump VPU state. To be used for debug purposes only. + * This command has no payload. + * NOTE: Please introduce new ASYNC commands before this one. + */ + VPU_JSM_MSG_STATE_DUMP = 0x11FF, + + /** IPC Host -> Device, base id for general commands */ + VPU_JSM_MSG_GENERAL_CMD = 0x1200, + /** Unsupported command */ + VPU_JSM_MSG_BLOB_DEINIT_DEPRECATED = VPU_JSM_MSG_GENERAL_CMD, + /** + * Control dyndbg behavior by executing a dyndbg command; equivalent to + * Linux command: + * @verbatim echo '' > /dynamic_debug/control @endverbatim + * @see vpu_ipc_msg_payload_dyndbg_control + */ + VPU_JSM_MSG_DYNDBG_CONTROL = 0x1201, + /** + * Perform the save procedure for the D0i3 entry + */ + VPU_JSM_MSG_PWR_D0I3_ENTER = 0x1202, + + /** + * IPC Device -> Host, Job completion + * @see struct vpu_ipc_msg_payload_job_done + */ + VPU_JSM_MSG_JOB_DONE = 0x2100, + /** + * IPC Device -> Host, Fence signalled + * @see vpu_ipc_msg_payload_native_fence_signalled + */ + VPU_JSM_MSG_NATIVE_FENCE_SIGNALLED = 0x2101, + + /* IPC Device -> Host, Async command completion */ + VPU_JSM_MSG_ASYNC_CMD_DONE = 0x2200, + /** + * IPC Device -> Host, engine reset complete + * @see vpu_ipc_msg_payload_engine_reset_done + */ + VPU_JSM_MSG_ENGINE_RESET_DONE = VPU_JSM_MSG_ASYNC_CMD_DONE, + /** + * Preempt complete message + * @see vpu_ipc_msg_payload_engine_preempt_done + */ + VPU_JSM_MSG_ENGINE_PREEMPT_DONE = 0x2201, + VPU_JSM_MSG_REGISTER_DB_DONE = 0x2202, + VPU_JSM_MSG_UNREGISTER_DB_DONE = 0x2203, + /** + * Response to query engine heartbeat. + * @see vpu_ipc_msg_payload_query_engine_hb_done + */ + VPU_JSM_MSG_QUERY_ENGINE_HB_DONE = 0x2204, + VPU_JSM_MSG_GET_POWER_LEVEL_COUNT_DONE = 0x2205, + VPU_JSM_MSG_GET_POWER_LEVEL_DONE = 0x2206, + VPU_JSM_MSG_SET_POWER_LEVEL_DONE = 0x2207, + /* @deprecated */ + VPU_JSM_MSG_METRIC_STREAMER_OPEN_DONE = 0x2208, + /* @deprecated */ + VPU_JSM_MSG_METRIC_STREAMER_CLOSE_DONE = 0x2209, + /** Response to VPU_JSM_MSG_TRACE_SET_CONFIG. */ + VPU_JSM_MSG_TRACE_SET_CONFIG_RSP = 0x220a, + /** Response to VPU_JSM_MSG_TRACE_GET_CONFIG. */ + VPU_JSM_MSG_TRACE_GET_CONFIG_RSP = 0x220b, + /** Response to VPU_JSM_MSG_TRACE_GET_CAPABILITY. */ + VPU_JSM_MSG_TRACE_GET_CAPABILITY_RSP = 0x220c, + /** Response to VPU_JSM_MSG_TRACE_GET_NAME. */ + VPU_JSM_MSG_TRACE_GET_NAME_RSP = 0x220d, + /** + * Response to VPU_JSM_MSG_SSID_RELEASE. + * @see vpu_ipc_msg_payload_ssid_release + */ + VPU_JSM_MSG_SSID_RELEASE_DONE = 0x220e, + /** + * Response to VPU_JSM_MSG_METRIC_STREAMER_START. + * VPU will return an error result if metric collection cannot be started, + * e.g. when the specified metric mask is invalid. + * @see vpu_jsm_metric_streamer_done + */ + VPU_JSM_MSG_METRIC_STREAMER_START_DONE = 0x220f, + /** + * Response to VPU_JSM_MSG_METRIC_STREAMER_STOP. + * Returns information about collected metric data. + * @see vpu_jsm_metric_streamer_done + */ + VPU_JSM_MSG_METRIC_STREAMER_STOP_DONE = 0x2210, + /** + * Response to VPU_JSM_MSG_METRIC_STREAMER_UPDATE. + * Returns information about collected metric data. + * @see vpu_jsm_metric_streamer_done + */ + VPU_JSM_MSG_METRIC_STREAMER_UPDATE_DONE = 0x2211, + /** + * Response to VPU_JSM_MSG_METRIC_STREAMER_INFO. + * Returns a description of the metric groups and metric counters. + * @see vpu_jsm_metric_streamer_done + */ + VPU_JSM_MSG_METRIC_STREAMER_INFO_DONE = 0x2212, + /** + * Asynchronous event sent from the VPU to the host either when the current + * metric buffer is full or when the VPU has collected a multiple of + * @ref vpu_jsm_metric_streamer_start::notify_sample_count samples as indicated + * through the start command (VPU_JSM_MSG_METRIC_STREAMER_START). Returns + * information about collected metric data. + * @see vpu_jsm_metric_streamer_done + */ + VPU_JSM_MSG_METRIC_STREAMER_NOTIFICATION = 0x2213, + /** + * Response to control command: Priority band setup + * @see vpu_ipc_msg_payload_hws_priority_band_setup + */ + VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP_RSP = 0x2214, + /** + * Response to control command: Create command queue + * @see vpu_ipc_msg_payload_hws_create_cmdq_rsp + */ + VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP = 0x2215, + /** + * Response to control command: Destroy command queue + * @see vpu_ipc_msg_payload_hws_destroy_cmdq + */ + VPU_JSM_MSG_DESTROY_CMD_QUEUE_RSP = 0x2216, + /** + * Response to control command: Set context scheduling properties + * @see vpu_ipc_msg_payload_hws_set_context_sched_properties + */ + VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP = 0x2217, + /** + * Response to control command: Log buffer setting + * @see vpu_ipc_msg_payload_hws_set_scheduling_log + */ + VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP = 0x2218, + /** + * IPC Device -> Host, HWS notify index entry of log buffer written + * @see vpu_ipc_msg_payload_hws_scheduling_log_notification + */ + VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION = 0x2219, + /** + * IPC Device -> Host, HWS completion of a context suspend request + * @see vpu_ipc_msg_payload_hws_suspend_cmdq + */ + VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE = 0x221a, + /** + * Response to control command: Resume command queue + * @see vpu_ipc_msg_payload_hws_resume_cmdq + */ + VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP = 0x221b, + /** + * Response to control command: Resume engine command response + * @see vpu_ipc_msg_payload_hws_resume_engine + */ + VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE = 0x221c, + /** + * Response to control command: Enable survivability/DCT mode + * This command has no payload + */ + VPU_JSM_MSG_DCT_ENABLE_DONE = 0x221d, + /** + * Response to control command: Disable survivability/DCT mode + * This command has no payload + */ + VPU_JSM_MSG_DCT_DISABLE_DONE = 0x221e, + /** + * Response to control command: Configure VPU frequency scaling parameters. + * @see vpu_ipc_msg_payload_freq_config + */ + VPU_JSM_MSG_FREQ_CONFIG_RSP = 0x221f, + /** + * Response to state dump control command. + * This command has no payload. + * NOTE: Please introduce new ASYNC responses before this one. + */ + VPU_JSM_MSG_STATE_DUMP_RSP = 0x22FF, + + /* IPC Device -> Host, General command completion */ + VPU_JSM_MSG_GENERAL_CMD_DONE = 0x2300, + VPU_JSM_MSG_BLOB_DEINIT_DONE = VPU_JSM_MSG_GENERAL_CMD_DONE, + /** Response to VPU_JSM_MSG_DYNDBG_CONTROL. */ + VPU_JSM_MSG_DYNDBG_CONTROL_RSP = 0x2301, + /** + * Acknowledgment of completion of the save procedure initiated by + * VPU_JSM_MSG_PWR_D0I3_ENTER + */ + VPU_JSM_MSG_PWR_D0I3_ENTER_DONE = 0x2302, +}; + +enum vpu_ipc_msg_status { VPU_JSM_MSG_FREE, VPU_JSM_MSG_ALLOCATED }; + +/** + * Engine reset request payload + * @see VPU_JSM_MSG_ENGINE_RESET + */ +struct vpu_ipc_msg_payload_engine_reset { + /** Engine to be reset. */ + u32 engine_idx; + /** Reserved */ + u32 reserved_0; +}; + +/** + * Engine preemption request struct + * @see VPU_JSM_MSG_ENGINE_PREEMPT + */ +struct vpu_ipc_msg_payload_engine_preempt { + /** Engine to be preempted. */ + u32 engine_idx; + /** ID of the preemption request. */ + u32 preempt_id; +}; + +/** + * Register doorbell command structure. + * This structure supports doorbell registration for only OS scheduling. + * @see VPU_JSM_MSG_REGISTER_DB + */ +struct vpu_ipc_msg_payload_register_db { + /** Index of the doorbell to register. */ + u32 db_idx; + /** Reserved */ + u32 reserved_0; + /** Virtual address in Global GTT pointing to the start of job queue. */ + u64 jobq_base; + /** Size of the job queue in bytes. */ + u32 jobq_size; + /** Host sub-stream ID for the context assigned to the doorbell. */ + u32 host_ssid; +}; + +/** + * Unregister doorbell command structure. + * Request structure to unregister a doorbell for both HW and OS scheduling. + * @see VPU_JSM_MSG_UNREGISTER_DB + */ +struct vpu_ipc_msg_payload_unregister_db { + /** Index of the doorbell to unregister. */ + u32 db_idx; + /** Reserved */ + u32 reserved_0; +}; + +/** + * Heartbeat request structure + * @see VPU_JSM_MSG_QUERY_ENGINE_HB + */ +struct vpu_ipc_msg_payload_query_engine_hb { + /** Engine to return heartbeat value. */ + u32 engine_idx; + /** Reserved */ + u32 reserved_0; +}; + +struct vpu_ipc_msg_payload_power_level { + /** + * Requested power level. The power level value is in the + * range [0, power_level_count-1] where power_level_count + * is the number of available power levels as returned by + * the get power level count command. A power level of 0 + * corresponds to the maximum possible power level, while + * power_level_count-1 corresponds to the minimum possible + * power level. Values outside of this range are not + * considered to be valid. + */ + u32 power_level; + /* Reserved */ + u32 reserved_0; +}; + +/** + * Structure for requesting ssid release + * @see VPU_JSM_MSG_SSID_RELEASE + */ +struct vpu_ipc_msg_payload_ssid_release { + /** Host sub-stream ID for the context to be released. */ + u32 host_ssid; + /** Reserved */ + u32 reserved_0; +}; + +/** + * @brief Metric streamer start command structure. + * This structure is also used with VPU_JSM_MSG_METRIC_STREAMER_INFO to request metric + * groups and metric counters description from the firmware. + * @see VPU_JSM_MSG_METRIC_STREAMER_START + * @see VPU_JSM_MSG_METRIC_STREAMER_INFO + */ +struct vpu_jsm_metric_streamer_start { + /** + * Bitmask to select the desired metric groups. + * A metric group can belong only to one metric streamer instance at a time. + * Since each metric streamer instance has a unique set of metric groups, it + * can also identify a metric streamer instance if more than one instance was + * started. If the VPU device does not support multiple metric streamer instances, + * then VPU_JSM_MSG_METRIC_STREAMER_START will return an error even if the second + * instance has different groups to the first. + */ + u64 metric_group_mask; + /** Sampling rate in nanoseconds. */ + u64 sampling_rate; + /** + * If > 0 the VPU will send a VPU_JSM_MSG_METRIC_STREAMER_NOTIFICATION message + * after every @ref notify_sample_count samples is collected or dropped by the VPU. + * If set to UINT_MAX the VPU will only generate a notification when the metric + * buffer is full. If set to 0 the VPU will never generate a notification. + */ + u32 notify_sample_count; + u32 reserved_0; + /** + * Address and size of the buffer where the VPU will write metric data. The + * VPU writes all counters from enabled metric groups one after another. If + * there is no space left to write data at the next sample period the VPU + * will switch to the next buffer (@ref next_buffer_addr) and will optionally + * send a notification to the host driver if @ref notify_sample_count is non-zero. + * If @ref next_buffer_addr is NULL the VPU will stop collecting metric data. + */ + u64 buffer_addr; + u64 buffer_size; + /** + * Address and size of the next buffer to write metric data to after the initial + * buffer is full. If the address is NULL the VPU will stop collecting metric + * data. + */ + u64 next_buffer_addr; + u64 next_buffer_size; +}; + +/** + * @brief Metric streamer stop command structure. + * @see VPU_JSM_MSG_METRIC_STREAMER_STOP + */ +struct vpu_jsm_metric_streamer_stop { + /** Bitmask to select the desired metric groups. */ + u64 metric_group_mask; +}; + +/** + * Provide VPU FW with buffers to write metric data. + * @see VPU_JSM_MSG_METRIC_STREAMER_UPDATE + */ +struct vpu_jsm_metric_streamer_update { + /** Metric group mask that identifies metric streamer instance. */ + u64 metric_group_mask; + /** + * Address and size of the buffer where the VPU will write metric data. + * This member dictates how the update operation should perform: + * 1. client needs information about the number of collected samples and the + * amount of data written to the current buffer + * 2. client wants to switch to a new buffer + * + * Case 1. is identified by the buffer address being 0 or the same as the + * currently used buffer address. In this case the buffer size is ignored and + * the size of the current buffer is unchanged. The VPU will return an update + * in the vpu_jsm_metric_streamer_done structure. The internal writing position + * into the buffer is not changed. + * + * Case 2. is identified by the address being non-zero and differs from the + * current buffer address. The VPU will immediately switch data collection to + * the new buffer. Then the VPU will return an update in the + * vpu_jsm_metric_streamer_done structure. + */ + u64 buffer_addr; + u64 buffer_size; + /** + * Address and size of the next buffer to write metric data after the initial + * buffer is full. If the address is NULL the VPU will stop collecting metric + * data but will continue to record dropped samples. + * + * Note that there is a hazard possible if both buffer_addr and the next_buffer_addr + * are non-zero in same update request. It is the host's responsibility to ensure + * that both addresses make sense even if the VPU just switched to writing samples + * from the current to the next buffer. + */ + u64 next_buffer_addr; + u64 next_buffer_size; +}; + +/** + * Device -> host job completion message. + * @see VPU_JSM_MSG_JOB_DONE + */ +struct vpu_ipc_msg_payload_job_done { + /** Engine to which the job was submitted. */ + u32 engine_idx; + /** Index of the doorbell to which the job was submitted */ + u32 db_idx; + /** ID of the completed job */ + u32 job_id; + /** Status of the completed job */ + u32 job_status; + /** Host SSID */ + u32 host_ssid; + /** Zero Padding */ + u32 reserved_0; + /** Command queue id */ + u64 cmdq_id; +}; + +/** + * Notification message upon native fence signalling. + * @see VPU_JSM_MSG_NATIVE_FENCE_SIGNALLED + */ +struct vpu_ipc_msg_payload_native_fence_signalled { + /** Engine ID. */ + u32 engine_idx; + /** Host SSID. */ + u32 host_ssid; + /** CMDQ ID */ + u64 cmdq_id; + /** Fence object handle. */ + u64 fence_handle; +}; + +/** + * vpu_ipc_msg_payload_engine_reset_done will contain an array of this structure + * which contains which queues caused reset if FW was able to detect any error. + * @see vpu_ipc_msg_payload_engine_reset_done + */ +struct vpu_jsm_engine_reset_context { + /** Host SSID */ + u32 host_ssid; + /** Zero Padding */ + u32 reserved_0; + /** Command queue id */ + u64 cmdq_id; + /** See VPU_ENGINE_RESET_CONTEXT_* defines */ + u64 flags; +}; + +/** + * Engine reset response. + * @see VPU_JSM_MSG_ENGINE_RESET_DONE + */ +struct vpu_ipc_msg_payload_engine_reset_done { + /** Engine ordinal */ + u32 engine_idx; + /** Number of impacted contexts */ + u32 num_impacted_contexts; + /** Array of impacted command queue ids and their flags */ + struct vpu_jsm_engine_reset_context + impacted_contexts[VPU_MAX_ENGINE_RESET_IMPACTED_CONTEXTS]; +}; + +/** + * Preemption response struct + * @see VPU_JSM_MSG_ENGINE_PREEMPT_DONE + */ +struct vpu_ipc_msg_payload_engine_preempt_done { + /** Engine preempted. */ + u32 engine_idx; + /** ID of the preemption request. */ + u32 preempt_id; +}; + +/** + * Response structure for register doorbell command for both OS + * and HW scheduling. + * @see VPU_JSM_MSG_REGISTER_DB + * @see VPU_JSM_MSG_HWS_REGISTER_DB + */ +struct vpu_ipc_msg_payload_register_db_done { + /* Index of the registered doorbell. */ + u32 db_idx; + /* Reserved */ + u32 reserved_0; +}; + +/** + * Response structure for unregister doorbell command for both OS + * and HW scheduling. + * @see VPU_JSM_MSG_UNREGISTER_DB + */ +struct vpu_ipc_msg_payload_unregister_db_done { + /* Index of the unregistered doorbell. */ + u32 db_idx; + /* Reserved */ + u32 reserved_0; +}; + +/** + * Structure for heartbeat response + * @see VPU_JSM_MSG_QUERY_ENGINE_HB_DONE + */ +struct vpu_ipc_msg_payload_query_engine_hb_done { + /** Engine returning heartbeat value. */ + u32 engine_idx; + /** Reserved */ + u32 reserved_0; + /** Heartbeat value. */ + u64 heartbeat; +}; + +struct vpu_ipc_msg_payload_get_power_level_count_done { + /** + * Number of supported power levels. The maximum possible + * value of power_level_count is 16 but this may vary across + * implementations. + */ + u32 power_level_count; + /* Reserved */ + u32 reserved_0; + /** + * Power consumption limit for each supported power level in + * [0-100%] range relative to power level 0. + */ + u8 power_limit[16]; +}; + +/** + * HWS priority band setup request / response + * @see VPU_JSM_MSG_SET_PRIORITY_BAND_SETUP + */ +struct vpu_ipc_msg_payload_hws_priority_band_setup { + /* + * Grace period in 100ns units when preempting another priority band for + * this priority band + */ + u32 grace_period[VPU_HWS_NUM_PRIORITY_BANDS]; + /* + * Default quantum in 100ns units for scheduling across processes + * within a priority band + * Minimum value supported by NPU is 1ms (10000 in 100ns units). + */ + u32 process_quantum[VPU_HWS_NUM_PRIORITY_BANDS]; + /* + * Default grace period in 100ns units for processes that preempt each + * other within a priority band + */ + u32 process_grace_period[VPU_HWS_NUM_PRIORITY_BANDS]; + /* + * For normal priority band, specifies the target VPU percentage + * in situations when it's starved by the focus band. + */ + u32 normal_band_percentage; + /* + * TDR timeout value in milliseconds. Default value of 0 meaning no timeout. + */ + u32 tdr_timeout; + /* Non-interactive queue timeout for no progress of heartbeat in milliseconds. + * Default value of 0 meaning no timeout. + */ + u32 non_interactive_no_progress_timeout; + /* + * Non-interactive queue upper limit timeout value in milliseconds. Default + * value of 0 meaning no timeout. + */ + u32 non_interactive_timeout; +}; + +/** + * @brief HWS create command queue request. + * Host will create a command queue via this command. + * Note: Cmdq group is a handle of an object which + * may contain one or more command queues. + * @see VPU_JSM_MSG_CREATE_CMD_QUEUE + */ +struct vpu_ipc_msg_payload_hws_create_cmdq { + /* Process id */ + u64 process_id; + /* Host SSID */ + u32 host_ssid; + /* Engine for which queue is being created */ + u32 engine_idx; + /* Cmdq group: only used for HWS logging of state changes */ + u64 cmdq_group; + /* Command queue id */ + u64 cmdq_id; + /* Command queue base */ + u64 cmdq_base; + /* Command queue size */ + u32 cmdq_size; + /* Zero padding */ + u32 reserved_0; +}; + +/** + * HWS create command queue response. + * @see VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP + */ +struct vpu_ipc_msg_payload_hws_create_cmdq_rsp { + /** Process id */ + u64 process_id; + /** Host SSID */ + u32 host_ssid; + /** Engine for which queue is being created */ + u32 engine_idx; + /** Command queue group */ + u64 cmdq_group; + /** Command queue id */ + u64 cmdq_id; +}; + +/** + * HWS destroy command queue request / response + * @see VPU_JSM_MSG_DESTROY_CMD_QUEUE + * @see VPU_JSM_MSG_DESTROY_CMD_QUEUE_RSP + */ +struct vpu_ipc_msg_payload_hws_destroy_cmdq { + /** Host SSID */ + u32 host_ssid; + /** Zero Padding */ + u32 reserved; + /** Command queue id */ + u64 cmdq_id; +}; + +/** + * HWS set context scheduling properties request / response + * @see VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES + * @see VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP + */ +struct vpu_ipc_msg_payload_hws_set_context_sched_properties { + /** Host SSID */ + u32 host_ssid; + /** Zero Padding */ + u32 reserved_0; + /** Command queue id */ + u64 cmdq_id; + /** + * Priority band to assign to work of this context. + * Available priority bands: @see enum vpu_job_scheduling_priority_band + */ + u32 priority_band; + /** Inside realtime band assigns a further priority */ + u32 realtime_priority_level; + /** Priority relative to other contexts in the same process */ + s32 in_process_priority; + /** Zero padding / Reserved */ + u32 reserved_1; + /** + * Context quantum relative to other contexts of same priority in the same process + * Minimum value supported by NPU is 1ms (10000 in 100ns units). + */ + u64 context_quantum; + /** Grace period when preempting context of the same priority within the same process */ + u64 grace_period_same_priority; + /** Grace period when preempting context of a lower priority within the same process */ + u64 grace_period_lower_priority; +}; + +/** + * Register doorbell command structure. + * This structure supports doorbell registration for both HW and OS scheduling. + * Note: Queue base and size are added here so that the same structure can be used for + * OS scheduling and HW scheduling. For OS scheduling, cmdq_id will be ignored + * and cmdq_base and cmdq_size will be used. For HW scheduling, cmdq_base and cmdq_size will be + * ignored and cmdq_id is used. + * @see VPU_JSM_MSG_HWS_REGISTER_DB + */ +struct vpu_jsm_hws_register_db { + /** Index of the doorbell to register. */ + u32 db_id; + /** Host sub-stream ID for the context assigned to the doorbell. */ + u32 host_ssid; + /** ID of the command queue associated with the doorbell. */ + u64 cmdq_id; + /** Virtual address pointing to the start of command queue. */ + u64 cmdq_base; + /** Size of the command queue in bytes. */ + u64 cmdq_size; +}; + +/** + * Structure to set another buffer to be used for scheduling-related logging. + * The size of the logging buffer and the number of entries is defined as part of the + * buffer itself as described next. + * The log buffer received from the host is made up of; + * - header: 32 bytes in size, as shown in @ref vpu_hws_log_buffer_header. + * The header contains the number of log entries in the buffer. + * - log entry: 0 to n-1, each log entry is 32 bytes in size, as shown in + * @ref vpu_hws_log_buffer_entry. + * The entry contains the VPU timestamp, operation type and data. + * The host should provide the notify index value of log buffer to VPU. This is a + * value defined within the log buffer and when written to will generate the + * scheduling log notification. + * The host should set engine_idx and vpu_log_buffer_va to 0 to disable logging + * for a particular engine. + * VPU will handle one log buffer for each of supported engines. + * VPU should allow the logging to consume one host_ssid. + * @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG + * @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP + * @see VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION + */ +struct vpu_ipc_msg_payload_hws_set_scheduling_log { + /** Engine ordinal */ + u32 engine_idx; + /** Host SSID */ + u32 host_ssid; + /** + * VPU log buffer virtual address. + * Set to 0 to disable logging for this engine. + */ + u64 vpu_log_buffer_va; + /** + * Notify index of log buffer. VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION + * is generated when an event log is written to this index. + */ + u64 notify_index; + /** + * Field is now deprecated, will be removed when KMD is updated to support removal + */ + u32 enable_extra_events; + /** Zero Padding */ + u32 reserved_0; +}; + +/** + * The scheduling log notification is generated by VPU when it writes + * an event into the log buffer at the notify_index. VPU notifies host with + * VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION. This is an asynchronous + * message from VPU to host. + * @see VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION + * @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG + */ +struct vpu_ipc_msg_payload_hws_scheduling_log_notification { + /** Engine ordinal */ + u32 engine_idx; + /** Zero Padding */ + u32 reserved_0; +}; + +/** + * HWS suspend command queue request and done structure. + * Host will request the suspend of contexts and VPU will; + * - Suspend all work on this context + * - Preempt any running work + * - Asynchronously perform the above and return success immediately once + * all items above are started successfully + * - Notify the host of completion of these operations via + * VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE + * - Reject any other context operations on a context with an in-flight + * suspend request running + * Same structure used when VPU notifies host of completion of a context suspend + * request. The ids and suspend fence value reported in this command will match + * the one in the request from the host to suspend the context. Once suspend is + * complete, VPU will not access any data relating to this command queue until + * it is resumed. + * @see VPU_JSM_MSG_HWS_SUSPEND_CMDQ + * @see VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE + */ +struct vpu_ipc_msg_payload_hws_suspend_cmdq { + /** Host SSID */ + u32 host_ssid; + /** Zero Padding */ + u32 reserved_0; + /** Command queue id */ + u64 cmdq_id; + /** + * Suspend fence value - reported by the VPU suspend context + * completed once suspend is complete. + */ + u64 suspend_fence_value; +}; + +/** + * HWS Resume command queue request / response structure. + * Host will request the resume of a context; + * - VPU will resume all work on this context + * - Scheduler will allow this context to be scheduled + * @see VPU_JSM_MSG_HWS_RESUME_CMDQ + * @see VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP + */ +struct vpu_ipc_msg_payload_hws_resume_cmdq { + /** Host SSID */ + u32 host_ssid; + /** Zero Padding */ + u32 reserved_0; + /** Command queue id */ + u64 cmdq_id; +}; + +/** + * HWS Resume engine request / response structure. + * After a HWS engine reset, all scheduling is stopped on VPU until an engine resume. + * Host shall send this command to resume scheduling of any valid queue. + * @see VPU_JSM_MSG_HWS_ENGINE_RESUME + * @see VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE + */ +struct vpu_ipc_msg_payload_hws_resume_engine { + /** Engine to be resumed */ + u32 engine_idx; + /** Reserved */ + u32 reserved_0; +}; + +/** + * Payload for VPU_JSM_MSG_TRACE_SET_CONFIG[_RSP] and + * VPU_JSM_MSG_TRACE_GET_CONFIG_RSP messages. + * + * The payload is interpreted differently depending on the type of message: + * + * - For VPU_JSM_MSG_TRACE_SET_CONFIG, the payload specifies the desired + * logging configuration to be set. + * + * - For VPU_JSM_MSG_TRACE_SET_CONFIG_RSP, the payload reports the logging + * configuration that was set after a VPU_JSM_MSG_TRACE_SET_CONFIG request. + * The host can compare this payload with the one it sent in the + * VPU_JSM_MSG_TRACE_SET_CONFIG request to check whether or not the + * configuration was set as desired. + * + * - VPU_JSM_MSG_TRACE_GET_CONFIG_RSP, the payload reports the current logging + * configuration. + */ +struct vpu_ipc_msg_payload_trace_config { + /** + * Logging level (currently set or to be set); see 'mvLog_t' enum for + * acceptable values. The specified logging level applies to all + * destinations and HW components + */ + u32 trace_level; + /** + * Bitmask of logging destinations (currently enabled or to be enabled); + * bitwise OR of values defined in logging_destination enum. + */ + u32 trace_destination_mask; + /** + * Bitmask of loggable HW components (currently enabled or to be enabled); + * bitwise OR of values defined in loggable_hw_component enum. + */ + u64 trace_hw_component_mask; + u64 reserved_0; /**< Reserved for future extensions. */ +}; + +/** + * Payload for VPU_JSM_MSG_TRACE_GET_CAPABILITY_RSP messages. + */ +struct vpu_ipc_msg_payload_trace_capability_rsp { + u32 trace_destination_mask; /**< Bitmask of supported logging destinations. */ + u32 reserved_0; + u64 trace_hw_component_mask; /**< Bitmask of supported loggable HW components. */ + u64 reserved_1; /**< Reserved for future extensions. */ +}; + +/** + * Payload for VPU_JSM_MSG_TRACE_GET_NAME requests. + */ +struct vpu_ipc_msg_payload_trace_get_name { + /** + * The type of the entity to query name for; see logging_entity_type for + * possible values. + */ + u32 entity_type; + u32 reserved_0; + /** + * The ID of the entity to query name for; possible values depends on the + * entity type. + */ + u64 entity_id; +}; + +/** + * Payload for VPU_JSM_MSG_TRACE_GET_NAME_RSP responses. + */ +struct vpu_ipc_msg_payload_trace_get_name_rsp { + /** + * The type of the entity whose name was queried; see logging_entity_type + * for possible values. + */ + u32 entity_type; + u32 reserved_0; + /** + * The ID of the entity whose name was queried; possible values depends on + * the entity type. + */ + u64 entity_id; + /** Reserved for future extensions. */ + u64 reserved_1; + /** The name of the entity. */ + char entity_name[VPU_TRACE_ENTITY_NAME_MAX_LEN]; +}; + +/** + * Data sent from the VPU to the host in all metric streamer response messages + * and in asynchronous notification. + * @see VPU_JSM_MSG_METRIC_STREAMER_START_DONE + * @see VPU_JSM_MSG_METRIC_STREAMER_STOP_DONE + * @see VPU_JSM_MSG_METRIC_STREAMER_UPDATE_DONE + * @see VPU_JSM_MSG_METRIC_STREAMER_INFO_DONE + * @see VPU_JSM_MSG_METRIC_STREAMER_NOTIFICATION + */ +struct vpu_jsm_metric_streamer_done { + /** Metric group mask that identifies metric streamer instance. */ + u64 metric_group_mask; + /** + * Size in bytes of single sample - total size of all enabled counters. + * Some VPU implementations may align sample_size to more than 8 bytes. + */ + u32 sample_size; + u32 reserved_0; + /** + * Number of samples collected since the metric streamer was started. + * This will be 0 if the metric streamer was not started. + */ + u32 samples_collected; + /** + * Number of samples dropped since the metric streamer was started. This + * is incremented every time the metric streamer is not able to write + * collected samples because the current buffer is full and there is no + * next buffer to switch to. + */ + u32 samples_dropped; + /** Address of the buffer that contains the latest metric data. */ + u64 buffer_addr; + /** + * Number of bytes written into the metric data buffer. In response to the + * VPU_JSM_MSG_METRIC_STREAMER_INFO request this field contains the size of + * all group and counter descriptors. The size is updated even if the buffer + * in the request was NULL or too small to hold descriptors of all counters + */ + u64 bytes_written; +}; + +/** + * Metric group description placed in the metric buffer after successful completion + * of the VPU_JSM_MSG_METRIC_STREAMER_INFO command. This is followed by one or more + * @ref vpu_jsm_metric_counter_descriptor records. + * @see VPU_JSM_MSG_METRIC_STREAMER_INFO + */ +struct vpu_jsm_metric_group_descriptor { + /** + * Offset to the next metric group (8-byte aligned). If this offset is 0 this + * is the last descriptor. The value of metric_info_size must be greater than + * or equal to sizeof(struct vpu_jsm_metric_group_descriptor) + name_string_size + * + description_string_size and must be 8-byte aligned. + */ + u32 next_metric_group_info_offset; + /** + * Offset to the first metric counter description record (8-byte aligned). + * @see vpu_jsm_metric_counter_descriptor + */ + u32 next_metric_counter_info_offset; + /** Index of the group. This corresponds to bit index in metric_group_mask. */ + u32 group_id; + /** Number of counters in the metric group. */ + u32 num_counters; + /** Data size for all counters, must be a multiple of 8 bytes.*/ + u32 metric_group_data_size; + /** + * Metric group domain number. Cannot use multiple, simultaneous metric groups + * from the same domain. + */ + u32 domain; + /** + * Counter name string size. The string must include a null termination character. + * The FW may use a fixed size name or send a different name for each counter. + * If the VPU uses fixed size strings, all characters from the end of the name + * to the of the fixed size character array must be zeroed. + */ + u32 name_string_size; + /** Counter description string size, @see name_string_size */ + u32 description_string_size; + u64 reserved_0; + /** + * Right after this structure, the VPU writes name and description of + * the metric group. + */ +}; + +/** + * Metric counter description, placed in the buffer after vpu_jsm_metric_group_descriptor. + * @see VPU_JSM_MSG_METRIC_STREAMER_INFO + */ +struct vpu_jsm_metric_counter_descriptor { + /** + * Offset to the next counter in a group (8-byte aligned). If this offset is + * 0 this is the last counter in the group. + */ + u32 next_metric_counter_info_offset; + /** + * Offset to the counter data from the start of samples in this metric group. + * Note that metric_data_offset % metric_data_size must be 0. + */ + u32 metric_data_offset; + /** Size of the metric counter data in bytes. */ + u32 metric_data_size; + /** Metric type, see Level Zero API for definitions. */ + u32 tier; + /** Metric type, see set_metric_type_t for definitions. */ + u32 metric_type; + /** Metric type, see set_value_type_t for definitions. */ + u32 metric_value_type; + /** + * Counter name string size. The string must include a null termination character. + * The FW may use a fixed size name or send a different name for each counter. + * If the VPU uses fixed size strings, all characters from the end of the name + * to the of the fixed size character array must be zeroed. + */ + u32 name_string_size; + /** Counter description string size, @see name_string_size */ + u32 description_string_size; + /** Counter component name string size, @see name_string_size */ + u32 component_string_size; + /** Counter string size, @see name_string_size */ + u32 units_string_size; + u64 reserved_0; + /** + * Right after this structure, the VPU writes name, description + * component and unit strings. + */ +}; + +/** + * Payload for @ref VPU_JSM_MSG_DYNDBG_CONTROL requests. + * + * VPU_JSM_MSG_DYNDBG_CONTROL requests are used to control the VPU FW dynamic debug + * feature, which allows developers to selectively enable/disable code to obtain + * additional FW information. This is equivalent to the dynamic debug functionality + * provided by Linux. The host can control dynamic debug behavior by sending dyndbg + * commands, using the same syntax as for Linux dynamic debug commands. + * + * @see https://www.kernel.org/doc/html/latest/admin-guide/dynamic-debug-howto.html. + * + * NOTE: + * As the dynamic debug feature uses MVLOG messages to provide information, the host + * must first set the logging level to MVLOG_DEBUG, using the @ref VPU_JSM_MSG_TRACE_SET_CONFIG + * command. + */ +struct vpu_ipc_msg_payload_dyndbg_control { + /** + * Dyndbg command to be executed. + */ + char dyndbg_cmd[VPU_DYNDBG_CMD_MAX_LEN]; +}; + +/** + * Payload for VPU_JSM_MSG_PWR_D0I3_ENTER + * + * This is a bi-directional payload. + */ +struct vpu_ipc_msg_payload_pwr_d0i3_enter { + /** + * 0: VPU_JSM_MSG_PWR_D0I3_ENTER_DONE is not sent to the host driver + * The driver will poll for D0i2 Idle state transitions. + * 1: VPU_JSM_MSG_PWR_D0I3_ENTER_DONE is sent after VPU state save is complete + */ + u32 send_response; + u32 reserved_0; +}; + +/** + * Payload for @ref VPU_JSM_MSG_DCT_ENABLE message. + * + * Default values for DCT active/inactive times are 5.3ms and 30ms respectively, + * corresponding to a 85% duty cycle. This payload allows the host to tune these + * values according to application requirements. + */ +struct vpu_ipc_msg_payload_pwr_dct_control { + /** Duty cycle active time in microseconds */ + u32 dct_active_us; + /** Duty cycle inactive time in microseconds */ + u32 dct_inactive_us; +}; + +/** + * Payload for @ref VPU_JSM_MSG_FREQ_CONFIG message. + * + * This payload allows the host to configure the VPU frequency scaling parameters. + */ +struct vpu_ipc_msg_payload_freq_config { + /** Minimum frequency PLL ratio */ + u32 min_freq_pll_ratio; + /** Efficiency frequency PLL ratio */ + u32 pn_freq_pll_ratio; + /** Maximum frequency PLL ratio */ + u32 max_freq_pll_ratio; + /** Reserved for 64-bit alignment */ + u32 reserved_0; +}; + +/* + * Payloads union, used to define complete message format. + */ +union vpu_ipc_msg_payload { + struct vpu_ipc_msg_payload_engine_reset engine_reset; + struct vpu_ipc_msg_payload_engine_preempt engine_preempt; + struct vpu_ipc_msg_payload_register_db register_db; + struct vpu_ipc_msg_payload_unregister_db unregister_db; + struct vpu_ipc_msg_payload_query_engine_hb query_engine_hb; + struct vpu_ipc_msg_payload_power_level power_level; + struct vpu_jsm_metric_streamer_start metric_streamer_start; + struct vpu_jsm_metric_streamer_stop metric_streamer_stop; + struct vpu_jsm_metric_streamer_update metric_streamer_update; + struct vpu_ipc_msg_payload_ssid_release ssid_release; + struct vpu_jsm_hws_register_db hws_register_db; + struct vpu_ipc_msg_payload_job_done job_done; + struct vpu_ipc_msg_payload_native_fence_signalled native_fence_signalled; + struct vpu_ipc_msg_payload_engine_reset_done engine_reset_done; + struct vpu_ipc_msg_payload_engine_preempt_done engine_preempt_done; + struct vpu_ipc_msg_payload_register_db_done register_db_done; + struct vpu_ipc_msg_payload_unregister_db_done unregister_db_done; + struct vpu_ipc_msg_payload_query_engine_hb_done query_engine_hb_done; + struct vpu_ipc_msg_payload_get_power_level_count_done get_power_level_count_done; + struct vpu_jsm_metric_streamer_done metric_streamer_done; + struct vpu_ipc_msg_payload_trace_config trace_config; + struct vpu_ipc_msg_payload_trace_capability_rsp trace_capability; + struct vpu_ipc_msg_payload_trace_get_name trace_get_name; + struct vpu_ipc_msg_payload_trace_get_name_rsp trace_get_name_rsp; + struct vpu_ipc_msg_payload_dyndbg_control dyndbg_control; + struct vpu_ipc_msg_payload_hws_priority_band_setup hws_priority_band_setup; + struct vpu_ipc_msg_payload_hws_create_cmdq hws_create_cmdq; + struct vpu_ipc_msg_payload_hws_create_cmdq_rsp hws_create_cmdq_rsp; + struct vpu_ipc_msg_payload_hws_destroy_cmdq hws_destroy_cmdq; + struct vpu_ipc_msg_payload_hws_set_context_sched_properties + hws_set_context_sched_properties; + struct vpu_ipc_msg_payload_hws_set_scheduling_log hws_set_scheduling_log; + struct vpu_ipc_msg_payload_hws_scheduling_log_notification hws_scheduling_log_notification; + struct vpu_ipc_msg_payload_hws_suspend_cmdq hws_suspend_cmdq; + struct vpu_ipc_msg_payload_hws_resume_cmdq hws_resume_cmdq; + struct vpu_ipc_msg_payload_hws_resume_engine hws_resume_engine; + struct vpu_ipc_msg_payload_pwr_d0i3_enter pwr_d0i3_enter; + struct vpu_ipc_msg_payload_pwr_dct_control pwr_dct_control; + struct vpu_ipc_msg_payload_freq_config freq_config; +}; + +/** + * Host <-> NPU IPC message base structure. + * + * NOTE: All instances of this object must be aligned on a 64B boundary + * to allow proper handling of VPU cache operations. + */ +struct vpu_jsm_msg { + /** Reserved */ + u64 reserved_0; + /** Message type, see @ref vpu_ipc_msg_type. */ + u32 type; + /** Buffer status, see @ref vpu_ipc_msg_status. */ + u32 status; + /** + * Request ID, provided by the host in a request message and passed + * back by VPU in the response message. + */ + u32 request_id; + /** Request return code set by the VPU, see VPU_JSM_STATUS_* defines. */ + u32 result; + u64 reserved_1; + /** Message payload depending on message type, see vpu_ipc_msg_payload union. */ + union vpu_ipc_msg_payload payload; +}; + +#pragma pack(pop) + +#endif + +///@} -- cgit v1.3.1