From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- drivers/gpu/drm/tyr/Kconfig | 26 + drivers/gpu/drm/tyr/Makefile | 3 + drivers/gpu/drm/tyr/driver.rs | 237 ++++ drivers/gpu/drm/tyr/file.rs | 65 ++ drivers/gpu/drm/tyr/fw.rs | 321 ++++++ drivers/gpu/drm/tyr/fw/parser.rs | 588 ++++++++++ drivers/gpu/drm/tyr/gem.rs | 160 +++ drivers/gpu/drm/tyr/gpu.rs | 175 +++ drivers/gpu/drm/tyr/mmu.rs | 120 ++ drivers/gpu/drm/tyr/mmu/address_space.rs | 511 +++++++++ drivers/gpu/drm/tyr/regs.rs | 1779 ++++++++++++++++++++++++++++++ drivers/gpu/drm/tyr/slot.rs | 404 +++++++ drivers/gpu/drm/tyr/tyr.rs | 26 + drivers/gpu/drm/tyr/vm.rs | 950 ++++++++++++++++ 14 files changed, 5365 insertions(+) create mode 100644 drivers/gpu/drm/tyr/Kconfig create mode 100644 drivers/gpu/drm/tyr/Makefile create mode 100644 drivers/gpu/drm/tyr/driver.rs create mode 100644 drivers/gpu/drm/tyr/file.rs create mode 100644 drivers/gpu/drm/tyr/fw.rs create mode 100644 drivers/gpu/drm/tyr/fw/parser.rs create mode 100644 drivers/gpu/drm/tyr/gem.rs create mode 100644 drivers/gpu/drm/tyr/gpu.rs create mode 100644 drivers/gpu/drm/tyr/mmu.rs create mode 100644 drivers/gpu/drm/tyr/mmu/address_space.rs create mode 100644 drivers/gpu/drm/tyr/regs.rs create mode 100644 drivers/gpu/drm/tyr/slot.rs create mode 100644 drivers/gpu/drm/tyr/tyr.rs create mode 100644 drivers/gpu/drm/tyr/vm.rs (limited to 'drivers/gpu/drm/tyr') diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig new file mode 100644 index 000000000..8f13e49f1 --- /dev/null +++ b/drivers/gpu/drm/tyr/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0 or MIT + +config DRM_TYR + tristate "Tyr (Rust DRM support for ARM Mali CSF-based GPUs)" + depends on DRM=y + depends on RUST + depends on ARM || ARM64 || COMPILE_TEST + depends on MMU + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE + depends on COMMON_CLK + depends on IOMMU_SUPPORT + default n + select IOMMU_IO_PGTABLE_LPAE + select RUST_DRM_GEM_SHMEM_HELPER + select RUST_DRM_GPUVM + select RUST_FW_LOADER_ABSTRACTIONS + help + Rust DRM driver for ARM Mali CSF-based GPUs. + + This driver is for Mali (or Immortalis) Valhall Gxxx GPUs. + + Note that the Mali-G68 and Mali-G78, while Valhall architecture, will + be supported with the panfrost driver as they are not CSF GPUs. + + if M is selected, the module will be called tyr. This driver is work + in progress and may not be functional. diff --git a/drivers/gpu/drm/tyr/Makefile b/drivers/gpu/drm/tyr/Makefile new file mode 100644 index 000000000..ba545f65f --- /dev/null +++ b/drivers/gpu/drm/tyr/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 or MIT + +obj-$(CONFIG_DRM_TYR) += tyr.o diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs new file mode 100644 index 000000000..bfb0ba19c --- /dev/null +++ b/drivers/gpu/drm/tyr/driver.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +use kernel::{ + clk::{ + Clk, + OptionalClk, // + }, + device::{ + Bound, + Core, + Device, + DeviceContext, // + }, + dma::{ + Device as DmaDevice, + DmaMask, // + }, + drm, + drm::ioctl, + io::{ + poll, + Io, // + }, + new_mutex, + of, + platform, + prelude::*, + regulator, + regulator::Regulator, + sizes::SZ_2M, + sync::{ + Arc, + Mutex, // + }, + time, // +}; + +use crate::{ + file::TyrDrmFileData, + fw::Firmware, + gem::Bo, + gpu, + gpu::GpuInfo, + mmu::Mmu, + regs::gpu_control::*, // +}; + +pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>; + +pub(crate) struct TyrDrmDriver; + +/// Convenience type alias for the DRM device type for this driver. +pub(crate) type TyrDrmDevice = drm::Device; + +pub(crate) struct TyrPlatformDriver; + +#[pin_data(PinnedDrop)] +pub(crate) struct TyrPlatformDriverData<'bound> { + _reg: drm::Registration<'bound, TyrDrmDriver>, +} + +/// Data owned by the DRM [`Registration`]. +/// +/// This data can have references tied to the parent platform device binding scope +/// and is accessible only while the DRM device is registered with userspace. +#[pin_data] +pub(crate) struct TyrDrmRegistrationData<'drm> { + /// Parent platform device. + pub(crate) pdev: &'drm platform::Device, + + /// Firmware sections. + pub(crate) fw: Firmware<'drm>, + + #[pin] + clks: Mutex, + + #[pin] + regulators: Mutex, + + /// GPU MMIO register mapping. + pub(crate) iomem: Arc>, + + /// GPU information read from hardware during probe. + pub(crate) gpu_info: GpuInfo, +} + +fn issue_soft_reset(dev: &Device, iomem: &IoMem<'_>) -> Result { + iomem.write_reg(GPU_COMMAND::reset(ResetMode::SoftReset)); + + poll::read_poll_timeout( + || Ok(iomem.read(GPU_IRQ_RAWSTAT)), + |status| status.reset_completed(), + time::Delta::from_millis(1), + time::Delta::from_millis(100), + ) + .inspect_err(|_| dev_err!(dev, "GPU reset failed."))?; + + Ok(()) +} + +kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [ + (of::DeviceId::new(c"rockchip,rk3588-mali"), ()), + (of::DeviceId::new(c"arm,mali-valhall-csf"), ()) + ] +); + +impl platform::Driver for TyrPlatformDriver { + type IdInfo = (); + type Data<'bound> = TyrPlatformDriverData<'bound>; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; + let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?; + let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?; + + core_clk.prepare_enable()?; + stacks_clk.prepare_enable()?; + coregroup_clk.prepare_enable()?; + + let mali_regulator = Regulator::::get(pdev.as_ref(), c"mali")?; + let sram_regulator = Regulator::::get(pdev.as_ref(), c"sram")?; + + let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + + let iomem = Arc::new(request.iomap_sized::()?, GFP_KERNEL)?; + + issue_soft_reset(pdev.as_ref(), &iomem)?; + gpu::l2_power_on(pdev.as_ref(), &iomem)?; + + let gpu_info = GpuInfo::new(&iomem); + gpu_info.log(pdev.as_ref()); + + let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features) + .pa_bits() + .get(); + // SAFETY: No concurrent DMA allocations or mappings can be made because + // the device is still being probed and therefore isn't being used by + // other threads of execution. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? }; + + let unreg_dev = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + + let mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?; + + let firmware = Firmware::new( + pdev.as_ref(), + iomem.clone(), + &unreg_dev, + mmu.as_arc_borrow(), + &gpu_info, + )?; + + firmware.boot()?; + + let reg_data = pin_init!(TyrDrmRegistrationData { + pdev, + fw: firmware, + clks <- new_mutex!(Clocks { + core: core_clk, + stacks: stacks_clk, + coregroup: coregroup_clk, + }), + regulators <- new_mutex!(Regulators { + _mali: mali_regulator, + _sram: sram_regulator, + }), + iomem, + gpu_info, + }); + + // SAFETY: `reg` is stored in `TyrPlatformDriverData` and dropped when the driver is + // unbound; it is never forgotten. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), unreg_dev, reg_data, 0)? }; + + let driver = TyrPlatformDriverData { _reg: reg }; + + dev_dbg!(pdev, "Tyr initialized correctly."); + Ok(driver) + } +} + +#[pinned_drop] +impl PinnedDrop for TyrPlatformDriverData<'_> { + fn drop(self: Pin<&mut Self>) {} +} + +// We need to retain the name "panthor" to achieve drop-in compatibility with +// the C driver in the userspace stack. +const INFO: drm::DriverInfo = drm::DriverInfo { + major: 1, + minor: 5, + patchlevel: 0, + name: c"panthor", + desc: c"ARM Mali Tyr DRM driver", +}; + +#[vtable] +impl drm::Driver for TyrDrmDriver { + type Data = (); + type RegistrationData<'drm> = TyrDrmRegistrationData<'drm>; + type File = TyrDrmFileData; + type Object = Bo; + type ParentDevice = platform::Device; + + const INFO: drm::DriverInfo = INFO; + const FEAT_RENDER: bool = true; + + kernel::declare_drm_ioctls! { + (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, TyrDrmFileData::dev_query), + } +} + +struct Clocks { + core: Clk, + stacks: OptionalClk, + coregroup: OptionalClk, +} + +impl Drop for Clocks { + fn drop(&mut self) { + self.core.disable_unprepare(); + self.stacks.disable_unprepare(); + self.coregroup.disable_unprepare(); + } +} + +struct Regulators { + _mali: Regulator, + _sram: Regulator, +} diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs new file mode 100644 index 000000000..9f60a90d4 --- /dev/null +++ b/drivers/gpu/drm/tyr/file.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +use kernel::{ + drm::{ + self, + Registered, // + }, + prelude::*, + uaccess::UserSlice, + uapi, // +}; + +use crate::driver::{ + TyrDrmDevice, + TyrDrmDriver, + TyrDrmRegistrationData, // +}; + +#[pin_data] +pub(crate) struct TyrDrmFileData {} + +/// Convenience type alias for our DRM `File` type +pub(crate) type TyrDrmFile = drm::file::File; + +impl drm::file::DriverFile for TyrDrmFileData { + type Driver = TyrDrmDriver; + + fn open(_dev: &drm::Device) -> Result>> { + KBox::try_pin_init(try_pin_init!(Self {}), GFP_KERNEL) + } +} + +impl TyrDrmFileData { + pub(crate) fn dev_query( + _ddev: &TyrDrmDevice, + reg_data: &TyrDrmRegistrationData<'_>, + devquery: &mut uapi::drm_panthor_dev_query, + _file: &TyrDrmFile, + ) -> Result { + if devquery.pointer == 0 { + match devquery.type_ { + uapi::drm_panthor_dev_query_type_DRM_PANTHOR_DEV_QUERY_GPU_INFO => { + devquery.size = core::mem::size_of_val(®_data.gpu_info) as u32; + Ok(0) + } + _ => Err(EINVAL), + } + } else { + match devquery.type_ { + uapi::drm_panthor_dev_query_type_DRM_PANTHOR_DEV_QUERY_GPU_INFO => { + let mut writer = UserSlice::new( + UserPtr::from_addr(devquery.pointer as usize), + devquery.size as usize, + ) + .writer(); + + writer.write(®_data.gpu_info)?; + + Ok(0) + } + _ => Err(EINVAL), + } + } + } +} diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs new file mode 100644 index 000000000..47d25c901 --- /dev/null +++ b/drivers/gpu/drm/tyr/fw.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Firmware loading and management for Mali CSF GPU. +//! +//! This module handles loading the Mali GPU firmware binary, parsing it into sections, +//! and mapping those sections into the MCU's virtual address space. Each firmware section +//! has specific properties (read/write/execute permissions, cache modes) and must be loaded +//! at specific virtual addresses expected by the MCU. +//! +//! See [`Firmware`] for the main firmware management interface and [`Section`] for +//! individual firmware sections. +//! +//! [`Firmware`]: crate::fw::Firmware +//! [`Section`]: crate::fw::Section + +use kernel::{ + device::{ + Bound, + Device, // + }, + drm::{ + gem::BaseObject, // + }, + io::{ + poll, + Io, // + }, + num::Bounded, + prelude::*, + register, + str::CString, + sync::{ + Arc, + ArcBorrow, // + }, + time, // +}; + +use crate::{ + driver::{ + IoMem, + TyrDrmDevice, // + }, + fw::parser::{ + FwParser, + ParsedSection, // + }, + gem, + gem::{ + KernelBo, + KernelBoVaAlloc, // + }, + gpu::GpuInfo, + + mmu::Mmu, + regs::{ + gpu_control::{ + McuControlMode, + McuStatus, + GPU_ID, + MCU_CONTROL, + MCU_STATUS, // + }, // + job_control::{ + JOB_IRQ_CLEAR, + JOB_IRQ_RAWSTAT, // + }, // + }, + vm::Vm, // +}; + +mod parser; + +pub(super) const CSF_MCU_SHARED_REGION_START: u32 = 0x04000000; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(super) enum CacheMode { + None = 0, + Cached = 1, + UncachedCoherent = 2, + CachedCoherent = 3, +} + +impl From> for CacheMode { + fn from(value: Bounded) -> Self { + match value.get() { + 0 => Self::None, + 1 => Self::Cached, + 2 => Self::UncachedCoherent, + 3 => Self::CachedCoherent, + _ => unreachable!(), + } + } +} + +impl From for Bounded { + fn from(value: CacheMode) -> Self { + Bounded::try_new(value as u32).unwrap() + } +} + +register! { + #[allow(non_upper_case_globals)] + pub(super) SectionFlags(u32) @ 0x0 { + 0:0 read => bool; + 1:1 write => bool; + 2:2 exec => bool; + 4:3 cache_mode => CacheMode; + 5:5 prot => bool; + 30:30 shared => bool; + 31:31 zero => bool; + } +} + +impl SectionFlags { + const VALID_MASK: u32 = Self::READ_MASK + | Self::WRITE_MASK + | Self::EXEC_MASK + | Self::CACHE_MODE_MASK + | Self::PROT_MASK + | Self::SHARED_MASK + | Self::ZERO_MASK; + + fn try_from_fw(value: u32) -> Result { + if value & !Self::VALID_MASK != 0 { + Err(EINVAL) + } else { + Ok(Self::from_raw(value)) + } + } +} + +/// A parsed section of the firmware binary. +struct Section<'drm> { + // Raw firmware section data for reset purposes + #[expect(dead_code)] + data: KVec, + + // Keep the BO backing this firmware section so that both the + // GPU mapping and CPU mapping remain valid until the Section is dropped. + #[expect(dead_code)] + mem: gem::KernelBo<'drm>, +} + +/// Loaded firmware with sections mapped into MCU VM. +pub(crate) struct Firmware<'drm> { + /// Iomem need to access registers. + iomem: Arc>, + + /// MCU VM. + vm: Arc>, + + /// List of firmware sections. + #[expect(dead_code)] + sections: KVec>, +} + +impl<'drm> Drop for Firmware<'drm> { + fn drop(&mut self) { + // Stop the MCU before releasing its firmware mappings and memory. + let _ = self.stop(); + + // AS slots retain a VM ref, we need to kill the circular ref manually. + self.vm.kill(); + } +} + +impl<'drm> Firmware<'drm> { + fn init_section_mem(dev: &Device, mem: &mut KernelBo<'drm>, data: &KVec) -> Result { + if data.is_empty() { + return Ok(()); + } + + let vmap = mem.bo().vmap::<0>()?; + let size = mem.bo().size(); + + if data.len() > size { + dev_err!(dev, "fw section {} bigger than BO {}", data.len(), size); + return Err(EINVAL); + } + + for (i, &byte) in data.iter().enumerate() { + vmap.try_write8(byte, i)?; + } + + Ok(()) + } + + fn request(ddev: &TyrDrmDevice, gpu_info: &GpuInfo) -> Result { + let gpu_id = GPU_ID::from_raw(gpu_info.gpu_id); + + let path = CString::try_from_fmt(fmt!( + "arm/mali/arch{}.{}/mali_csffw.bin", + gpu_id.arch_major().get(), + gpu_id.arch_minor().get() + ))?; + + kernel::firmware::Firmware::request(&path, ddev.as_ref().as_ref()) + } + + fn load( + dev: &Device, + ddev: &TyrDrmDevice, + gpu_info: &GpuInfo, + ) -> Result<(kernel::firmware::Firmware, KVec)> { + let fw = Self::request(ddev, gpu_info)?; + let mut parser = FwParser::new(dev, fw.data()); + + let parsed_sections = parser.parse()?; + + Ok((fw, parsed_sections)) + } + + /// Load firmware and map sections into MCU VM. + pub(crate) fn new( + dev: &'drm Device, + iomem: Arc>, + ddev: &TyrDrmDevice, + mmu: ArcBorrow<'_, Mmu<'drm>>, + gpu_info: &GpuInfo, + ) -> Result> { + let vm = Vm::new(dev, ddev, mmu, gpu_info)?; + vm.activate()?; + + let result = (|| { + let (fw, parsed_sections) = Self::load(dev, ddev, gpu_info)?; + let mut sections = KVec::new(); + for parsed in parsed_sections { + let size = u64::from(parsed.va.end.checked_sub(parsed.va.start).ok_or(EINVAL)?); + + let va = u64::from(parsed.va.start); + + let mut mem = KernelBo::new( + ddev, + vm.clone(), + size, + KernelBoVaAlloc::Explicit(va), + parsed.vm_map_flags, + )?; + + let section_start = parsed.data_range.start as usize; + let section_end = parsed.data_range.end as usize; + let mut data = KVec::new(); + + // Ensure that the firmware slice is not out of bounds. + let fw_data = fw.data(); + let bytes = fw_data.get(section_start..section_end).ok_or(EINVAL)?; + data.extend_from_slice(bytes, GFP_KERNEL)?; + + Self::init_section_mem(dev, &mut mem, &data)?; + + sections.push(Section { data, mem }, GFP_KERNEL)?; + } + + Ok(Firmware { + iomem, + vm: vm.clone(), + sections, + }) + })(); + + if result.is_err() { + vm.kill(); + } + + result + } + + pub(crate) fn boot(&self) -> Result { + let io = &self.iomem; + + // Discard any stale global interrupt. + io.write_reg(JOB_IRQ_CLEAR::zeroed().with_glb(true)); + + io.write_reg(MCU_CONTROL::zeroed().with_req(McuControlMode::Auto)); + + if let Err(e) = poll::read_poll_timeout( + || Ok((io.read(MCU_STATUS), io.read(JOB_IRQ_RAWSTAT))), + |(mcu_status, irq_rawstat)| { + mcu_status.value() == McuStatus::Enabled && irq_rawstat.glb() + }, + time::Delta::from_millis(1), + time::Delta::from_millis(100), + ) { + let status = io.read(MCU_STATUS); + dev_err!( + self.vm.dev(), + "MCU failed to boot, status: {:?}", + status.value() + ); + return Err(e); + } + + io.write_reg(JOB_IRQ_CLEAR::zeroed().with_glb(true)); + + Ok(()) + } + + fn stop(&self) -> Result { + let io = &self.iomem; + io.write_reg(MCU_CONTROL::zeroed().with_req(McuControlMode::Disable)); + + if let Err(e) = poll::read_poll_timeout( + || Ok(io.read(MCU_STATUS)), + |status| status.value() == McuStatus::Disabled, + time::Delta::from_micros(10), + time::Delta::from_millis(100), + ) { + let status = io.read(MCU_STATUS); + dev_err!( + self.vm.dev(), + "MCU failed to stop, status: {:?}", + status.value() + ); + return Err(e); + } + + Ok(()) + } +} diff --git a/drivers/gpu/drm/tyr/fw/parser.rs b/drivers/gpu/drm/tyr/fw/parser.rs new file mode 100644 index 000000000..c4d0ad1d7 --- /dev/null +++ b/drivers/gpu/drm/tyr/fw/parser.rs @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Firmware binary parser for Mali CSF (Command Stream Frontend) GPU. +//! +//! This module implements a parser for the Mali GPU firmware binary format. The firmware +//! file contains a header followed by a sequence of entries, each describing how to load +//! firmware sections into the MCU (Microcontroller Unit) memory. The parser extracts section +//! metadata including: +//! - Virtual address ranges where sections should be mapped +//! - Data ranges (byte offsets) within the firmware binary +//! - Section flags (permissions, cache modes) + +use core::{ + mem::size_of, + ops::Range, // +}; + +use kernel::{ + bits::bit_u32, + device::Device, + prelude::*, + sizes::SZ_4K, // +}; + +use crate::{ + fw::{ + CacheMode, + SectionFlags, + CSF_MCU_SHARED_REGION_START, // + }, + vm::{ + VmFlag, + VmMapFlags, // + }, // +}; + +/// A parsed firmware section ready for loading into MCU memory. +/// +/// Represents a single firmware section extracted from the firmware binary, containing +/// all information needed to map the section's data into the MCU's virtual address space. +pub(super) struct ParsedSection { + /// Byte offset range within the firmware binary where this section's data resides. + pub(super) data_range: Range, + /// MCU virtual address range where this section should be mapped. + pub(super) va: Range, + /// Memory protection and caching flags for the mapping. + pub(super) vm_map_flags: VmMapFlags, +} + +/// A bare-bones `std::io::Cursor<[u8]>` clone to keep track of the current position in the +/// firmware binary. +/// +/// Provides methods to sequentially read primitive types and byte arrays from the firmware +/// binary while maintaining the current read position. +struct Cursor<'a> { + dev: &'a Device, + data: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(dev: &'a Device, data: &'a [u8]) -> Self { + Self { dev, data, pos: 0 } + } + + fn len(&self) -> usize { + self.data.len() + } + + fn pos(&self) -> usize { + self.pos + } + + /// Returns a view into the cursor's data. + /// + /// This spawns a new cursor, leaving the current cursor unchanged. + fn view(&self, range: Range) -> Result> { + if range.start < self.pos || range.end > self.data.len() { + dev_err!( + self.dev, + "Invalid cursor range {:?} for data of length {}", + range, + self.data.len() + ); + + Err(EINVAL) + } else { + Ok(Self { + dev: self.dev, + data: &self.data[range], + pos: 0, + }) + } + } + + /// Reads a slice of bytes from the current position and advances the cursor. + /// + /// Returns an error if the read would exceed the data bounds. + fn read(&mut self, nbytes: usize) -> Result<&[u8]> { + let start = self.pos; + let end = start + nbytes; + + if end > self.data.len() { + dev_err!( + self.dev, + "Invalid firmware file: read of size {} at position {} is out of bounds", + nbytes, + start, + ); + return Err(EINVAL); + } + + self.pos += nbytes; + Ok(&self.data[start..end]) + } + + /// Reads a little-endian `u8` from the current position and advances the cursor. + fn read_u8(&mut self) -> Result { + let bytes = self.read(size_of::())?; + Ok(bytes[0]) + } + + /// Reads a little-endian `u16` from the current position and advances the cursor. + fn read_u16(&mut self) -> Result { + let bytes: [u8; 2] = self + .read(size_of::())? + .try_into() + .map_err(|_| EINVAL)?; + + Ok(u16::from_le_bytes(bytes)) + } + + /// Reads a little-endian `u32` from the current position and advances the cursor. + fn read_u32(&mut self) -> Result { + let bytes: [u8; 4] = self + .read(size_of::())? + .try_into() + .map_err(|_| EINVAL)?; + + Ok(u32::from_le_bytes(bytes)) + } + + /// Advances the cursor position by the specified number of bytes. + /// + /// Returns an error if the advance would exceed the data bounds. + fn advance(&mut self, nbytes: usize) -> Result { + if self.pos + nbytes > self.data.len() { + dev_err!( + self.dev, + "Invalid firmware file: advance of size {} at position {} is out of bounds", + nbytes, + self.pos, + ); + return Err(EINVAL); + } + self.pos += nbytes; + Ok(()) + } +} + +/// Parser for Mali CSF GPU firmware binaries. +/// +/// Parses the firmware binary format, extracting section metadata including virtual +/// address ranges, data offsets, and memory protection flags needed to load firmware +/// into the MCU's memory. +pub(super) struct FwParser<'a> { + cursor: Cursor<'a>, +} + +impl<'a> FwParser<'a> { + /// Creates a new firmware parser for the given firmware binary data. + pub(super) fn new(dev: &'a Device, data: &'a [u8]) -> Self { + Self { + cursor: Cursor::new(dev, data), + } + } + + /// Parses the firmware binary and returns a collection of parsed sections. + /// + /// This method validates the firmware header and iterates through all entries + /// in the binary, extracting section information needed for loading. + pub(super) fn parse(&mut self) -> Result> { + let fw_header = self.parse_fw_header()?; + let header_end = fw_header.size as usize; + + let mut parsed_sections = KVec::new(); + while self.cursor.pos() < header_end { + let entry_section = self.parse_entry(header_end)?; + + if let Some(inner) = entry_section.inner { + parsed_sections.push(inner, GFP_KERNEL)?; + } + } + + if parsed_sections.is_empty() { + dev_err!(self.cursor.dev, "Firmware contains no loadable sections"); + return Err(EINVAL); + } + + Ok(parsed_sections) + } + + fn parse_fw_header(&mut self) -> Result { + let fw_header: FirmwareHeader = match FirmwareHeader::new(&mut self.cursor) { + Ok(fw_header) => fw_header, + Err(e) => { + dev_err!(self.cursor.dev, "Invalid firmware file: {}", e.to_errno()); + return Err(e); + } + }; + + if fw_header.size as usize > self.cursor.len() { + dev_err!(self.cursor.dev, "Firmware image is truncated"); + return Err(EINVAL); + } + Ok(fw_header) + } + + fn parse_entry(&mut self, header_end: usize) -> Result { + let entry_start = self.cursor.pos(); + + let entry_header_end = entry_start + .checked_add(size_of::()) + .ok_or(EINVAL)?; + + if entry_header_end > header_end { + dev_err!( + self.cursor.dev, + "Firmware entry header at {:#x} exceeds header region ending at {:#x}", + entry_start, + header_end + ); + return Err(EINVAL); + } + + let entry_section = EntrySection { + entry_hdr: EntryHeader(self.cursor.read_u32()?), + inner: None, + }; + + let firmware_size = self.cursor.len(); + let entry_size = entry_section.entry_hdr.size() as usize; + + if self.cursor.pos() % size_of::() != 0 + || entry_size % size_of::() != 0 + || entry_size < size_of::() + { + dev_err!( + self.cursor.dev, + "Firmware entry isn't 32 bit aligned, offset={:#x} size={:#x}", + self.cursor.pos() - size_of::(), + entry_size + ); + return Err(EINVAL); + } + + let entry_end = entry_start.checked_add(entry_size).ok_or(EINVAL)?; + + if entry_end > header_end { + dev_err!( + self.cursor.dev, + "Firmware entry at {:#x} extends beyond header region ending at {:#x}", + entry_start, + header_end + ); + return Err(EINVAL); + } + + let section_hdr_size = entry_size - size_of::(); + + let entry_section = { + let mut entry_cursor = self.cursor.view(self.cursor.pos()..entry_end)?; + + match entry_section.entry_hdr.entry_type() { + Ok(EntryType::Iface) => Ok(EntrySection { + entry_hdr: entry_section.entry_hdr, + inner: Self::parse_section_entry(&mut entry_cursor, firmware_size)?, + }), + Ok( + EntryType::Config + | EntryType::FutfTest + | EntryType::TraceBuffer + | EntryType::TimelineMetadata + | EntryType::BuildInfoMetadata, + ) => Ok(entry_section), + + Err(_) => { + if entry_section.entry_hdr.optional() { + Ok(entry_section) + } else { + dev_err!( + self.cursor.dev, + "Failed to handle firmware entry type: {}", + entry_section.entry_hdr.entry_type_raw() + ); + Err(EINVAL) + } + } + } + }; + + if entry_section.is_ok() { + self.cursor.advance(section_hdr_size)?; + } + + entry_section + } + + fn parse_section_entry( + entry_cursor: &mut Cursor<'_>, + firmware_size: usize, + ) -> Result> { + let section_hdr: SectionHeader = SectionHeader::new(entry_cursor)?; + + if section_hdr.data.end < section_hdr.data.start { + dev_err!( + entry_cursor.dev, + "Firmware corrupted, data.end < data.start (0x{:x} < 0x{:x})", + section_hdr.data.end, + section_hdr.data.start + ); + return Err(EINVAL); + } + + if section_hdr.data.end as usize > firmware_size { + dev_err!( + entry_cursor.dev, + "Firmware data range {:#x}..{:#x} exceeds firmware size {:#x}", + section_hdr.data.start, + section_hdr.data.end, + firmware_size, + ); + return Err(EINVAL); + } + + if section_hdr.va.start as usize % SZ_4K != 0 || section_hdr.va.end as usize % SZ_4K != 0 { + dev_err!( + entry_cursor.dev, + "Firmware virtual address range {:#x}..{:#x} is not page aligned", + section_hdr.va.start, + section_hdr.va.end + ); + return Err(EINVAL); + } + + if section_hdr.section_flags.prot() { + dev_dbg!( + entry_cursor.dev, + "Firmware protected mode entry not supported, ignoring" + ); + return Ok(None); + } + + if section_hdr.va.start == CSF_MCU_SHARED_REGION_START + && !section_hdr.section_flags.shared() + { + dev_err!( + entry_cursor.dev, + "Interface at 0x{:x} must be shared", + CSF_MCU_SHARED_REGION_START + ); + return Err(EINVAL); + } + + if section_hdr.va.is_empty() { + return Ok(None); + } + + let mut vm_map_flags = VmMapFlags::empty(); + + if !section_hdr.section_flags.write() { + vm_map_flags |= VmFlag::Readonly; + } + + if !section_hdr.section_flags.exec() { + vm_map_flags |= VmFlag::Noexec; + } + + // TODO: As in Panthor, map coherent firmware sections uncached until the VM + // supports a coherent mapping attribute. + if section_hdr.section_flags.cache_mode() != CacheMode::Cached { + vm_map_flags |= VmFlag::Uncached; + } + + Ok(Some(ParsedSection { + data_range: section_hdr.data.clone(), + va: section_hdr.va, + vm_map_flags, + })) + } +} + +/// Firmware binary header containing version and size information. +/// +/// The header is located at the beginning of the firmware binary and contains +/// a magic value for validation, version information, and the total size of +/// all structured headers that follow. +#[expect(dead_code)] +struct FirmwareHeader { + /// Magic value to check binary validity. + magic: u32, + + /// Minor firmware version. + minor: u8, + + /// Major firmware version. + major: u8, + + /// Padding. Must be set to zero. + _padding1: u16, + + /// Firmware version hash. + version_hash: u32, + + /// Padding. Must be set to zero. + _padding2: u32, + + /// Total size of all the structured data headers at beginning of firmware binary. + size: u32, +} + +impl FirmwareHeader { + const FW_BINARY_MAGIC: u32 = 0xc3f13a6e; + const FW_BINARY_MAJOR_MAX: u8 = 0; + + /// Reads and validates a firmware header from the cursor. + /// + /// Verifies the magic value, version compatibility, and padding fields. + fn new(cursor: &mut Cursor<'_>) -> Result { + let magic = cursor.read_u32()?; + if magic != Self::FW_BINARY_MAGIC { + dev_err!(cursor.dev, "Invalid firmware magic"); + return Err(EINVAL); + } + + let minor = cursor.read_u8()?; + let major = cursor.read_u8()?; + + if major > Self::FW_BINARY_MAJOR_MAX { + dev_err!( + cursor.dev, + "Unsupported firmware binary header version {}.{} (expected {}.x)", + major, + minor, + Self::FW_BINARY_MAJOR_MAX + ); + return Err(EINVAL); + } + + let padding1 = cursor.read_u16()?; + let version_hash = cursor.read_u32()?; + let padding2 = cursor.read_u32()?; + let size = cursor.read_u32()?; + + if padding1 != 0 || padding2 != 0 { + dev_err!( + cursor.dev, + "Invalid firmware file: header padding is not zero" + ); + return Err(EINVAL); + } + + let fw_header = Self { + magic, + minor, + major, + _padding1: padding1, + version_hash, + _padding2: padding2, + size, + }; + + Ok(fw_header) + } +} + +/// Firmware section header for loading binary sections into MCU memory. +#[derive(Debug)] +struct SectionHeader { + section_flags: SectionFlags, + /// MCU virtual range to map this binary section to. + va: Range, + /// References the data in the FW binary. + data: Range, +} + +impl SectionHeader { + /// Reads and validates a section header from the cursor. + /// + /// Parses section flags, virtual address range, and data range from the firmware binary. + fn new(cursor: &mut Cursor<'_>) -> Result { + let section_flags = SectionFlags::try_from_fw(cursor.read_u32()?)?; + + let va_start = cursor.read_u32()?; + let va_end = cursor.read_u32()?; + + let va = va_start..va_end; + + if va.end < va.start { + dev_err!( + cursor.dev, + "Invalid firmware file: VA end precedes start at pos {}", + cursor.pos(), + ); + return Err(EINVAL); + } + + let data_start = cursor.read_u32()?; + let data_end = cursor.read_u32()?; + let data = data_start..data_end; + + Ok(Self { + section_flags, + va, + data, + }) + } +} + +/// A firmware entry containing a header and optional parsed section data. +/// +/// Represents a single entry in the firmware binary, which may contain loadable +/// section data or metadata that doesn't require loading. +struct EntrySection { + entry_hdr: EntryHeader, + inner: Option, +} + +/// Header for a firmware entry, packed into a single u32. +/// +/// The entry header encodes the entry type, size, and optional flag in a +/// 32-bit value with the following layout: +/// - Bits 0-7: Entry type +/// - Bits 8-15: Size in bytes +/// - Bit 31: Optional flag +struct EntryHeader(u32); + +impl EntryHeader { + fn entry_type_raw(&self) -> u8 { + (self.0 & 0xff) as u8 + } + + fn entry_type(&self) -> Result { + let v = self.entry_type_raw(); + EntryType::try_from(v) + } + + fn optional(&self) -> bool { + self.0 & bit_u32(31) != 0 + } + + fn size(&self) -> u32 { + self.0 >> 8 & 0xff + } +} + +#[derive(Clone, Copy, Debug)] +#[repr(u8)] +enum EntryType { + /// Host <-> FW interface. + Iface = 0, + /// FW config. + Config = 1, + /// Unit tests. + FutfTest = 2, + /// Trace buffer interface. + TraceBuffer = 3, + /// Timeline metadata interface. + TimelineMetadata = 4, + /// Metadata about how the FW binary was built. + BuildInfoMetadata = 6, +} + +impl TryFrom for EntryType { + type Error = Error; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(EntryType::Iface), + 1 => Ok(EntryType::Config), + 2 => Ok(EntryType::FutfTest), + 3 => Ok(EntryType::TraceBuffer), + 4 => Ok(EntryType::TimelineMetadata), + 6 => Ok(EntryType::BuildInfoMetadata), + _ => Err(EINVAL), + } + } +} diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs new file mode 100644 index 000000000..3bf3787f5 --- /dev/null +++ b/drivers/gpu/drm/tyr/gem.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT +//! GEM buffer object management for the Tyr driver. +//! +//! This module provides buffer object (BO) management functionality using +//! DRM's GEM subsystem with shmem backing. + +use core::ops::Range; + +use kernel::{ + drm::gem::{ + self, + shmem, // + }, + prelude::*, + sync::{ + aref::ARef, + Arc, // + }, // +}; + +use crate::{ + driver::{ + TyrDrmDevice, + TyrDrmDriver, // + }, + vm::{ + Vm, + VmMapFlags, // + }, +}; + +/// Tyr's DriverObject type for GEM objects. +#[pin_data] +pub(crate) struct BoData { + flags: u32, +} + +/// Provides a way to pass arguments when creating BoData +/// as required by the gem::DriverObject trait. +pub(crate) struct BoCreateArgs { + flags: u32, +} + +impl gem::DriverObject for BoData { + type Driver = TyrDrmDriver; + type Args = BoCreateArgs; + + fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit { + try_pin_init!(Self { flags: args.flags }) + } +} + +/// Type alias for Tyr GEM buffer objects. +pub(crate) type Bo = gem::shmem::Object; + +/// Creates a dummy GEM object to serve as the root of a GPUVM. +pub(crate) fn new_dummy_object(ddev: &TyrDrmDevice) -> Result> { + let bo = Bo::new( + ddev, + 4096, + shmem::ObjectConfig { + map_wc: true, + parent_resv_obj: None, + }, + BoCreateArgs { flags: 0 }, + )?; + + Ok(bo) +} + +/// Specifies how to choose a GPU virtual address for a [`KernelBo`]. +/// An automatic VA allocation strategy will be added in the future. +pub(crate) enum KernelBoVaAlloc { + /// Explicit VA address specified by the caller. + Explicit(u64), +} + +/// A kernel-owned buffer object with automatic GPU virtual address mapping. +/// +/// This structure represents a buffer object that is created and managed entirely +/// by the kernel driver, as opposed to userspace-created GEM objects. It combines +/// a GEM object with automatic GPU virtual address (VA) space mapping and cleanup. +/// +/// When dropped, the buffer is automatically unmapped from the GPU VA space. +pub(crate) struct KernelBo<'drm> { + /// The underlying GEM buffer object. + bo: ARef, + /// The GPU VM this buffer is mapped into. + vm: Arc>, + /// The GPU VA range occupied by this buffer. + va_range: Range, +} + +impl<'drm> KernelBo<'drm> { + /// Creates a new kernel-owned buffer object and maps it into GPU VA space. + /// + /// This function allocates a new shmem-backed GEM object and immediately maps + /// it into the specified GPU virtual memory space. The mapping is automatically + /// cleaned up when the [`KernelBo`] is dropped. + pub(crate) fn new( + ddev: &TyrDrmDevice, + vm: Arc>, + size: u64, + va_alloc: KernelBoVaAlloc, + flags: VmMapFlags, + ) -> Result { + if size == 0 { + dev_err!(vm.dev(), "Cannot create KernelBo with size 0"); + return Err(EINVAL); + } + + let KernelBoVaAlloc::Explicit(va) = va_alloc; + + let bo_size = usize::try_from(size).map_err(|_| EOVERFLOW)?; + let va_end = va.checked_add(size).ok_or(EINVAL)?; + + let bo = Bo::new( + ddev, + bo_size, + shmem::ObjectConfig { + map_wc: true, + parent_resv_obj: None, + }, + BoCreateArgs { flags: 0 }, + )?; + + vm.map_bo_range(&bo, 0, size, va, flags)?; + + Ok(KernelBo { + bo, + vm, + va_range: va..va_end, + }) + } + + pub(crate) fn bo(&self) -> &Bo { + &self.bo + } +} + +impl Drop for KernelBo<'_> { + fn drop(&mut self) { + let va = self.va_range.start; + let size = self.va_range.end - self.va_range.start; + + if let Err(e) = self.vm.unmap_range(va, size) { + // If unmap_range fails, it is still safe to drop the + // KernelBo and its ARef to the GEM buffer object because + // GPUVM also holds a reference to the GEM buffer object. + // The physical pages won't be freed or reallocated. + dev_err!( + self.vm.dev(), + "Failed to unmap KernelBo range {:#x}..{:#x}: {:?}", + self.va_range.start, + self.va_range.end, + e + ); + } + } +} diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs new file mode 100644 index 000000000..592b8bb16 --- /dev/null +++ b/drivers/gpu/drm/tyr/gpu.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +use core::ops::{ + Deref, + DerefMut, // +}; +use kernel::{ + device::{ + Bound, + Device, // + }, + io::{ + poll, + register::Array, + Io, // + }, + prelude::*, + time::Delta, + transmute::AsBytes, + uapi, // +}; + +use crate::{ + driver::IoMem, + regs::{ + gpu_control::*, + join_u64, // + }, // +}; + +/// Struct containing information that can be queried by userspace. This is read from +/// the GPU's registers. +/// +/// # Invariants +/// +/// - The layout of this struct is identical to the C `struct drm_panthor_gpu_info`. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub(crate) struct GpuInfo(pub(crate) uapi::drm_panthor_gpu_info); + +impl GpuInfo { + pub(crate) fn new(io: &IoMem<'_>) -> Self { + Self(uapi::drm_panthor_gpu_info { + gpu_id: io.read(GPU_ID).into_raw(), + gpu_rev: io.read(REVIDR).into_raw(), + csf_id: io.read(CSF_ID).into_raw(), + l2_features: io.read(L2_FEATURES).into_raw(), + tiler_features: io.read(TILER_FEATURES).into_raw(), + mem_features: io.read(MEM_FEATURES).into_raw(), + mmu_features: io.read(MMU_FEATURES).into_raw(), + thread_features: io.read(THREAD_FEATURES).into_raw(), + max_threads: io.read(THREAD_MAX_THREADS).into_raw(), + thread_max_workgroup_size: io.read(THREAD_MAX_WORKGROUP_SIZE).into_raw(), + thread_max_barrier_size: io.read(THREAD_MAX_BARRIER_SIZE).into_raw(), + coherency_features: io.read(COHERENCY_FEATURES).into_raw(), + texture_features: [ + io.read(TEXTURE_FEATURES::at(0)).supported_formats().get(), + io.read(TEXTURE_FEATURES::at(1)).supported_formats().get(), + io.read(TEXTURE_FEATURES::at(2)).supported_formats().get(), + io.read(TEXTURE_FEATURES::at(3)).supported_formats().get(), + ], + as_present: io.read(AS_PRESENT).into_raw(), + selected_coherency: uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE, + shader_present: join_u64( + io.read(SHADER_PRESENT_LO).into_raw(), + io.read(SHADER_PRESENT_HI).into_raw(), + ), + l2_present: join_u64( + io.read(L2_PRESENT_LO).into_raw(), + io.read(L2_PRESENT_HI).into_raw(), + ), + tiler_present: join_u64( + io.read(TILER_PRESENT_LO).into_raw(), + io.read(TILER_PRESENT_HI).into_raw(), + ), + core_features: io.read(CORE_FEATURES).into_raw(), + // Padding must be zero. + pad: 0, + //GPU_FEATURES register is not available; it was introduced in arch 11.x. + gpu_features: 0, + }) + } + + pub(crate) fn log(&self, dev: &Device) { + let gpu_id = GPU_ID::from_raw(self.gpu_id); + + let model_name = if let Some(model) = GPU_MODELS.iter().find(|&f| { + f.arch_major == gpu_id.arch_major().get() && f.prod_major == gpu_id.prod_major().get() + }) { + model.name + } else { + "unknown" + }; + + dev_info!( + dev, + "mali-{} GPU_ID 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}", + model_name, + gpu_id.into_raw(), + gpu_id.ver_major().get(), + gpu_id.ver_minor().get(), + gpu_id.ver_status().get() + ); + + dev_info!( + dev, + "Features: L2:{:#x} Tiler:{:#x} Mem:{:#x} MMU:{:#x} AS:{:#x}", + self.l2_features, + self.tiler_features, + self.mem_features, + self.mmu_features, + self.as_present, + ); + + dev_info!( + dev, + "shader_present=0x{:016x} l2_present=0x{:016x} tiler_present=0x{:016x}", + self.shader_present, + self.l2_present, + self.tiler_present, + ); + } +} + +impl Deref for GpuInfo { + type Target = uapi::drm_panthor_gpu_info; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for GpuInfo { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +// SAFETY: `GpuInfo`'s invariant guarantees that it is the same type that is +// already exposed to userspace by the C driver. This implies that it fulfills +// the requirements for `AsBytes`. +// +// This means: +// +// - No implicit padding, +// - No kernel pointers, +// - No interior mutability. +unsafe impl AsBytes for GpuInfo {} + +struct GpuModels { + name: &'static str, + arch_major: u32, + prod_major: u32, +} + +const GPU_MODELS: [GpuModels; 1] = [GpuModels { + name: "g610", + arch_major: 10, + prod_major: 7, +}]; + +/// Powers on the l2 block. +pub(crate) fn l2_power_on(dev: &Device, io: &IoMem<'_>) -> Result { + io.write_reg(L2_PWRON_LO::zeroed().with_const_request::<1>()); + + poll::read_poll_timeout( + || Ok(io.read(L2_READY_LO)), + |status| status.ready() == 1, + Delta::from_millis(1), + Delta::from_millis(100), + ) + .inspect_err(|_| dev_err!(dev, "Failed to power on the GPU."))?; + + Ok(()) +} diff --git a/drivers/gpu/drm/tyr/mmu.rs b/drivers/gpu/drm/tyr/mmu.rs new file mode 100644 index 000000000..5e88c9e6a --- /dev/null +++ b/drivers/gpu/drm/tyr/mmu.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Memory Management Unit (MMU) module. +//! +//! The GPU MMU provides a limited number of memory address spaces for use by command streams. +//! The MMU translates virtual addresses to physical addresses and manages memory configuration +//! and access permissions. +//! +//! This MMU module is essentially a locked wrapper around a [`SlotManager`] instance. +//! The [`SlotManager`] manages the assignment of virtual address spaces to hardware address-space +//! (AS) slots. MMU commands such as updates and flushes are carried out by the +//! [`AddressSpaceManager`] which actually writes to the MMU registers. + +use core::ops::Range; + +use kernel::{ + device::{ + Bound, + Device, // + }, + new_mutex, + prelude::*, + sync::{ + Arc, + ArcBorrow, + Mutex, // + }, // +}; + +use crate::{ + driver::IoMem, + gpu::GpuInfo, + mmu::address_space::{ + AddressSpaceManager, + VmAsData, // + }, + regs::{ + gpu_control::AS_PRESENT, + MAX_AS, // + }, + slot::SlotManager, // +}; + +pub(crate) mod address_space; + +pub(crate) type AsSlotManager<'drm> = SlotManager, MAX_AS>; + +/// Locked wrapper for carrying out virtual memory (VM) operations on the MMU. +#[pin_data] +pub(crate) struct Mmu<'drm> { + /// Slot Manager instance used to allocate hardware slots and write to MMU registers. + #[pin] + pub(crate) as_manager: Mutex>, +} + +impl<'drm> Mmu<'drm> { + /// Create an MMU component for this device. + pub(crate) fn new( + dev: &'drm Device, + iomem: ArcBorrow<'_, IoMem<'drm>>, + gpu_info: &GpuInfo, + ) -> Result>> { + let present = AS_PRESENT::from_raw(gpu_info.as_present).present().get(); + let slot_count = present.count_ones().try_into()?; + + let address_space_manager = AddressSpaceManager::new(dev, iomem.into(), present)?; + let as_slot_manager = + SlotManager::new(address_space_manager, slot_count).inspect_err(|e| { + dev_err!( + dev, + "Failed to initialize MMU slot manager with {} slots: {:?}", + slot_count, + e + ); + })?; + let mmu_init = try_pin_init!(Self{ + as_manager <- new_mutex!(as_slot_manager), + }); + Arc::pin_init(mmu_init, GFP_KERNEL) + } + + /// Assign a VM to an AS slot, provide a translation table, + /// and update the MMU to make the VM resident. + pub(crate) fn activate_vm(&self, vm_as_data: ArcBorrow<'_, VmAsData<'drm>>) -> Result { + self.as_manager.lock().activate_vm(vm_as_data) + } + + /// Evict a VM from its AS slot and flush the MMU. + pub(crate) fn deactivate_vm(&self, vm_as_data: &VmAsData<'drm>) -> Result { + self.as_manager.lock().deactivate_vm(vm_as_data) + } + + /// Flush MMU translation caches after a VM update. + pub(crate) fn flush_vm(&self, vm_as_data: &VmAsData<'drm>) -> Result { + self.as_manager.lock().flush_vm(vm_as_data) + } + + /// Flags the start of a VM update. + /// + /// If the VM is resident, any GPU access on the memory range being + /// updated will be blocked until `Mmu::end_vm_update()` is called. + /// This guarantees the atomicity of a VM update. + /// If the VM is not resident, this is a NOP. + pub(crate) fn start_vm_update( + &self, + vm_as_data: &VmAsData<'drm>, + region: &Range, + ) -> Result { + self.as_manager.lock().start_vm_update(vm_as_data, region) + } + + /// Flags the end of a VM update. + /// + /// If the VM is resident, this will let GPU accesses on the updated + /// range go through, in case any of them were blocked. + /// If the VM is not resident, this is a NOP. + pub(crate) fn end_vm_update(&self, vm_as_data: &VmAsData<'drm>) -> Result { + self.as_manager.lock().end_vm_update(vm_as_data) + } +} diff --git a/drivers/gpu/drm/tyr/mmu/address_space.rs b/drivers/gpu/drm/tyr/mmu/address_space.rs new file mode 100644 index 000000000..76e3d2df6 --- /dev/null +++ b/drivers/gpu/drm/tyr/mmu/address_space.rs @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Address space module. +//! +//! This module handles the hardware interaction for MMU operations through +//! MMIO register access. +//! + +use core::ops::Range; + +use kernel::{ + device::{ + Bound, + Device, // + }, // + error::Result, + io::{ + poll, + register::Array, + Io, // + }, + iommu::pgtable::{ + Config, + IoPageTable, + ARM64LPAES1, // + }, + num::Bounded, + prelude::*, + sizes::{ + SZ_2M, + SZ_4K, // + }, + sync::{ + Arc, + ArcBorrow, + LockedBy, // + }, + time::Delta, // +}; + +use crate::{ + driver::IoMem, + mmu::{ + AsSlotManager, + Mmu, // + }, + regs::{ + mmu_control::mmu_as_control, + mmu_control::mmu_as_control::*, + MAX_AS, // + }, + slot::{ + LockedSeat, + Seat, + SlotOperations, // + }, // +}; + +/// Address space configuration values to be written to MMU registers. +#[derive(Clone, Copy)] +struct AddressSpaceConfig { + /// Translation configuration. Configures how the MMU walks the page table for this + /// address space. + transcfg: u64, + + /// Translation table base address. The address of the page table. + transtab: u64, + + /// Memory attributes such as cacheability. + memattr: u64, +} + +/// Virtual memory (VM) address space data for use in MMU operations. +#[pin_data] +pub(crate) struct VmAsData<'drm> { + /// This address-space seat tracks this VM's binding to a hardware address space slot. + /// It can only be accessed when holding the `Mmu::as_manager` lock. + as_seat: LockedSeat, MAX_AS>, + + /// Virtual address bits for this address space. + va_bits: u8, + + /// The page table which maps GPU virtual addresses to physical addresses for this VM. + #[pin] + pub(crate) page_table: IoPageTable<'drm, ARM64LPAES1>, +} + +impl<'drm> VmAsData<'drm> { + /// Creates VM address space data by initializing all of its fields. + pub(crate) fn new<'a>( + mmu: &'a Mmu<'drm>, + dev: &'drm Device, + va_bits: u32, + pa_bits: u32, + ) -> impl pin_init::PinInit, Error> + 'a { + let pt_config = Config { + quirks: 0, + pgsize_bitmap: SZ_4K | SZ_2M, + ias: va_bits, + oas: pa_bits, + coherent_walk: false, + }; + + let page_table_init = IoPageTable::new(dev, pt_config); + + try_pin_init!(Self { + as_seat: LockedBy::new(&mmu.as_manager, Seat::NoSeat), + va_bits: va_bits as u8, + page_table <- page_table_init, + }? Error) + } + + /// Computes the hardware configuration for this address space. + fn as_config(&self) -> Result { + let pt = &self.page_table; + // The hardware computes the valid input address range as: + // INA_BITS_VALID = min(HW_INA_BITS, 55 - INA_BITS) + // To configure our desired va_bits, we solve for INA_BITS: + // INA_BITS = 55 - va_bits + // This assumes HW_INA_BITS (hardware capability) >= va_bits. + let field = 55u64.checked_sub(self.va_bits.into()).ok_or(EINVAL)?; + let ina_bits = + match mmu_as_control::InaBits::try_from(Bounded::try_new(field).ok_or(EINVAL)?)? { + mmu_as_control::InaBits::Reset => return Err(EINVAL), + bits => bits, + }; + + let transcfg = mmu_as_control::TRANSCFG::zeroed() + .with_ptw_memattr(mmu_as_control::PtwMemattr::WriteBack) + .with_r_allocate(true) + .with_mode(mmu_as_control::AddressSpaceMode::Aarch64_4K) + .with_ina_bits(ina_bits) + .into_raw(); + + Ok(AddressSpaceConfig { + transcfg, + // SAFETY: The SlotManager holds an `Arc` as SlotData while this + // TTBR is programmed and stores that Arc in the active slot before + // returning. Eviction flushes and disables the slot before releasing + // the Arc; if eviction fails, the slot retains it. Therefore the page + // table cannot be dropped while the GPU is using it. + transtab: unsafe { pt.ttbr() }, + memattr: MEMATTR::from_mair(pt.mair()).into_raw(), + }) + } +} + +/// Coordinates all hardware-level address space operations through MMIO register +/// operations including enabling, disabling, flushing, and updating address spaces. +pub(crate) struct AddressSpaceManager<'drm> { + /// Parent device used for logging. + dev: &'drm Device, + + /// Memory-mapped I/O region for GPU register access. + iomem: Arc>, + + /// Bitmask of present address space slots from GPU_AS_PRESENT register. + as_present: u32, +} + +impl<'drm> AddressSpaceManager<'drm> { + /// Creates a new address space manager. + /// + /// Initializes the manager with references to the platform device and + /// I/O memory region, along with the bitmask of available AS slots. + pub(super) fn new( + dev: &'drm Device, + iomem: Arc>, + as_present: u32, + ) -> Result> { + if as_present.trailing_ones() != as_present.count_ones() { + dev_err!( + dev, + "Sparse AS_PRESENT mask is unsupported: {:#x}", + as_present + ); + return Err(EINVAL); + } + Ok(Self { + dev, + iomem, + as_present, + }) + } + + /// Validates that an AS slot number is within range and present in hardware. + /// + /// Checks that the slot index is less than [`MAX_AS`] and that + /// the corresponding bit is set in the `as_present` mask read from the GPU. + /// + /// Returns [`EINVAL`] if the slot is out of range or not present in hardware. + fn validate_as_slot(&self, as_nr: usize) -> Result { + if as_nr >= MAX_AS { + dev_err!( + self.dev, + "AS slot {} out of valid range (max {})", + as_nr, + MAX_AS + ); + return Err(EINVAL); + } + + if (self.as_present & (1 << as_nr)) == 0 { + dev_err!( + self.dev, + "AS slot {} not present in hardware (AS_PRESENT={:#x})", + as_nr, + self.as_present + ); + return Err(EINVAL); + } + Ok(()) + } + + /// Waits for an AS slot to become ready (not active). + /// + /// Returns an error if polling times out after 10ms or if register access fails. + fn as_wait_ready(&self, as_nr: usize) -> Result { + let io = &*self.iomem; + let op = || { + let status_reg = STATUS::try_at(as_nr).ok_or(EINVAL)?; + Ok(io.read(status_reg)) + }; + let cond = |status: &STATUS| -> bool { !status.active_ext() }; + poll::read_poll_timeout(op, cond, Delta::from_micros(50), Delta::from_millis(10))?; + + Ok(()) + } + + /// Sends a command to an AS slot. + /// + /// Returns an error if waiting for ready times out or if register write fails. + fn as_send_cmd(&mut self, as_nr: usize, cmd: MmuCommand) -> Result { + self.as_wait_ready(as_nr)?; + let io = &*self.iomem; + let command_reg = COMMAND::try_at(as_nr).ok_or(EINVAL)?; + io.write(command_reg, COMMAND::zeroed().with_command(cmd)); + Ok(()) + } + + /// Sends a command to an AS slot and waits for completion. + /// + /// Returns an error if sending the command fails or if waiting for completion times out. + fn as_send_cmd_and_wait(&mut self, as_nr: usize, cmd: MmuCommand) -> Result { + self.as_send_cmd(as_nr, cmd)?; + self.as_wait_ready(as_nr)?; + Ok(()) + } + + /// Enables an AS slot with the provided configuration. + /// + /// Returns an error if the slot is invalid or if register writes/commands fail. + fn as_enable(&mut self, as_nr: usize, as_config: &AddressSpaceConfig) -> Result { + self.validate_as_slot(as_nr)?; + + let io = &*self.iomem; + + let transtab = as_config.transtab; + io.write( + TRANSTAB_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_LO::from_raw(transtab as u32), + ); + io.write( + TRANSTAB_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_HI::from_raw((transtab >> 32) as u32), + ); + + let transcfg = as_config.transcfg; + io.write( + TRANSCFG_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_LO::from_raw(transcfg as u32), + ); + io.write( + TRANSCFG_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_HI::from_raw((transcfg >> 32) as u32), + ); + + let memattr = as_config.memattr; + io.write( + MEMATTR_LO::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_LO::from_raw(memattr as u32), + ); + io.write( + MEMATTR_HI::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_HI::from_raw((memattr >> 32) as u32), + ); + + self.as_send_cmd_and_wait(as_nr, MmuCommand::Update)?; + + Ok(()) + } + + /// Disables an AS slot and clears its configuration. + /// + /// Returns an error if the slot is invalid or if register writes/commands fail. + fn as_disable(&mut self, as_nr: usize) -> Result { + self.validate_as_slot(as_nr)?; + + // Flush AS before disabling + self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushMem)?; + + let io = &*self.iomem; + + io.write( + TRANSTAB_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_LO::from_raw(0), + ); + io.write( + TRANSTAB_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_HI::from_raw(0), + ); + + io.write( + MEMATTR_LO::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_LO::from_raw(0), + ); + io.write( + MEMATTR_HI::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_HI::from_raw(0), + ); + + let transcfg = TRANSCFG::zeroed() + .with_mode(AddressSpaceMode::Unmapped) + .into_raw(); + + io.write( + TRANSCFG_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_LO::from_raw(transcfg as u32), + ); + io.write( + TRANSCFG_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_HI::from_raw((transcfg >> 32) as u32), + ); + + self.as_send_cmd_and_wait(as_nr, MmuCommand::Update)?; + + Ok(()) + } + + /// Locks a region of the translation tables for an atomic update. + /// + /// Programs the MMU [`LOCKADDR`] register for the given address space and issues + /// the lock command. The hardware rounds the requested range up to a + /// power-of-two region aligned to its size. + /// + /// Returns an error if the slot is invalid or if register writes/commands fail. + fn as_start_update(&mut self, as_nr: usize, region: &Range) -> Result { + self.validate_as_slot(as_nr)?; + + // Avoid both an empty range and an inverted range. + if region.start >= region.end { + return Err(EINVAL); + } + + // The lock operates on full 64-byte cache lines of translation table entries. + // Since each translation table entry (TTE) is 8 bytes, a cache line has 8 TTEs. + // Since each TTE maps one page, the minimum locked region size will be 8 pages. + // + // With 4KiB pages (Aarch64_4K mode), the minimum locked region is 32KiB. + let lock_region_min_size: u64 = 4096 * 8; + + // Count the number of trailing zero bits (zeros at the right/least-significant + // end of the binary representation). For a power-of-two value, this equals the + // base-2 exponent (e.g., 32 KiB = 2^15 → 15). + let lock_region_min_size_log2 = lock_region_min_size.trailing_zeros() as u8; + + // XOR the first and last addresses to identify which bits differ between them. + // The highest set bit in the result determines the exponent of the smallest + // power-of-two region that can contain both addresses. + // + // Example: + // addr_xor = 0x1000 ^ 0x2FFF = 0x3FFF + // highest set bit in 0x3FFF is bit 13 + // minimum region size = 2^(13 + 1) = 16 KiB + let addr_xor = region.start ^ (region.end - 1); + let region_size_log2 = 64 - addr_xor.leading_zeros() as u8; + + let lock_region_log2 = core::cmp::max(region_size_log2, lock_region_min_size_log2); + + let lock_region_size = 1u64.checked_shl(lock_region_log2.into()).ok_or(EINVAL)?; + // Align the LOCKADDR base address down to the lock region size (1 << lock_region_log2). + // + // The MMU ignores the low lock_region_log2 bits of LOCKADDR base, so ensure + // they are cleared in software to avoid ambiguity. + // + // Example: + // lock_region_log2 = 14 (16 KiB) + // region.start = 0x1000 + // lockaddr_base = 0x1000 & ~(0x3FFF) = 0x0000 + let lockaddr_base = region.start & !(lock_region_size - 1); + + // The LOCKADDR size field encodes the lock region size as log2(size) - 1, + // per the hardware definition. For example, a 32 KiB region is encoded as 14 + // because log2(32 KiB) = 15. + let lockaddr_size = lock_region_log2 - 1; + + let io = &*self.iomem; + + // The LOCKADDR base field stores address bits 63:12, so remove the low 12 bits + // before passing this value to the register macro helper. + // These bits are guaranteed to be zero anyway because of the minimum + // size of the locked region. + let lockaddr_base_field = lockaddr_base >> 12; + let lockaddr_val = LOCKADDR::zeroed() + .try_with_size(lockaddr_size)? + .try_with_base(lockaddr_base_field)? + .into_raw(); + + io.write( + LOCKADDR_LO::try_at(as_nr).ok_or(EINVAL)?, + LOCKADDR_LO::from_raw(lockaddr_val as u32), + ); + io.write( + LOCKADDR_HI::try_at(as_nr).ok_or(EINVAL)?, + LOCKADDR_HI::from_raw((lockaddr_val >> 32) as u32), + ); + + self.as_send_cmd_and_wait(as_nr, MmuCommand::Lock) + } + + /// Completes an atomic translation table update. + /// + /// Returns an error if the slot is invalid or if the flush command fails. + fn as_end_update(&mut self, as_nr: usize) -> Result { + self.validate_as_slot(as_nr)?; + self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushPt)?; + Ok(()) + } + + /// Flushes the translation table cache for an AS slot. + /// + /// Returns an error if the slot is invalid or if the flush command fails. + fn as_flush(&mut self, as_nr: usize) -> Result { + self.validate_as_slot(as_nr)?; + self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushPt) + } +} + +impl<'drm> SlotOperations for AddressSpaceManager<'drm> { + /// VM address space data associated with a hardware slot. + type SlotData = Arc>; + + fn seat(slot_data: &Self::SlotData) -> &LockedSeat { + &slot_data.as_seat + } + + /// Activates a VM in a hardware slot. + fn activate(&mut self, slot_idx: usize, slot_data: &Self::SlotData) -> Result { + let as_config = slot_data.as_config()?; + self.as_enable(slot_idx, &as_config) + } + + /// Evicts a VM from a hardware slot. + fn evict(&mut self, slot_idx: usize, _slot_data: &Self::SlotData) -> Result { + self.as_flush(slot_idx)?; + self.as_disable(slot_idx)?; + Ok(()) + } +} + +impl<'drm> AsSlotManager<'drm> { + /// Locks a region for translation table updates if the VM has an active slot. + pub(super) fn start_vm_update( + &mut self, + vm_as_data: &VmAsData<'drm>, + region: &Range, + ) -> Result { + let seat = vm_as_data.as_seat.access(self); + match seat.slot() { + Some(slot) => { + let as_nr = slot as usize; + self.as_start_update(as_nr, region) + } + _ => Ok(()), + } + } + + /// Completes translation table updates and unlocks the region. + pub(super) fn end_vm_update(&mut self, vm_as_data: &VmAsData<'drm>) -> Result { + let seat = vm_as_data.as_seat.access(self); + match seat.slot() { + Some(slot) => { + let as_nr = slot as usize; + self.as_end_update(as_nr) + } + _ => Ok(()), + } + } + + /// Flushes the translation table cache if the VM has an active slot. + pub(super) fn flush_vm(&mut self, vm_as_data: &VmAsData<'drm>) -> Result { + let seat = vm_as_data.as_seat.access(self); + match seat.slot() { + Some(slot) => { + let as_nr = slot as usize; + self.as_flush(as_nr) + } + _ => Ok(()), + } + } + + /// Activates a VM by assigning it to a hardware slot. + pub(super) fn activate_vm(&mut self, vm_as_data: ArcBorrow<'_, VmAsData<'drm>>) -> Result { + self.activate(vm_as_data.into()) + } + + /// Deactivates a VM by evicting it from its hardware slot. + pub(super) fn deactivate_vm(&mut self, vm_as_data: &VmAsData<'drm>) -> Result { + self.evict(&vm_as_data.as_seat) + } +} diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs new file mode 100644 index 000000000..a62724378 --- /dev/null +++ b/drivers/gpu/drm/tyr/regs.rs @@ -0,0 +1,1779 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! # Definitions +//! +//! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions) +//! from the command stream. +//! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular +//! job or sequence of jobs. The instructions exist in one or more command buffers. +//! - **CSF**: Command Stream Frontend - The interface and implementation for job submission +//! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS +//! interfaces. +//! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple +//! CSGs, and each CSG contains multiple CSs. +//! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the +//! iterator control aspects. Implements the CSF in conjunction with the MCU. +//! - **GLB**: Global - Prefix for global interface registers that control operations common to +//! all CSs. +//! - **JASID**: Job Address Space ID - Identifies the address space for a job. +//! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream +//! hardware. +//! - **MMU**: Memory Management Unit - Handles address translation and memory access protection. + +// We don't expect that all the registers and fields will be used, even in the +// future. +// +// Nevertheless, it is useful to have most of them defined, like the C driver +// does. +#![expect(dead_code)] + +/// Combine two 32-bit values into a single 64-bit value. +pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { + (u64::from(lo)) | ((u64::from(hi)) << 32) +} + +/// Read a logical 64-bit value from split 32-bit registers without tearing. +pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 { + loop { + let hi1 = hi_read(); + let lo = lo_read(); + let hi2 = hi_read(); + + if hi1 == hi2 { + return join_u64(lo, hi1); + } + } +} + +pub(crate) use mmu_control::mmu_as_control::MAX_AS; + +/// These registers correspond to the GPU_CONTROL register page. +/// They are involved in GPU configuration and control. +pub(crate) mod gpu_control { + use kernel::{ + num::Bounded, + prelude::*, + register, + uapi, // + }; + + register! { + /// GPU identification register. + pub(crate) GPU_ID(u32) @ 0x0 { + /// Status of the GPU release. + 3:0 ver_status; + /// Minor release version number. + 11:4 ver_minor; + /// Major release version number. + 15:12 ver_major; + /// Product identifier. + 19:16 prod_major; + /// Architecture patch revision. + 23:20 arch_rev; + /// Architecture minor revision. + 27:24 arch_minor; + /// Architecture major revision. + 31:28 arch_major; + } + + /// Level 2 cache features register. + pub(crate) L2_FEATURES(u32) @ 0x4 { + /// Cache line size. + 7:0 line_size; + /// Cache associativity. + 15:8 associativity; + /// Cache slice size. + 23:16 cache_size; + /// External bus width. + 31:24 bus_width; + } + + /// Shader core features. + pub(crate) CORE_FEATURES(u32) @ 0x8 { + /// Shader core variant. + 7:0 core_variant; + } + + /// Tiler features. + pub(crate) TILER_FEATURES(u32) @ 0xc { + /// Log of the tiler's bin size. + 5:0 bin_size; + /// Maximum number of active levels. + 11:8 max_levels; + } + + /// Memory system features. + pub(crate) MEM_FEATURES(u32) @ 0x10 { + 0:0 coherent_core_group => bool; + 1:1 coherent_super_group => bool; + 11:8 l2_slices; + } + + /// Memory management unit features. + pub(crate) MMU_FEATURES(u32) @ 0x14 { + /// Number of bits supported in virtual addresses. + 7:0 va_bits; + /// Number of bits supported in physical addresses. + 15:8 pa_bits; + } + + /// Address spaces present. + pub(crate) AS_PRESENT(u32) @ 0x18 { + 31:0 present; + } + + /// CSF version information. + pub(crate) CSF_ID(u32) @ 0x1c { + /// MCU revision ID. + 3:0 mcu_rev; + /// MCU minor revision number. + 9:4 mcu_minor; + /// MCU major revision number. + 15:10 mcu_major; + /// CSHW revision ID. + 19:16 cshw_rev; + /// CSHW minor revision number. + 25:20 cshw_minor; + /// CSHW major revision number. + 31:26 cshw_major; + } + + /// IRQ sources raw status. + /// Writing to this register forces bits on, but does not clear them. + pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 { + /// A GPU fault has occurred, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// A GPU fault has occurred, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// Reset has completed, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// Set when a single power domain has powered up or down, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// Set when the all pending power domain changes are completed, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// Set when cache cleaning has completed, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag. + 18:18 doorbell_mirror => bool; + /// MCU requires attention, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + + /// IRQ sources to clear. Write only. + pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 { + /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + + /// IRQ sources enabled. + pub(crate) GPU_IRQ_MASK(u32) @ 0x28 { + /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag. + 18:18 doorbell_mirror => bool; + /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + + /// IRQ status for enabled sources. Read only. + pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c { + /// GPU_FAULT interrupt status, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// RESET_COMPLETED interrupt status, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag. + 18:18 doorbell_mirror => bool; + /// MCU_STATUS interrupt status, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + } + + /// Helpers for GPU_COMMAND Register + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum GpuCommand { + /// No operation. This is the default value. + Nop = 0, + /// Reset the GPU. + Reset = 1, + /// Flush caches. + FlushCaches = 4, + /// Clear GPU faults. + ClearFault = 7, + } + + impl TryFrom> for GpuCommand { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(GpuCommand::Nop), + 1 => Ok(GpuCommand::Reset), + 4 => Ok(GpuCommand::FlushCaches), + 7 => Ok(GpuCommand::ClearFault), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(cmd: GpuCommand) -> Self { + (cmd as u8).into() + } + } + + /// Reset mode for [`GPU_COMMAND::reset()`]. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum ResetMode { + /// Stop all external bus interfaces, then reset the entire GPU. + SoftReset = 1, + /// Force a full GPU reset. + HardReset = 2, + } + + impl TryFrom> for ResetMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 1 => Ok(ResetMode::SoftReset), + 2 => Ok(ResetMode::HardReset), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: ResetMode) -> Self { + Bounded::try_new(mode as u32).unwrap() + } + } + + /// Cache flush mode for [`GPU_COMMAND::flush_caches()`]. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum FlushMode { + /// No flush. + None = 0, + /// Clean the caches. + Clean = 1, + /// Invalidate the caches. + Invalidate = 2, + /// Clean and invalidate the caches. + CleanInvalidate = 3, + } + + impl TryFrom> for FlushMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(FlushMode::None), + 1 => Ok(FlushMode::Clean), + 2 => Ok(FlushMode::Invalidate), + 3 => Ok(FlushMode::CleanInvalidate), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: FlushMode) -> Self { + Bounded::try_new(mode as u32).unwrap() + } + } + + register! { + /// GPU command register. + /// + /// Use the constructor methods to create commands: + /// - [`GPU_COMMAND::nop()`] + /// - [`GPU_COMMAND::reset()`] + /// - [`GPU_COMMAND::flush_caches()`] + /// - [`GPU_COMMAND::clear_fault()`] + pub(crate) GPU_COMMAND (u32) @ 0x30 { + 7:0 command ?=> GpuCommand; + } + /// Internal alias for GPU_COMMAND in reset mode. + /// Use [`GPU_COMMAND::reset()`] instead. + GPU_COMMAND_RESET (u32) => GPU_COMMAND { + 7:0 command ?=> GpuCommand; + 11:8 reset_mode ?=> ResetMode; + } + + /// Internal alias for GPU_COMMAND in cache flush mode. + /// Use [`GPU_COMMAND::flush_caches()`] instead. + GPU_COMMAND_FLUSH (u32) => GPU_COMMAND { + 7:0 command ?=> GpuCommand; + /// L2 cache flush mode. + 11:8 l2_flush ?=> FlushMode; + /// Shader core load/store cache flush mode. + 15:12 lsc_flush ?=> FlushMode; + /// Shader core other caches flush mode. + 19:16 other_flush ?=> FlushMode; + } + } + + impl GPU_COMMAND { + /// Create a NOP command. + pub(crate) fn nop() -> Self { + Self::zeroed() + } + + /// Create a reset command with the specified reset mode. + pub(crate) fn reset(mode: ResetMode) -> Self { + Self::from_raw( + GPU_COMMAND_RESET::zeroed() + .with_command(GpuCommand::Reset) + .with_reset_mode(mode) + .into_raw(), + ) + } + + /// Create a cache flush command with the specified flush modes. + pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self { + Self::from_raw( + GPU_COMMAND_FLUSH::zeroed() + .with_command(GpuCommand::FlushCaches) + .with_l2_flush(l2) + .with_lsc_flush(lsc) + .with_other_flush(other) + .into_raw(), + ) + } + + /// Create a clear fault command. + pub(crate) fn clear_fault() -> Self { + Self::zeroed().with_command(GpuCommand::ClearFault) + } + } + + register! { + /// GPU status register. Read only. + pub(crate) GPU_STATUS(u32) @ 0x34 { + /// GPU active, a 1-bit boolean flag. + 0:0 gpu_active => bool; + /// Power manager active, a 1-bit boolean flag + 1:1 pwr_active => bool; + /// Page fault active, a 1-bit boolean flag. + 4:4 page_fault => bool; + /// Protected mode active, a 1-bit boolean flag. + 7:7 protected_mode_active => bool; + /// Debug mode active, a 1-bit boolean flag. + 8:8 gpu_dbg_enabled => bool; + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum ExceptionType { + /// Exception type: No error. + Ok = 0x00, + /// Exception type: GPU external bus error. + GpuBusFault = 0x80, + /// Exception type: GPU shareability error. + GpuShareabilityFault = 0x88, + /// Exception type: System shareability error. + SystemShareabilityFault = 0x89, + /// Exception type: GPU cacheability error. + GpuCacheabilityFault = 0x8A, + } + + impl TryFrom> for ExceptionType { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0x00 => Ok(ExceptionType::Ok), + 0x80 => Ok(ExceptionType::GpuBusFault), + 0x88 => Ok(ExceptionType::GpuShareabilityFault), + 0x89 => Ok(ExceptionType::SystemShareabilityFault), + 0x8A => Ok(ExceptionType::GpuCacheabilityFault), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(exc: ExceptionType) -> Self { + (exc as u8).into() + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum AccessType { + /// Access type: An atomic (read/write) transaction. + Atomic = 0, + /// Access type: An execute transaction. + Execute = 1, + /// Access type: A read transaction. + Read = 2, + /// Access type: A write transaction. + Write = 3, + } + + impl From> for AccessType { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => AccessType::Atomic, + 1 => AccessType::Execute, + 2 => AccessType::Read, + 3 => AccessType::Write, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(access: AccessType) -> Self { + Bounded::try_new(access as u32).unwrap() + } + } + + register! { + /// GPU fault status register. Read only. + pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { + /// Exception type. + 7:0 exception_type ?=> ExceptionType; + /// Access type. + 9:8 access_type => AccessType; + /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag. + 10:10 address_valid => bool; + /// The JASID field is valid, a 1-bit boolean flag. + 11:11 jasid_valid => bool; + /// JASID of the fault, if known. + 15:12 jasid; + /// ID of the source that triggered the fault. + 31:16 source_id; + } + + /// GPU fault address. Read only. + /// Once a fault is reported, it must be manually cleared by issuing a + /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU + /// faults will be reported until the previous fault has been cleared. + pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 { + 31:0 pointer; + } + + pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 { + 31:0 pointer; + } + + /// Level 2 cache configuration. + pub(crate) L2_CONFIG(u32) @ 0x48 { + /// Requested cache size. + 23:16 cache_size; + /// Requested hash function index. + 31:24 hash_function; + } + + /// Global time stamp offset. + pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 { + 31:0 offset; + } + + pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c { + 31:0 offset; + } + + /// GPU cycle counter. Read only. + pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 { + 31:0 count; + } + + pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 { + 31:0 count; + } + + /// Global time stamp. Read only. + pub(crate) TIMESTAMP_LO(u32) @ 0x98 { + 31:0 timestamp; + } + + pub(crate) TIMESTAMP_HI(u32) @ 0x9c { + 31:0 timestamp; + } + + /// Maximum number of threads per core. Read only constant. + pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 { + 31:0 threads; + } + + /// Maximum number of threads per workgroup. Read only constant. + pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 { + 31:0 threads; + } + + /// Maximum number of threads per barrier. Read only constant. + pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 { + 31:0 threads; + } + + /// Thread features. Read only constant. + pub(crate) THREAD_FEATURES(u32) @ 0xac { + /// Total number of registers per core. + 21:0 max_registers; + /// Implementation technology type. + 23:22 implementation_technology; + /// Maximum number of compute tasks waiting. + 31:24 max_task_queue; + } + + /// Support flags for compressed texture formats. Read only constant. + /// + /// A bitmap where each bit indicates support for a specific compressed texture format. + /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec). + pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 { + 31:0 supported_formats; + } + + /// Shader core present bitmap. Read only constant. + pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 { + 31:0 value; + } + + pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 { + 31:0 value; + } + + /// Tiler present bitmap. Read only constant. + pub(crate) TILER_PRESENT_LO(u32) @ 0x110 { + 31:0 present; + } + + pub(crate) TILER_PRESENT_HI(u32) @ 0x114 { + 31:0 present; + } + + /// L2 cache present bitmap. Read only constant. + pub(crate) L2_PRESENT_LO(u32) @ 0x120 { + 31:0 present; + } + + pub(crate) L2_PRESENT_HI(u32) @ 0x124 { + 31:0 present; + } + + /// Shader core ready bitmap. Read only. + pub(crate) SHADER_READY_LO(u32) @ 0x140 { + 31:0 ready; + } + + pub(crate) SHADER_READY_HI(u32) @ 0x144 { + 31:0 ready; + } + + /// Tiler ready bitmap. Read only. + pub(crate) TILER_READY_LO(u32) @ 0x150 { + 31:0 ready; + } + + pub(crate) TILER_READY_HI(u32) @ 0x154 { + 31:0 ready; + } + + /// L2 ready bitmap. Read only. + pub(crate) L2_READY_LO(u32) @ 0x160 { + 31:0 ready; + } + + pub(crate) L2_READY_HI(u32) @ 0x164 { + 31:0 ready; + } + + /// Shader core power up bitmap. + pub(crate) SHADER_PWRON_LO(u32) @ 0x180 { + 31:0 request; + } + + pub(crate) SHADER_PWRON_HI(u32) @ 0x184 { + 31:0 request; + } + + /// Tiler power up bitmap. + pub(crate) TILER_PWRON_LO(u32) @ 0x190 { + 31:0 request; + } + + pub(crate) TILER_PWRON_HI(u32) @ 0x194 { + 31:0 request; + } + + /// L2 power up bitmap. + pub(crate) L2_PWRON_LO(u32) @ 0x1a0 { + 31:0 request; + } + + pub(crate) L2_PWRON_HI(u32) @ 0x1a4 { + 31:0 request; + } + + /// Shader core power down bitmap. + pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 { + 31:0 request; + } + + pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 { + 31:0 request; + } + + /// Tiler power down bitmap. + pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 { + 31:0 request; + } + + pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 { + 31:0 request; + } + + /// L2 power down bitmap. + pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 { + 31:0 request; + } + + pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 { + 31:0 request; + } + + /// Shader core power transition bitmap. Read-only. + pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 { + 31:0 changing; + } + + pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 { + 31:0 changing; + } + + /// Tiler power transition bitmap. Read-only. + pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 { + 31:0 changing; + } + + pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 { + 31:0 changing; + } + + /// L2 power transition bitmap. Read-only. + pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 { + 31:0 changing; + } + + pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 { + 31:0 changing; + } + + /// Shader core active bitmap. Read-only. + pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 { + 31:0 active; + } + + pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 { + 31:0 active; + } + + /// Tiler active bitmap. Read-only. + pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 { + 31:0 active; + } + + pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 { + 31:0 active; + } + + /// L2 active bitmap. Read-only. + pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 { + 31:0 active; + } + + pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 { + 31:0 active; + } + + /// Revision ID. Read only constant. + pub(crate) REVIDR(u32) @ 0x280 { + 31:0 revision; + } + + /// Coherency features present. Read only constant. + /// Supported protocols on the interconnect between the GPU and the + /// system into which it is integrated. + pub(crate) COHERENCY_FEATURES(u32) @ 0x300 { + /// ACE-Lite protocol supported, a 1-bit boolean flag. + 0:0 ace_lite => bool; + /// ACE protocol supported, a 1-bit boolean flag. + 1:1 ace => bool; + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum CoherencyMode { + /// ACE-Lite coherency protocol. + AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8, + /// ACE coherency protocol. + Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8, + /// No coherency protocol. + None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8, + } + + impl TryFrom> for CoherencyMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(CoherencyMode::AceLite), + 1 => Ok(CoherencyMode::Ace), + 31 => Ok(CoherencyMode::None), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: CoherencyMode) -> Self { + (mode as u8).into() + } + } + + register! { + /// Coherency enable. An index of which coherency protocols should be used. + /// This register only selects the protocol for coherency messages on the + /// interconnect. This is not to enable or disable coherency controlled by MMU. + pub(crate) COHERENCY_ENABLE(u32) @ 0x304 { + 31:0 l2_cache_protocol_select ?=> CoherencyMode; + } + } + + /// Helpers for MCU_CONTROL register + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum McuControlMode { + /// Disable the MCU. + Disable = 0, + /// Enable the MCU. + Enable = 1, + /// Enable the MCU to execute and automatically reboot after a fast reset. + Auto = 2, + } + + impl TryFrom> for McuControlMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(McuControlMode::Disable), + 1 => Ok(McuControlMode::Enable), + 2 => Ok(McuControlMode::Auto), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: McuControlMode) -> Self { + Bounded::try_new(mode as u32).unwrap() + } + } + + register! { + /// MCU control. + pub(crate) MCU_CONTROL(u32) @ 0x700 { + /// Request MCU state change. + 1:0 req ?=> McuControlMode; + } + } + + /// Helpers for MCU_STATUS register + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum McuStatus { + /// MCU is disabled. + Disabled = 0, + /// MCU is enabled. + Enabled = 1, + /// The MCU has halted by itself in an orderly manner to enable the core group to be + /// powered down. + Halt = 2, + /// The MCU has encountered an error that prevents it from continuing. + Fatal = 3, + } + + impl From> for McuStatus { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => McuStatus::Disabled, + 1 => McuStatus::Enabled, + 2 => McuStatus::Halt, + 3 => McuStatus::Fatal, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(status: McuStatus) -> Self { + Bounded::try_new(status as u32).unwrap() + } + } + + register! { + /// MCU status. Read only. + pub(crate) MCU_STATUS(u32) @ 0x704 { + /// Read current state of MCU. + 1:0 value => McuStatus; + } + } +} + +/// These registers correspond to the JOB_CONTROL register page. +/// They are involved in communication between the firmware running on the MCU and the host. +pub(crate) mod job_control { + use kernel::register; + + register! { + /// Raw status of job interrupts. + /// + /// Write to this register to trigger these interrupts. + /// Writing a 1 to a bit forces that bit on. + pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 { + /// CSG request. These bits indicate that CSGn requires attention from the host. + 30:0 csg; + /// GLB request. Indicates that the GLB interface requires attention from the host. + 31:31 glb => bool; + } + + /// Clear job interrupts. Write only. + /// + /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`]. + pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 { + /// Clear CSG request interrupts. + 30:0 csg; + /// Clear GLB request interrupt. + 31:31 glb => bool; + } + + /// Mask for job interrupts. + /// + /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it. + pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 { + /// Enable CSG request interrupts. + 30:0 csg; + /// Enable GLB request interrupt. + 31:31 glb => bool; + } + + /// Active job interrupts. Read only. + /// + /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and + /// [`JOB_IRQ_MASK`]. + pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c { + /// CSG request interrupt status. + 30:0 csg; + /// GLB request interrupt status. + 31:31 glb => bool; + } + } +} + +/// These registers correspond to the MMU_CONTROL register page. +/// They are involved in MMU configuration and control. +pub(crate) mod mmu_control { + use kernel::register; + + register! { + /// IRQ sources raw status. + /// + /// This register contains the raw unmasked interrupt sources for MMU status and exception + /// handling. + /// + /// Writing to this register forces bits on. + /// Use [`IRQ_CLEAR`] to clear interrupts. + pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 { + /// Page fault for address spaces. + 15:0 page_fault; + /// Command completed in address spaces. + 31:16 command_completed; + } + + /// IRQ sources to clear. + /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`]. + pub(crate) IRQ_CLEAR(u32) @ 0x2004 { + /// Clear the PAGE_FAULT interrupt. + 15:0 page_fault; + /// Clear the COMMAND_COMPLETED interrupt. + 31:16 command_completed; + } + + /// IRQ sources enabled. + /// + /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it. + pub(crate) IRQ_MASK(u32) @ 0x2008 { + /// Enable the PAGE_FAULT interrupt. + 15:0 page_fault; + /// Enable the COMMAND_COMPLETED interrupt. + 31:16 command_completed; + } + + /// IRQ status for enabled sources. Read only. + /// + /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`]. + pub(crate) IRQ_STATUS(u32) @ 0x200c { + /// PAGE_FAULT interrupt status. + 15:0 page_fault; + /// COMMAND_COMPLETED interrupt status. + 31:16 command_completed; + } + } + + /// Per-address space registers ASn [0..15] within the MMU_CONTROL page. + /// + /// This array contains 16 instances of the MMU_AS_CONTROL register page. + pub(crate) mod mmu_as_control { + use kernel::{ + num::Bounded, + prelude::*, + register, // + }; + + use pin_init::Zeroable; + + /// Maximum number of hardware address space slots. + /// The actual number of slots available is usually lower. + pub(crate) const MAX_AS: usize = 16; + + /// Address space register stride. The elements in the array are spaced 64B apart. + const STRIDE: usize = 0x40; + + register! { + /// Translation table base address. A 64-bit pointer. + /// + /// This field contains the address of the top level of a translation table structure. + /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero. + pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 { + /// Base address of the translation table. + 63:0 base; + } + + // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 { + 31:0 value; + } + + pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 { + 31:0 value; + } + } + + /// Helpers for MEMATTR Register. + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum AllocPolicySelect { + /// Ignore ALLOC_R/ALLOC_W fields. + Impl = 2, + /// Use ALLOC_R/ALLOC_W fields for allocation policy. + Alloc = 3, + } + + impl TryFrom> for AllocPolicySelect { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 2 => Ok(Self::Impl), + 3 => Ok(Self::Alloc), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(val: AllocPolicySelect) -> Self { + Bounded::try_new(val as u8).unwrap() + } + } + + /// Coherency policy for memory attributes. Indicates the shareability of cached accesses. + /// + /// The hardware spec defines different interpretations of these values depending on + /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation + /// tables (all input addresses map to the same output address); it is deprecated and not + /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum Coherency { + /// Midgard inner domain coherency. + /// + /// Most flexible mode - can map non-coherent, internally coherent, and system/IO + /// coherent memory. Used for non-cacheable memory in MAIR conversion. + MidgardInnerDomain = 0, + /// CPU inner domain coherency. + /// + /// Can map non-coherent and system/IO coherent memory. Used for write-back + /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency. + CpuInnerDomain = 1, + /// CPU inner domain with shader coherency. + /// + /// Can map internally coherent and system/IO coherent memory. Used for + /// GPU-internal shared buffers requiring shader coherency. + CpuInnerDomainShaderCoh = 2, + } + + impl TryFrom> for Coherency { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(Self::MidgardInnerDomain), + 1 => Ok(Self::CpuInnerDomain), + 2 => Ok(Self::CpuInnerDomainShaderCoh), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(val: Coherency) -> Self { + Bounded::try_new(val as u8).unwrap() + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MemoryType { + /// Normal memory (shared). + Shared = 0, + /// Normal memory, inner/outer non-cacheable. + NonCacheable = 1, + /// Normal memory, inner/outer write-back cacheable. + WriteBack = 2, + /// Triggers MEMORY_ATTRIBUTE_FAULT. + Fault = 3, + } + + impl From> for MemoryType { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => Self::Shared, + 1 => Self::NonCacheable, + 2 => Self::WriteBack, + 3 => Self::Fault, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(val: MemoryType) -> Self { + Bounded::try_new(val as u8).unwrap() + } + } + + register! { + /// Stage 1 memory attributes (8-bit bitfield). + /// + /// This is not an actual register, but a bitfield definition used by the MEMATTR + /// register. Each of the 8 bytes in MEMATTR follows this layout. + MMU_MEMATTR_STAGE1(u8) @ 0x0 { + /// Inner cache write allocation policy. + 0:0 alloc_w => bool; + /// Inner cache read allocation policy. + 1:1 alloc_r => bool; + /// Inner allocation policy select. + 3:2 alloc_sel ?=> AllocPolicySelect; + /// Coherency policy. + 5:4 coherency ?=> Coherency; + /// Memory type. + 7:6 memory_type => MemoryType; + } + } + + impl TryFrom> for MMU_MEMATTR_STAGE1 { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + Ok(Self::from_raw(val.get() as u8)) + } + } + + impl From for Bounded { + fn from(val: MMU_MEMATTR_STAGE1) -> Self { + Bounded::try_new(u64::from(val.into_raw())).unwrap() + } + } + + register! { + /// Memory attributes. + /// + /// Each address space can configure up to 8 different memory attribute profiles. + /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout. + pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 { + 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1; + 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1; + 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1; + 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1; + 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1; + 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1; + 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1; + 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1; + } + + // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 { + 31:0 value; + } + + pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { + 31:0 value; + } + } + + impl MEMATTR { + /// Outer cache-policy nibble indicating device memory. + const ARM_MAIR_DEVICE_MEMORY: u8 = 0x0; + + /// In the ARM Architecture Reference Manual, the MAIR encoding for Normal memory + /// uses the format `0bxxRW` where: + /// - `W` (bit 0) = Write-Allocate policy + /// - `R` (bit 1) = Read-Allocate policy + /// E.g., `0b0011` would allow both read and write allocation on a cache miss. + /// + /// ARM MAIR Write-Allocate bit (bit 0 of a cache policy nibble). + const ARM_MAIR_WRITE_ALLOCATE: u8 = 0x1; + /// ARM MAIR Read-Allocate bit (bit 1 of a cache policy nibble). + const ARM_MAIR_READ_ALLOCATE: u8 = 0x2; + + /// Write-back policy bit. For cacheable encodings, it is necessary but not + /// sufficient to set bit 2 of the cache policy nibble. Bit 2 does not + /// definitively determine write back because bit 2 is also set in `0b0100` + /// which encodes Normal non-cacheable memory. + const ARM_MAIR_WRITE_BACK_BIT: u8 = 0x4; + + /// Complete cache-policy nibble encoding for Normal Non-cacheable memory. + const ARM_MAIR_NON_CACHEABLE: u8 = 0x4; + + /// Mask for the inner cache policy nibble in MAIR attribute bytes. + const ARM_MAIR_INNER_MASK: u8 = 0x0f; + + /// Check if a MAIR attribute byte represents device memory. + /// + /// Device memory (memory-mapped I/O, registers) cannot be cached because + /// reading and writing to this memory may have side effects. + fn is_device_memory(mair_attr: u8) -> bool { + // In AArch64 MAIR, outer nibble only is 0 for device memory. + (mair_attr >> 4) == Self::ARM_MAIR_DEVICE_MEMORY + } + + /// Check if normal memory is fully write-back cacheable. + /// + /// ARM MAIR has two cache policy levels (outer [7:4] and inner [3:0]). + /// For memory to be truly write-back, BOTH levels must have the write-back bit set. + /// If only one level is write-back, treat it as non-cacheable for GPU purposes. + fn is_writeback_cacheable(mair_attr: u8) -> bool { + let outer = mair_attr >> 4; + let inner = mair_attr & Self::ARM_MAIR_INNER_MASK; + + outer != Self::ARM_MAIR_NON_CACHEABLE + && inner != Self::ARM_MAIR_NON_CACHEABLE + && (outer & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 + && (inner & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 + } + + // Helper to encode a MEMATTR attribute from its individual fields. + fn encode_attribute( + alloc_w: bool, + alloc_r: bool, + alloc_sel: AllocPolicySelect, + coherency: Coherency, + memory_type: MemoryType, + ) -> MMU_MEMATTR_STAGE1 { + MMU_MEMATTR_STAGE1::zeroed() + .with_alloc_w(alloc_w) + .with_alloc_r(alloc_r) + .with_alloc_sel(alloc_sel) + .with_coherency(coherency) + .with_memory_type(memory_type) + } + + /// Convert one MAIR attribute byte into a MEMATTR attribute. + // TODO: Add a `coherent` parameter like panthor's mair_to_memattr(). + // For now, assume a non-coherent system and always encode write-back + // memory with MidgardInnerDomain coherency. + fn attribute_from_mair(mair_attr: u8) -> MMU_MEMATTR_STAGE1 { + // Device memory or non-write-back normal memory + if Self::is_device_memory(mair_attr) || !Self::is_writeback_cacheable(mair_attr) { + return Self::encode_attribute( + false, + false, + AllocPolicySelect::Alloc, + Coherency::MidgardInnerDomain, + MemoryType::NonCacheable, + ); + } + + // Write-back cacheable normal memory + let inner: u8 = mair_attr & Self::ARM_MAIR_INNER_MASK; + Self::encode_attribute( + (inner & Self::ARM_MAIR_WRITE_ALLOCATE) != 0, + (inner & Self::ARM_MAIR_READ_ALLOCATE) != 0, + AllocPolicySelect::Alloc, + Coherency::MidgardInnerDomain, + MemoryType::WriteBack, + ) + } + + /// Write one converted MAIR attribute into a corresponding MEMATTR slot. + fn with_encoded_attribute(self, index: usize, attr: MMU_MEMATTR_STAGE1) -> Self { + debug_assert!(index < 8); + + let shift = index * 8; + let mask = !(0xffu64 << shift); + let raw = (self.into_raw() & mask) | ((u64::from(attr.into_raw())) << shift); + + Self::from_raw(raw) + } + + /// Convert an AArch64 MAIR value into the GPU MEMATTR register encoding. + /// + /// Both MAIR and MEMATTR are 64-bit values with eight 8-bit memory + /// attribute entries, but the bits do not map directly. The GPU MEMATTR encoding + /// is less detailed than the MAIR encoding, so MAIR is converted to MEMATTR + /// conservatively as follows: + /// + /// 1. Device memory, or Normal Memory that is not write-back cacheable, is encoded + /// as GPU `NonCacheable` + /// + /// 2. Normal memory that is write-back cacheable is encoded as GPU `WriteBack`, + /// and the inner allocation hints are preserved. + pub(crate) fn from_mair(mair: u64) -> Self { + mair.to_le_bytes() + .into_iter() + .enumerate() + .fold(Self::zeroed(), |acc, (i, attr)| { + acc.with_encoded_attribute(i, Self::attribute_from_mair(attr)) + }) + } + } + + register! { + /// Lock region address for each address space. + pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { + /// Lock region size. + 5:0 size; + /// Lock region base address. + 63:12 base; + } + + // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 { + 31:0 value; + } + + pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 { + 31:0 value; + } + } + + /// Helpers for MMU COMMAND register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MmuCommand { + /// No operation, nothing happens. + Nop = 0, + /// Propagate settings to the MMU. + Update = 1, + /// Lock an address region. + Lock = 2, + /// Unlock an address region. + Unlock = 3, + /// Clean and invalidate the L2 cache, then unlock. + FlushPt = 4, + /// Clean and invalidate all caches, then unlock. + FlushMem = 5, + } + + impl TryFrom> for MmuCommand { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(MmuCommand::Nop), + 1 => Ok(MmuCommand::Update), + 2 => Ok(MmuCommand::Lock), + 3 => Ok(MmuCommand::Unlock), + 4 => Ok(MmuCommand::FlushPt), + 5 => Ok(MmuCommand::FlushMem), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(cmd: MmuCommand) -> Self { + (cmd as u8).into() + } + } + + register! { + /// MMU command register for each address space. Write only. + pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { + 7:0 command ?=> MmuCommand; + } + } + + /// MMU exception types for FAULTSTATUS register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MmuExceptionType { + /// No error. + Ok = 0x00, + /// Invalid translation table entry, level 0. + TranslationFault0 = 0xC0, + /// Invalid translation table entry, level 1. + TranslationFault1 = 0xC1, + /// Invalid translation table entry, level 2. + TranslationFault2 = 0xC2, + /// Invalid translation table entry, level 3. + TranslationFault3 = 0xC3, + /// Invalid block descriptor. + TranslationFault4 = 0xC4, + /// Page permission error, level 0. + PermissionFault0 = 0xC8, + /// Page permission error, level 1. + PermissionFault1 = 0xC9, + /// Page permission error, level 2. + PermissionFault2 = 0xCA, + /// Page permission error, level 3. + PermissionFault3 = 0xCB, + /// Access flag not set, level 1. + AccessFlag1 = 0xD9, + /// Access flag not set, level 2. + AccessFlag2 = 0xDA, + /// Access flag not set, level 3. + AccessFlag3 = 0xDB, + /// Virtual address out of range. + AddressSizeFaultIn = 0xE0, + /// Physical address out of range, level 0. + AddressSizeFaultOut0 = 0xE4, + /// Physical address out of range, level 1. + AddressSizeFaultOut1 = 0xE5, + /// Physical address out of range, level 2. + AddressSizeFaultOut2 = 0xE6, + /// Physical address out of range, level 3. + AddressSizeFaultOut3 = 0xE7, + /// Page attribute error, level 0. + MemoryAttributeFault0 = 0xE8, + /// Page attribute error, level 1. + MemoryAttributeFault1 = 0xE9, + /// Page attribute error, level 2. + MemoryAttributeFault2 = 0xEA, + /// Page attribute error, level 3. + MemoryAttributeFault3 = 0xEB, + } + + impl TryFrom> for MmuExceptionType { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0x00 => Ok(MmuExceptionType::Ok), + 0xC0 => Ok(MmuExceptionType::TranslationFault0), + 0xC1 => Ok(MmuExceptionType::TranslationFault1), + 0xC2 => Ok(MmuExceptionType::TranslationFault2), + 0xC3 => Ok(MmuExceptionType::TranslationFault3), + 0xC4 => Ok(MmuExceptionType::TranslationFault4), + 0xC8 => Ok(MmuExceptionType::PermissionFault0), + 0xC9 => Ok(MmuExceptionType::PermissionFault1), + 0xCA => Ok(MmuExceptionType::PermissionFault2), + 0xCB => Ok(MmuExceptionType::PermissionFault3), + 0xD9 => Ok(MmuExceptionType::AccessFlag1), + 0xDA => Ok(MmuExceptionType::AccessFlag2), + 0xDB => Ok(MmuExceptionType::AccessFlag3), + 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn), + 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0), + 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1), + 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2), + 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3), + 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0), + 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1), + 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2), + 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(exc: MmuExceptionType) -> Self { + (exc as u8).into() + } + } + + /// Access type for MMU faults. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MmuAccessType { + /// An atomic (read/write) transaction. + Atomic = 0, + /// An execute transaction. + Execute = 1, + /// A read transaction. + Read = 2, + /// A write transaction. + Write = 3, + } + + impl From> for MmuAccessType { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => MmuAccessType::Atomic, + 1 => MmuAccessType::Execute, + 2 => MmuAccessType::Read, + 3 => MmuAccessType::Write, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(access: MmuAccessType) -> Self { + Bounded::try_new(access as u32).unwrap() + } + } + + register! { + /// Fault status register for each address space. Read only. + pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { + /// Exception type. + 7:0 exception_type ?=> MmuExceptionType; + /// Access type. + 9:8 access_type => MmuAccessType; + /// ID of the source that triggered the fault. + 31:16 source_id; + } + + /// Fault address for each address space. Read only. + pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 { + 31:0 pointer; + } + + pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 { + 31:0 pointer; + } + + /// MMU status register for each address space. Read only. + pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 { + /// External address space command is active, a 1-bit boolean flag. + 0:0 active_ext => bool; + /// Internal address space command is active, a 1-bit boolean flag. + 1:1 active_int => bool; + } + } + + /// Helpers for TRANSCFG register. + /// + /// Address space mode for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum AddressSpaceMode { + /// The MMU forces all memory access to fail with a decode fault. + Unmapped = 1, + /// All input addresses map to the same output address (deprecated). + Identity = 2, + /// Translation tables interpreted according to AArch64 4kB granule specification. + Aarch64_4K = 6, + /// Translation tables interpreted according to AArch64 64kB granule specification. + Aarch64_64K = 8, + } + + impl TryFrom> for AddressSpaceMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 1 => Ok(AddressSpaceMode::Unmapped), + 2 => Ok(AddressSpaceMode::Identity), + 6 => Ok(AddressSpaceMode::Aarch64_4K), + 8 => Ok(AddressSpaceMode::Aarch64_64K), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: AddressSpaceMode) -> Self { + Bounded::try_new(mode as u64).unwrap() + } + } + + /// Input address range restriction for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum InaBits { + /// Invalid VA range (reset value). + Reset = 0, + /// 48-bit VA range. + Bits48 = 7, + /// 47-bit VA range. + Bits47 = 8, + /// 46-bit VA range. + Bits46 = 9, + /// 45-bit VA range. + Bits45 = 10, + /// 44-bit VA range. + Bits44 = 11, + /// 43-bit VA range. + Bits43 = 12, + /// 42-bit VA range. + Bits42 = 13, + /// 41-bit VA range. + Bits41 = 14, + /// 40-bit VA range. + Bits40 = 15, + /// 39-bit VA range. + Bits39 = 16, + /// 38-bit VA range. + Bits38 = 17, + /// 37-bit VA range. + Bits37 = 18, + /// 36-bit VA range. + Bits36 = 19, + /// 35-bit VA range. + Bits35 = 20, + /// 34-bit VA range. + Bits34 = 21, + /// 33-bit VA range. + Bits33 = 22, + /// 32-bit VA range. + Bits32 = 23, + /// 31-bit VA range. + Bits31 = 24, + /// 30-bit VA range. + Bits30 = 25, + /// 29-bit VA range. + Bits29 = 26, + /// 28-bit VA range. + Bits28 = 27, + /// 27-bit VA range. + Bits27 = 28, + /// 26-bit VA range. + Bits26 = 29, + /// 25-bit VA range. + Bits25 = 30, + } + + impl TryFrom> for InaBits { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(InaBits::Reset), + 7 => Ok(InaBits::Bits48), + 8 => Ok(InaBits::Bits47), + 9 => Ok(InaBits::Bits46), + 10 => Ok(InaBits::Bits45), + 11 => Ok(InaBits::Bits44), + 12 => Ok(InaBits::Bits43), + 13 => Ok(InaBits::Bits42), + 14 => Ok(InaBits::Bits41), + 15 => Ok(InaBits::Bits40), + 16 => Ok(InaBits::Bits39), + 17 => Ok(InaBits::Bits38), + 18 => Ok(InaBits::Bits37), + 19 => Ok(InaBits::Bits36), + 20 => Ok(InaBits::Bits35), + 21 => Ok(InaBits::Bits34), + 22 => Ok(InaBits::Bits33), + 23 => Ok(InaBits::Bits32), + 24 => Ok(InaBits::Bits31), + 25 => Ok(InaBits::Bits30), + 26 => Ok(InaBits::Bits29), + 27 => Ok(InaBits::Bits28), + 28 => Ok(InaBits::Bits27), + 29 => Ok(InaBits::Bits26), + 30 => Ok(InaBits::Bits25), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(bits: InaBits) -> Self { + Bounded::try_new(bits as u64).unwrap() + } + } + + /// Translation table memory attributes for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum PtwMemattr { + /// Invalid (reset value, not valid for enabled address space). + Invalid = 0, + /// Normal memory, inner/outer non-cacheable. + NonCacheable = 1, + /// Normal memory, inner/outer write-back cacheable. + WriteBack = 2, + } + + impl TryFrom> for PtwMemattr { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(PtwMemattr::Invalid), + 1 => Ok(PtwMemattr::NonCacheable), + 2 => Ok(PtwMemattr::WriteBack), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(attr: PtwMemattr) -> Self { + Bounded::try_new(attr as u64).unwrap() + } + } + + /// Translation table memory shareability for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + #[allow(clippy::enum_variant_names)] + pub(crate) enum PtwShareability { + /// Non-shareable. + NonShareable = 0, + /// Outer shareable. + OuterShareable = 2, + /// Inner shareable. + InnerShareable = 3, + } + + impl TryFrom> for PtwShareability { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(PtwShareability::NonShareable), + 2 => Ok(PtwShareability::OuterShareable), + 3 => Ok(PtwShareability::InnerShareable), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(sh: PtwShareability) -> Self { + Bounded::try_new(sh as u64).unwrap() + } + } + + register! { + /// Translation configuration and control. + pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { + /// Address space mode. + 3:0 mode ?=> AddressSpaceMode; + /// Address input restriction. + 10:6 ina_bits ?=> InaBits; + /// Address output restriction. + 18:14 outa_bits; + /// Translation table concatenation enable, a 1-bit boolean flag. + 22:22 sl_concat_en => bool; + /// Translation table memory attributes. + 25:24 ptw_memattr ?=> PtwMemattr; + /// Translation table memory shareability. + 29:28 ptw_sh ?=> PtwShareability; + /// Inner read allocation hint for translation table walks, a 1-bit boolean flag. + 30:30 r_allocate => bool; + /// Disable hierarchical access permissions. + 33:33 disable_hier_ap => bool; + /// Disable access fault checking. + 34:34 disable_af_fault => bool; + /// Disable execution on all writable pages. + 35:35 wxn => bool; + /// Enable execution on readable pages. + 36:36 xreadable => bool; + /// Page-based hardware attributes for translation table walks. + 63:60 ptw_pbha; + } + + // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 { + 31:0 value; + } + + pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 { + 31:0 value; + } + + /// Extra fault information for each address space. Read only. + pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 { + 31:0 value; + } + + pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c { + 31:0 value; + } + } + } +} + +/// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages. +pub(crate) mod doorbell_block { + use kernel::register; + + /// Number of doorbells available. + pub(crate) const NUM_DOORBELLS: usize = 64; + + /// Doorbell block stride (64KiB). + /// + /// Each block occupies a full page, allowing it to be mapped + /// separately into a virtual address space. + const STRIDE: usize = 0x10000; + + register! { + /// Doorbell request register. Write-only. + pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { + /// Doorbell set. Writing 1 triggers the doorbell. + 0:0 ring => bool; + } + } +} diff --git a/drivers/gpu/drm/tyr/slot.rs b/drivers/gpu/drm/tyr/slot.rs new file mode 100644 index 000000000..d194ead53 --- /dev/null +++ b/drivers/gpu/drm/tyr/slot.rs @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Slot management abstraction for limited hardware resources. +//! +//! This module provides a generic [`SlotManager`] that assigns limited hardware +//! slots to logical "seats". A seat represents an entity (such as a virtual memory +//! (VM) address space) that needs access to a hardware slot. +//! +//! The [`SlotManager`] tracks slot allocation using sequence numbers (seqno) to detect +//! when a seat's binding has been invalidated. When a seat requests activation, +//! the manager will either reuse the seat's existing slot (if still valid), +//! allocate a free slot (if any are available), or evict the oldest idle slot if any +//! slots are idle. +//! +//! Hardware-specific behavior is customized by implementing the [`SlotOperations`] +//! trait, which allows callbacks when slots are activated or evicted. +//! +//! This is currently used for managing address space slots in the GPU, and it will +//! also be used to manage Command Stream Group (CSG) interface slots in the future. +//! +//! [SlotOperations]: crate::slot::SlotOperations +//! [SlotManager]: crate::slot::SlotManager + +use core::{ + mem, + ops::{ + Deref, + DerefMut, // + }, // +}; + +use kernel::{ + prelude::*, + sync::LockedBy, // +}; + +/// Seat information. +/// +/// This can't be accessed directly by the element embedding a `Seat`, +/// but is used by the generic slot manager logic to control residency +/// of a certain object on a hardware slot. +pub(crate) struct SeatInfo { + /// Slot used by this seat. + /// + /// This index is only valid if the slot pointed to by this index + /// has its `SlotInfo::seqno` match `SeatInfo::seqno`. Otherwise, + /// it means the object has been evicted from the hardware slot, + /// and a new slot needs to be acquired to make this object + /// resident again. + slot: u8, + + /// Sequence number encoding the last time this seat was active. + /// We also use it to check if a slot is still bound to a seat. + seqno: u64, +} + +/// Seat state. +/// +/// This is meant to be embedded in the object that wants to acquire +/// hardware slots. It also starts in the `Seat::NoSeat` state, and +/// the slot manager will change the object value when an active/evict +/// request is issued. +#[derive(Default)] +pub(crate) enum Seat { + #[expect(clippy::enum_variant_names)] + /// Resource is not resident. + /// + /// All objects start with a seat in the `Seat::NoSeat` state. The seat also + /// gets back to that state if the user requests eviction. It + /// can also end up in that state next time an operation is done + /// on a `Seat::Idle` seat and the slot manager finds out this + /// object has been evicted from the slot. + #[default] + NoSeat, + + /// Resource is actively used and resident. + /// + /// When a seat is in the `Seat::Active` state, it can't be evicted, and the + /// slot pointed to by `SeatInfo::slot` is guaranteed to be reserved + /// for this object as long as the seat stays active. + Active(SeatInfo), + + /// Resource is idle and might or might not be resident. + /// + /// When a seat is in the`Seat::Idle` state, we can't know for sure if the + /// object is resident or evicted until the next request we issue + /// to the slot manager. This tells the slot manager it can + /// reclaim the underlying slot if needed. + /// In order for the hardware to use this object again, the seat + /// needs to be turned into an `Seat::Active` state again + /// with a `SlotManager::activate()` call. + Idle(SeatInfo), +} + +impl Seat { + /// Get the slot index this seat is pointing to. + /// + /// If the seat is not `Seat::Active` we can't trust the + /// `SeatInfo`. In that case `None` is returned, otherwise + /// `Some(SeatInfo::slot)` is returned. + pub(crate) fn slot(&self) -> Option { + match self { + Self::Active(info) => Some(info.slot), + _ => None, + } + } +} + +/// Information related to a slot. +struct SlotInfo { + /// Type specific data attached to a slot. + slot_data: D, + + /// Sequence number from when this slot was last activated. + seqno: u64, +} + +/// Slot state. +#[derive(Default)] +enum Slot { + /// Slot is free. + #[default] + Free, + + /// Slot is active. + Active(SlotInfo), + + /// Slot is idle. + Idle(SlotInfo), +} + +pub(crate) type LockedSeat = LockedBy>; + +/// Trait describing the slot-related operations. +pub(crate) trait SlotOperations: Sized { + /// Implementation-specific data associated with each slot. + type SlotData; + + /// Returns the seat belonging to this slot data. + fn seat(slot_data: &Self::SlotData) -> &LockedSeat; + + /// Called when a slot is being activated for a seat. + fn activate(&mut self, _slot_idx: usize, _slot_data: &Self::SlotData) -> Result { + Ok(()) + } + + /// Called when a slot is being evicted and freed. + fn evict(&mut self, _slot_idx: usize, _slot_data: &Self::SlotData) -> Result { + Ok(()) + } +} + +/// A generic slot manager that provides access to a limited number of hardware slots. +pub(crate) struct SlotManager, const MAX_SLOTS: usize> { + /// A specific implementation of the generic slot manager. + manager: T, + + /// Number of slots actually available. + slot_count: usize, + + /// Slot array used to track the state of each slot. + slots: [Slot; MAX_SLOTS], + + /// Sequence number incremented each time a Seat is successfully activated + use_seqno: u64, +} + +impl, const MAX_SLOTS: usize> SlotManager { + /// Creates a specific instance of a slot manager. + pub(crate) fn new(manager: T, slot_count: usize) -> Result { + if slot_count == 0 { + return Err(EINVAL); + } + if slot_count > MAX_SLOTS { + return Err(EINVAL); + } + // Since the slot index is stored in SeatInfo as a u8, the maximum number of slots is 256. + if slot_count > u8::MAX as usize + 1 { + return Err(EINVAL); + } + + Ok(Self { + manager, + slot_count, + slots: [const { Slot::Free }; MAX_SLOTS], + use_seqno: 1, + }) + } + + /// Records a newly activated slot for the given seat. + /// The slot manager takes ownership of the hardware-specific slot data. + fn record_active_slot(&mut self, slot_idx: usize, slot_data: T::SlotData) { + let cur_seqno = self.use_seqno; + + *T::seat(&slot_data).access_mut(self) = Seat::Active(SeatInfo { + slot: slot_idx as u8, + seqno: cur_seqno, + }); + + self.slots[slot_idx] = Slot::Active(SlotInfo { + slot_data, + seqno: cur_seqno, + }); + + self.use_seqno += 1; + } + + /// Reactivates an active/idle slot for a given seat without reprogramming the hardware. + /// The SlotManager reuses the existing slot_data. This ensures that the hardware-specific + /// information is not changed between subsequent uses. It also ensures that resources + /// owned by the existing slot_data remain alive while the hardware is configured to use them. + fn reactivate_slot(&mut self, slot_idx: usize, slot_data: &T::SlotData) -> Result { + let cur_seqno = self.use_seqno; + + let mut slot_info = match mem::take(&mut self.slots[slot_idx]) { + Slot::Active(slot_info) | Slot::Idle(slot_info) => slot_info, + Slot::Free => { + *T::seat(slot_data).access_mut(self) = Seat::NoSeat; + return Err(EINVAL); + } + }; + + *T::seat(slot_data).access_mut(self) = Seat::Active(SeatInfo { + slot: slot_idx as u8, + seqno: cur_seqno, + }); + + slot_info.seqno = cur_seqno; + self.slots[slot_idx] = Slot::Active(slot_info); + + self.use_seqno += 1; + + Ok(()) + } + + /// Activates a slot for the given seat. + fn activate_slot(&mut self, slot_idx: usize, slot_data: T::SlotData) -> Result { + self.manager.activate(slot_idx, &slot_data)?; + self.record_active_slot(slot_idx, slot_data); + Ok(()) + } + + /// Finds a slot for the given seat. A free slot is preferred, but if none + /// are available, the oldest idle slot is evicted and reused. Otherwise, if + /// there are no free or idle slots, return [`EBUSY`]. + fn allocate_slot(&mut self, slot_data: T::SlotData) -> Result { + let slots = &self.slots[..self.slot_count]; + + let mut idle_slot_idx = None; + let mut idle_slot_seqno: u64 = 0; + + for (slot_idx, slot) in slots.iter().enumerate() { + match slot { + Slot::Free => { + return self.activate_slot(slot_idx, slot_data); + } + Slot::Idle(slot_info) => { + if idle_slot_idx.is_none() || slot_info.seqno < idle_slot_seqno { + idle_slot_idx = Some(slot_idx); + idle_slot_seqno = slot_info.seqno; + } + } + Slot::Active(_) => (), + } + } + + match idle_slot_idx { + Some(slot_idx) => { + // Lazily evict idle slot just before it is reused. + if let Slot::Idle(slot_info) = &self.slots[slot_idx] { + self.manager.evict(slot_idx, &slot_info.slot_data)?; + mem::take(&mut self.slots[slot_idx]); + } + self.activate_slot(slot_idx, slot_data) + } + None => Err(EBUSY), + } + } + + /// Converts an active slot and its seat to idle state. + fn idle_slot(&mut self, slot_idx: usize, locked_seat: &LockedSeat) -> Result { + let slot = mem::take(&mut self.slots[slot_idx]); + + self.slots[slot_idx] = match slot { + // If the slot was active, make it idle. + Slot::Active(slot_info) => Slot::Idle(slot_info), + + // Preserve an already-idle slot. + Slot::Idle(slot_info) => Slot::Idle(slot_info), + + // A free slot remains free. + Slot::Free => Slot::Free, + }; + + // If the seat was active, make it idle, or keep it idle if it was already idle. + *locked_seat.access_mut(self) = match locked_seat.access(self) { + Seat::Active(seat_info) | Seat::Idle(seat_info) => Seat::Idle(SeatInfo { + slot: seat_info.slot, + seqno: seat_info.seqno, + }), + Seat::NoSeat => Seat::NoSeat, + }; + Ok(()) + } + + /// Evicts an active or idle slot: calls the eviction callback and marks the slot as free + /// and the seat as NoSeat. + fn evict_slot(&mut self, slot_idx: usize, locked_seat: &LockedSeat) -> Result { + match &self.slots[slot_idx] { + Slot::Active(slot_info) | Slot::Idle(slot_info) => { + // If hardware eviction fails (e.g. times out), the slot retains + // its SlotData so that any resources still referenced by the hardware + // will remain alive. This prevents use-after-free errors. + self.manager.evict(slot_idx, &slot_info.slot_data)?; + mem::take(&mut self.slots[slot_idx]); + } + _ => (), + } + + *locked_seat.access_mut(self) = Seat::NoSeat; + Ok(()) + } + + /// Checks that the seat state matches the slot's state. + /// If they don't match, the seat is stale and is reset to `NoSeat`. + fn check_seat(&mut self, locked_seat: &LockedSeat) { + let (slot_idx, seat_seqno, is_active) = match locked_seat.access(self) { + Seat::Active(seat_info) => (seat_info.slot as usize, seat_info.seqno, true), + Seat::Idle(seat_info) => (seat_info.slot as usize, seat_info.seqno, false), + _ => return, + }; + + let valid = if is_active { + !kernel::warn_on!(!matches!( + &self.slots[slot_idx], + Slot::Active(slot_info) if slot_info.seqno == seat_seqno + )) + } else { + matches!( + &self.slots[slot_idx], + Slot::Idle(slot_info) if slot_info.seqno == seat_seqno + ) + }; + + if !valid { + *locked_seat.access_mut(self) = Seat::NoSeat; + } + } + + /// Activates a resource on any available/reclaimable slot. + pub(crate) fn activate(&mut self, slot_data: T::SlotData) -> Result { + self.check_seat(T::seat(&slot_data)); + + // Copy out only the slot index so the borrow of slot_data ends here. + let slot_idx = match T::seat(&slot_data).access(self) { + Seat::Active(seat_info) | Seat::Idle(seat_info) => Some(seat_info.slot as usize), + Seat::NoSeat => None, + }; + + match slot_idx { + Some(slot_idx) => self.reactivate_slot(slot_idx, &slot_data), + None => self.allocate_slot(slot_data), + } + } + + /// Flag a resource as idle. This method will be used for user VM support. + #[expect(dead_code)] + pub(crate) fn idle(&mut self, locked_seat: &LockedSeat) -> Result { + self.check_seat(locked_seat); + if let Seat::Active(seat_info) = locked_seat.access(self) { + self.idle_slot(seat_info.slot as usize, locked_seat)?; + } + Ok(()) + } + + /// Evict a resource from its slot. + pub(crate) fn evict(&mut self, locked_seat: &LockedSeat) -> Result { + self.check_seat(locked_seat); + + match locked_seat.access(self) { + Seat::Active(seat_info) | Seat::Idle(seat_info) => { + let slot_idx = seat_info.slot as usize; + self.evict_slot(slot_idx, locked_seat)?; + } + _ => (), + } + + Ok(()) + } +} + +impl, const MAX_SLOTS: usize> Deref for SlotManager { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.manager + } +} + +impl, const MAX_SLOTS: usize> DerefMut for SlotManager { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.manager + } +} diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs new file mode 100644 index 000000000..e7ec450bd --- /dev/null +++ b/drivers/gpu/drm/tyr/tyr.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Arm Mali Tyr DRM driver. +//! +//! The name "Tyr" is inspired by Norse mythology, reflecting Arm's tradition of +//! naming their GPUs after Nordic mythological figures and places. + +use crate::driver::TyrPlatformDriver; + +mod driver; +mod file; +mod fw; +mod gem; +mod gpu; +mod mmu; +mod regs; +mod slot; +mod vm; + +kernel::module_platform_driver! { + type: TyrPlatformDriver, + name: "tyr", + authors: ["The Tyr driver authors"], + description: "Arm Mali Tyr DRM driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/gpu/drm/tyr/vm.rs b/drivers/gpu/drm/tyr/vm.rs new file mode 100644 index 000000000..c5e307b1e --- /dev/null +++ b/drivers/gpu/drm/tyr/vm.rs @@ -0,0 +1,950 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! GPU virtual memory management using the DRM GPUVM framework. +//! +//! This module manages GPU virtual address spaces, providing memory isolation and +//! the illusion of owning the entire virtual address (VA) range, similar to CPU virtual memory. +//! Each virtual memory (VM) area is backed by ARM64 LPAE Stage 1 page tables and can be +//! mapped into hardware address space (AS) slots for GPU execution. + +use core::marker::PhantomData; +use core::ops::Range; + +use kernel::{ + device::{ + Bound, + Device, // + }, + drm::{ + gem::BaseObject, + gpuvm::{ + DriverGpuVm, + GpuVaAlloc, + GpuVm, + GpuVmBo, + OpMap, + OpMapRequest, + OpMapped, + OpRemap, + OpRemapped, + OpUnmap, + OpUnmapped, + UniqueRefGpuVm, // + }, // + }, + fmt, + impl_flags, + io::PhysAddr, + iommu::pgtable::{ + prot, + IoPageTable, + ARM64LPAES1, // + }, + new_mutex, + prelude::*, + sizes::{ + SZ_1G, + SZ_2M, + SZ_4K, // + }, + sync::{ + aref::ARef, + Arc, + ArcBorrow, + Mutex, // + }, + uapi, // +}; + +use crate::{ + driver::{ + TyrDrmDevice, + TyrDrmDriver, // + }, + gem, + gem::Bo, + gpu::GpuInfo, + mmu::{ + address_space::VmAsData, + Mmu, // + }, + regs::gpu_control::MMU_FEATURES, +}; + +impl_flags!( + /// Flags controlling virtual memory mapping behavior. + /// + /// These flags control access permissions and caching behavior for GPU virtual + /// memory mappings. + #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] + pub(crate) struct VmMapFlags(u32); + + /// Individual flags that can be combined in [`VmMapFlags`]. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum VmFlag { + /// Map as read-only. + Readonly = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_READONLY as u32, + /// Map as non-executable. + Noexec = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC as u32, + /// Map as uncached. + Uncached = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED as u32, + } +); + +impl VmMapFlags { + /// Convert the flags to `pgtable::prot`. + fn to_prot(self) -> u32 { + let mut prot = 0; + + if self.contains(VmFlag::Readonly) { + prot |= prot::READ; + } else { + prot |= prot::READ | prot::WRITE; + } + + if self.contains(VmFlag::Noexec) { + prot |= prot::NOEXEC; + } + + if !self.contains(VmFlag::Uncached) { + prot |= prot::CACHE; + } + + prot + } +} + +impl fmt::Display for VmMapFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut first = true; + + if self.contains(VmFlag::Readonly) { + write!(f, "READONLY")?; + first = false; + } + if self.contains(VmFlag::Noexec) { + if !first { + write!(f, " | ")?; + } + write!(f, "NOEXEC")?; + first = false; + } + + if self.contains(VmFlag::Uncached) { + if !first { + write!(f, " | ")?; + } + write!(f, "UNCACHED")?; + } + + Ok(()) + } +} + +impl TryFrom for VmMapFlags { + type Error = Error; + + fn try_from(value: u32) -> Result { + let valid = VmFlag::Readonly as u32 | VmFlag::Noexec as u32 | VmFlag::Uncached as u32; + + if value & !valid != 0 { + return Err(EINVAL); + } + Ok(Self(value)) + } +} + +/// Arguments for a virtual memory map operation. +struct VmMapArgs<'drm> { + /// Access permissions and caching behavior for the mapping. + flags: VmMapFlags, + /// GEM buffer object registered with the GPUVM framework. + vm_bo: ARef>>, + /// Offset in bytes from the start of the buffer object. + bo_offset: u64, +} + +/// Type of virtual memory operation. +enum VmOpType<'drm> { + /// Map a GEM buffer object into the virtual address space. + Map(VmMapArgs<'drm>), + /// Unmap a region from the virtual address space. + Unmap, +} + +/// Preallocated resources needed to execute a VM operation. +/// +/// VM operations may require allocating new GPUVA objects to track mappings. +/// To avoid allocation failures during the operation, preallocate the +/// maximum number of GPUVAs that might be needed. +struct VmOpResources<'drm> { + /// Preallocated GPUVA objects for remap operations. + /// + /// Partial unmap requests or map requests overlapping existing mappings + /// will trigger a remap call, which needs to register up to three VA + /// objects (one for the new mapping, and two for the previous and next + /// mappings). + preallocated_gpuvas: [Option>>; 3], +} + +/// Request to execute a virtual memory operation. +struct VmOpRequest<'drm> { + /// Request type. + op_type: VmOpType<'drm>, + + /// Region of the virtual address space covered by this request. + region: Range, +} + +/// Arguments for a page table map operation. +struct PtMapArgs { + /// Memory protection flags describing allowed accesses for this mapping. + /// + /// This is directly derived from [`VmMapFlags`] via [`VmMapFlags::to_prot`]. + prot: u32, +} + +/// Type of page table operation. +enum PtOpType { + /// Map pages into the page table. + Map(PtMapArgs), + /// Unmap pages from the page table. + Unmap, +} + +/// Context for updating the GPU page table. +/// +/// This context is created when beginning a page table update operation and +/// automatically flushes changes when dropped. It ensures that the +/// Memory Management Unit (MMU) state is properly managed and Translation +/// Lookaside Buffer (TLB) entries are flushed. +pub(crate) struct PtUpdateContext<'ctx, 'drm> { + /// Device used for DMA-mapping GEM shmem SG tables. + dev: &'ctx Device, + + /// Page table. + pt: &'ctx IoPageTable<'drm, ARM64LPAES1>, + + /// MMU manager. + mmu: &'ctx Mmu<'drm>, + + /// Reference to the address space data to pass to the MMU functions. + as_data: &'ctx VmAsData<'drm>, + + /// Region of the virtual address space covered by this request. + region: Range, + + /// Operation type. + op_type: PtOpType, + + /// Preallocated resources that can be used when executing the request. + resources: &'ctx mut VmOpResources<'drm>, +} + +impl<'ctx, 'drm> PtUpdateContext<'ctx, 'drm> { + /// Creates a new page table update context. + /// + /// This prepares the MMU for a page table update. + /// The context will automatically flush the TLB and + /// complete the update when dropped. + fn new( + dev: &'ctx Device, + pt: &'ctx IoPageTable<'drm, ARM64LPAES1>, + mmu: &'ctx Mmu<'drm>, + as_data: &'ctx VmAsData<'drm>, + region: Range, + op_type: PtOpType, + resources: &'ctx mut VmOpResources<'drm>, + ) -> Result> { + mmu.start_vm_update(as_data, ®ion)?; + + Ok(Self { + dev, + pt, + mmu, + as_data, + region, + op_type, + resources, + }) + } + + /// Finds one of our pre-allocated VAs. + fn preallocated_gpuva(&mut self) -> Result>> { + self.resources + .preallocated_gpuvas + .iter_mut() + .find_map(|f| f.take()) + .ok_or(EINVAL) + } + + /// Returns an unused GPUVA object to the preallocated pool. + /// If the pool is already full, the unused allocation is simply dropped. + fn return_preallocated_gpuva(&mut self, gpuva: GpuVaAlloc>) { + if let Some(slot) = self + .resources + .preallocated_gpuvas + .iter_mut() + .find(|slot| slot.is_none()) + { + *slot = Some(gpuva); + } + } +} + +impl Drop for PtUpdateContext<'_, '_> { + fn drop(&mut self) { + if let Err(e) = self.mmu.end_vm_update(self.as_data) { + dev_err!(self.dev, "Failed to end VM update {:?}", e); + } + + if let Err(e) = self.mmu.flush_vm(self.as_data) { + dev_err!(self.dev, "Failed to flush VM {:?}", e); + } + } +} + +/// Driver implementation for the GPUVM framework. +/// +/// Implements [`DriverGpuVm`] to provide VM operation callbacks (map, unmap, remap) +/// and associated types for buffer objects, virtual addresses, and contexts. +pub(crate) struct GpuVmData<'drm> { + _phantom: PhantomData<&'drm ()>, +} + +/// GPU virtual address space. +/// +/// Each VM can be mapped into a hardware address space slot. +#[pin_data] +pub(crate) struct Vm<'drm> { + /// Data referenced by an AS when the VM is active + as_data: Arc>, + /// MMU manager. + mmu: Arc>, + /// Parent device used for DMA mapping and page-table operations. + dev: &'drm Device, + /// DRM GPUVM core for managing virtual address space. + #[pin] + gpuvm_unique: Mutex>>, + /// Non-core part of the GPUVM. Can be used for stuff that doesn't modify the + /// internal mapping tree, like GpuVm::obtain() + gpuvm: ARef>>, + /// VA range for this VM. + va_range: Range, +} + +impl<'drm> Vm<'drm> { + /// Creates a new GPU virtual address space. + /// + /// The VM is initialized with a page table configured according to the GPU's + /// address translation capabilities and registered with the GPUVM framework. + pub(crate) fn new( + dev: &'drm Device, + ddev: &TyrDrmDevice, + mmu: ArcBorrow<'_, Mmu<'drm>>, + gpu_info: &GpuInfo, + ) -> Result>> { + let mmu_features = MMU_FEATURES::from_raw(gpu_info.mmu_features); + let va_bits = mmu_features.va_bits().get(); + let pa_bits = mmu_features.pa_bits().get(); + + let range = 0..(1u64 << va_bits); + let reserve_range = 0..0u64; + + // dummy_obj is used to initialize the GPUVM tree. + let dummy_obj = gem::new_dummy_object(ddev).inspect_err(|e| { + dev_err!(dev, "Failed to create dummy GEM object: {:?}", e); + })?; + + let gpuvm_unique = GpuVm::new::( + c"Tyr::GpuVm", + ddev, + &*dummy_obj, + range.clone(), + reserve_range, + GpuVmData::<'drm> { + _phantom: PhantomData::<&()>, + }, + ) + .inspect_err(|e| { + dev_err!(dev, "Failed to create GpuVm: {:?}", e); + })?; + let gpuvm = ARef::from(&*gpuvm_unique); + + let as_data = Arc::pin_init(VmAsData::new(&mmu, dev, va_bits, pa_bits), GFP_KERNEL)?; + + let vm = Arc::pin_init( + pin_init!(Self{ + as_data, + dev, + mmu: mmu.into(), + gpuvm, + gpuvm_unique <- new_mutex!(gpuvm_unique), + va_range: range, + }), + GFP_KERNEL, + )?; + + Ok(vm) + } + + /// Returns the parent device used by this VM for DMA mapping and page-table operations. + pub(crate) fn dev(&self) -> &'drm Device { + self.dev + } + + /// Activate the VM in a hardware address space slot. + pub(crate) fn activate(&self) -> Result { + self.mmu + .activate_vm(self.as_data.as_arc_borrow()) + .inspect_err(|e| { + dev_err!(self.dev, "Failed to activate VM: {:?}", e); + }) + } + + /// Deactivate the VM by evicting it from its address space slot. + fn deactivate(&self) -> Result { + self.mmu.deactivate_vm(&self.as_data).inspect_err(|e| { + dev_err!(self.dev, "Failed to deactivate VM: {:?}", e); + }) + } + + /// Kills the VM by deactivating it and unmapping all regions. + pub(crate) fn kill(&self) { + // TODO: Turn the VM into a state where it can't be used. + let _ = self.deactivate(); + let _ = self + .unmap_range(self.va_range.start, self.va_range.end - self.va_range.start) + .inspect_err(|e| { + dev_err!(self.dev, "Failed to unmap range during deactivate: {:?}", e); + }); + } + + /// Executes a virtual memory operation. + /// + /// This handles both map and unmap operations by coordinating between the + /// GPUVM framework and the hardware page table. + fn exec_op<'a>( + &self, + gpuvm_unique: &mut UniqueRefGpuVm>, + req: VmOpRequest<'drm>, + resources: &'a mut VmOpResources<'drm>, + ) -> Result { + let pt = &self.as_data.page_table; + + match req.op_type { + VmOpType::Map(args) => { + let mut pt_upd = PtUpdateContext::new( + self.dev, + pt, + &self.mmu, + &self.as_data, + req.region, + PtOpType::Map(PtMapArgs { + prot: args.flags.to_prot(), + }), + resources, + )?; + + gpuvm_unique.sm_map(OpMapRequest { + addr: pt_upd.region.start, + range: pt_upd.region.end - pt_upd.region.start, + gem_offset: args.bo_offset, + vm_bo: &args.vm_bo, + context: &mut pt_upd, + }) + //PtUpdateContext drops here flushing the page table + } + VmOpType::Unmap => { + let mut pt_upd = PtUpdateContext::new( + self.dev, + pt, + &self.mmu, + &self.as_data, + req.region, + PtOpType::Unmap, + resources, + )?; + + gpuvm_unique.sm_unmap( + pt_upd.region.start, + pt_upd.region.end - pt_upd.region.start, + &mut pt_upd, + ) + //PtUpdateContext drops here flushing the page table + } + } + } + + /// Maps a GEM buffer object range into the VM at the specified virtual address. + /// + /// This creates a mapping from GPU virtual address `va` to the physical pages + /// backing the GEM object, starting at `bo_offset` bytes into the object and + /// spanning `map_size` bytes. The mapping respects the access permissions and + /// caching behavior specified in `flags`. + pub(crate) fn map_bo_range( + &self, + bo: &Bo, + bo_offset: u64, + map_size: u64, + va: u64, + flags: VmMapFlags, + ) -> Result { + if map_size == 0 + || va % SZ_4K as u64 != 0 + || bo_offset % SZ_4K as u64 != 0 + || map_size % SZ_4K as u64 != 0 + { + return Err(EINVAL); + } + + let bo_size = u64::try_from(bo.size()).map_err(|_| EOVERFLOW)?; + let bo_end = bo_offset.checked_add(map_size).ok_or(EINVAL)?; + + if bo_end > bo_size { + dev_err!( + self.dev, + "BO mapping range {:#x}..{:#x} exceeds BO size {:#x}", + bo_offset, + bo_end, + bo_size + ); + return Err(EINVAL); + } + + let va_end: u64 = va.checked_add(map_size).ok_or(EINVAL)?; + + let req = VmOpRequest { + op_type: VmOpType::Map(VmMapArgs { + vm_bo: self.gpuvm.obtain(bo, ())?, + flags, + bo_offset, + }), + region: va..va_end, + }; + let mut resources = VmOpResources { + preallocated_gpuvas: [ + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + ], + }; + let result = { + let mut gpuvm_unique = self.gpuvm_unique.lock(); + self.exec_op(gpuvm_unique.as_mut().get_mut(), req, &mut resources) + }; + // We flush the defer cleanup list now. Things will be different in + // the asynchronous VM_BIND path, where we want the cleanup to + // happen outside the DMA signalling path. + self.gpuvm.deferred_cleanup(); + result + } + + /// Unmaps a virtual address range from the VM. + /// + /// This removes any existing mappings in the specified range, freeing the + /// virtual address space for reuse. + pub(crate) fn unmap_range(&self, va: u64, size: u64) -> Result { + if size == 0 || va % SZ_4K as u64 != 0 || size % SZ_4K as u64 != 0 { + return Err(EINVAL); + } + + let end = va.checked_add(size).ok_or(EINVAL)?; + + if va < self.va_range.start || end > self.va_range.end { + dev_err!( + self.dev, + "Unmap range {:#x}..{:#x} exceeds VM range {:#x}..{:#x}", + va, + end, + self.va_range.start, + self.va_range.end + ); + return Err(EINVAL); + } + + let req = VmOpRequest { + op_type: VmOpType::Unmap, + region: va..end, + }; + + let full_vm = va == self.va_range.start && end == self.va_range.end; + + let mut resources = VmOpResources { + preallocated_gpuvas: if full_vm { + // Unmapping the entire VM cannot split an existing mapping, + // so no GPUVA objects are needed for remap operations. + [None, None, None] + } else { + [ + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + ] + }, + }; + let result = { + let mut gpuvm_unique = self.gpuvm_unique.lock(); + self.exec_op(gpuvm_unique.as_mut().get_mut(), req, &mut resources) + }; + // We flush the defer cleanup list now. Things will be different in + // the asynchronous VM_BIND path, where we want the cleanup to + // happen outside the DMA signalling path. + self.gpuvm.deferred_cleanup(); + result + } +} + +impl<'drm> DriverGpuVm for GpuVmData<'drm> { + type Driver = TyrDrmDriver; + type Object = Bo; + type VmBoData = (); + type VaData = (); + type SmContext<'ctx> + = PtUpdateContext<'ctx, 'drm> + where + Self: 'ctx; + + /// Create a new mapping. + fn sm_step_map<'op>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'_>, + ) -> Result, Error> { + let start_iova = op.addr(); + let mut iova = start_iova; + let mut bytes_left_to_map = op.length(); + let mut gem_offset = op.gem_offset(); + + // Make sure that the end of the requested GEM range doesn't run past the + // end of the GEM buffer itself. + let gem_range_end = op.gem_offset().checked_add(op.length()).ok_or(EINVAL)?; + + if gem_range_end > op.obj().size() as u64 { + dev_err!( + context.dev, + "Requested GEM range ends at {} which is beyond the GEM buffer size {}", + gem_range_end, + op.obj().size() + ); + return Err(EINVAL); + } + + let sgt = op.obj().sg_table(context.dev).inspect_err(|e| { + dev_err!(context.dev, "Failed to get sg_table: {:?}", e); + })?; + let prot = match &context.op_type { + PtOpType::Map(args) => args.prot, + _ => { + return Err(EINVAL); + } + }; + + for sgt_entry in sgt.iter() { + // Expressly convert to u64 to work with arm 32-bit builds. + #[allow(clippy::useless_conversion)] + let mut paddr = u64::from(sgt_entry.dma_address()); + #[allow(clippy::useless_conversion)] + let mut sgt_entry_length = u64::from(sgt_entry.dma_len()); + + if bytes_left_to_map == 0 { + break; + } + + if gem_offset > 0 { + // Skip the entire SGT entry if the gem_offset exceeds its length. + let skip = u64::min(sgt_entry_length, gem_offset); + paddr += skip; + sgt_entry_length -= skip; + gem_offset -= skip; + } + + if sgt_entry_length == 0 { + continue; + } + + let len = u64::min(sgt_entry_length, bytes_left_to_map); + + let segment_mapped = match pt_map(context.dev, context.pt, iova, paddr, len, prot) { + Ok(segment_mapped) => segment_mapped, + Err(e) => { + // clean up any successful mappings from previous SGT entries. + let total_mapped = iova - start_iova; + if total_mapped > 0 { + let _ = pt_unmap( + context.dev, + context.pt, + start_iova..(start_iova + total_mapped), + ); + } + return Err(e); + } + }; + + bytes_left_to_map -= segment_mapped; + iova += segment_mapped; + } + + if bytes_left_to_map != 0 { + let total_mapped = iova - start_iova; + + if total_mapped > 0 { + let _ = pt_unmap(context.dev, context.pt, start_iova..iova); + } + + dev_err!( + context.dev, + "SG table is too small for requested mapping: {} bytes remain", + bytes_left_to_map + ); + + return Err(EINVAL); + } + + let gpuva = context.preallocated_gpuva()?; + let op = op.insert(gpuva, pin_init::init_zeroed()); + + Ok(op) + } + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'_>, + ) -> Result, Error> { + let start_iova = op.va().addr(); + let length = op.va().length(); + + let region = start_iova..(start_iova + length); + pt_unmap(context.dev, context.pt, region.clone()).inspect_err(|e| { + dev_err!( + context.dev, + "Failed to unmap region {:#x}..{:#x}: {:?}", + region.start, + region.end, + e + ); + })?; + + let (op_unmapped, _va_removed) = op.remove(); + + Ok(op_unmapped) + } + + /// Split up an existing mapping. + fn sm_step_remap<'op>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'_>, + ) -> Result, Error> { + let unmap_start = if let Some(prev) = op.prev() { + prev.addr() + prev.length() + } else { + op.va_to_unmap().addr() + }; + + let unmap_end = if let Some(next) = op.next() { + next.addr() + } else { + op.va_to_unmap().addr() + op.va_to_unmap().length() + }; + + let unmap_length = unmap_end - unmap_start; + + if unmap_length > 0 { + let region = unmap_start..(unmap_start + unmap_length); + pt_unmap(context.dev, context.pt, region.clone()).inspect_err(|e| { + dev_err!( + context.dev, + "Failed to unmap remap region {:#x}..{:#x}: {:?}", + region.start, + region.end, + e + ); + })?; + } + + let prev_va = context.preallocated_gpuva()?; + let next_va = context.preallocated_gpuva()?; + + let (op_remapped, remap_ret) = op.remap( + [prev_va, next_va], + pin_init::init_zeroed(), + pin_init::init_zeroed(), + ); + + if let Some(unused_va) = remap_ret.unused_va { + context.return_preallocated_gpuva(unused_va); + } + + Ok(op_remapped) + } +} + +/// This function selects the largest supported block size (currently 4KB or 2MB) +/// that can be used for a mapping at the given address and size, respecting alignment constraints. +/// +/// We can map multiple pages at once but we can't exceed the size of the +/// table entry itself. So, if mapping 4KB pages, figure out how many pages +/// can be mapped before we hit the 2MB boundary. Or, if mapping 2MB pages, +/// figure out how many pages can be mapped before hitting the 1GB boundary +/// Returns the page size (4KB or 2MB) and the number of pages that can be mapped at that size. +fn get_pgsize(addr: u64, size: u64) -> (u64, u64) { + // Get the distance to the next boundary of 2MB block + let blk_offset_2m = addr.wrapping_neg() % (SZ_2M as u64); + + // Use 4K blocks if the address is not 2MB aligned, or we have less than 2MB to map + if blk_offset_2m != 0 || size < SZ_2M as u64 { + let pgcount = if blk_offset_2m == 0 { + size / SZ_4K as u64 + } else { + u64::min(blk_offset_2m, size) / SZ_4K as u64 + }; + return (SZ_4K as u64, pgcount); + } + + let blk_offset_1g = addr.wrapping_neg() % (SZ_1G as u64); + let blk_offset = if blk_offset_1g == 0 { + SZ_1G as u64 + } else { + blk_offset_1g + }; + let pgcount = u64::min(blk_offset, size) / SZ_2M as u64; + + (SZ_2M as u64, pgcount) +} + +/// Maps a physical address range into the page table at the specified virtual address. +/// +/// This function maps `len` bytes of physical memory starting at `paddr` to the +/// virtual address `iova`, using the protection flags specified in `prot`. It +/// automatically selects optimal page sizes to minimize page table overhead. +/// +/// If the mapping fails partway through, all successfully mapped pages are +/// unmapped before returning an error. +/// +/// Returns the number of bytes successfully mapped. +fn pt_map( + dev: &Device, + pt: &IoPageTable<'_, ARM64LPAES1>, + iova: u64, + paddr: u64, + len: u64, + prot: u32, +) -> Result { + let mut segment_mapped = 0u64; + while segment_mapped < len { + let remaining = len - segment_mapped; + let curr_iova = iova + segment_mapped; + let curr_paddr = paddr + segment_mapped; + + let (pgsize, pgcount) = get_pgsize(curr_iova | curr_paddr, remaining); + + // On 32-bit systems, usize is only 32 bits, so check that + // the iova can be converted without truncation. + let curr_iova = match usize::try_from(curr_iova) { + Ok(curr_iova) => curr_iova, + Err(_) => { + dev_err!( + dev, + "curr_iova {:#x} cannot be represented as usize (max {:#x})", + curr_iova, + usize::MAX + ); + + if segment_mapped > 0 { + let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped)); + } + + return Err(EOVERFLOW); + } + }; + + // SAFETY: + // No other io-pgtable operation can currently access this range because Tyr holds + // the gpuvm_unique mutex for the entire sm_map() operation. + // The addresses being mapped won't overlap any existing mappings in this + // page table because drm_gpuvm_sm_map() checks each requested mapping and either unmaps + // or remaps any overlap before creating the new mapping. + let (mapped, result) = unsafe { + pt.map_pages( + curr_iova, + curr_paddr as PhysAddr, + pgsize as usize, + pgcount as usize, + prot, + GFP_KERNEL, + ) + }; + + if let Err(e) = result { + // If map_pages fails, mapped will be zero because the ARM LPAE backend + // only updates the mapped value after the entire request succeeds. + dev_err!(dev, "pt.map_pages failed at iova {:#x}: {:?}", curr_iova, e); + if segment_mapped > 0 { + let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped)); + } + return Err(e); + } + + if mapped == 0 { + dev_err!(dev, "Failed to map any pages at iova {:#x}", curr_iova); + if segment_mapped > 0 { + let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped)); + } + return Err(ENOMEM); + } + + segment_mapped += mapped as u64; + } + + Ok(segment_mapped) +} + +/// Unmaps a virtual address range from the page table. +/// +/// This function removes all page table entries in the specified range, +/// automatically handling different page sizes that may be present. +fn pt_unmap(dev: &Device, pt: &IoPageTable<'_, ARM64LPAES1>, range: Range) -> Result { + let mut iova = range.start; + let mut bytes_left_to_unmap = range.end - range.start; + + while bytes_left_to_unmap > 0 { + // It is fine to use just the iova to determine the page size + // because if the actual mapping was represented with smaller page sizes, + // (e.g. because the physical address was not 2MiB aligned) + // the ARM LPAE backend will notice and handle the lower-level table correctly. + let (pgsize, pgcount) = get_pgsize(iova, bytes_left_to_unmap); + + // On 32-bit systems, usize is only 32 bits, so check that + // the iova can be converted without truncation. + let iova_usize = usize::try_from(iova).map_err(|_| { + dev_err!( + dev, + "IOVA {:#x} cannot be represented as usize (max {:#x})", + iova, + usize::MAX + ); + EOVERFLOW + })?; + + // SAFETY: + // No other io-pgtable operation can currently access this range because Tyr holds + // the gpuvm_unique mutex for the entire sm_unmap() operation. + // We know that this page table has one or more consecutive mappings + // starting at `iova` with the total size of `pgcount * pgsize` because + // gpuvm callbacks provide exactly the range that was previously mapped. + let unmapped = unsafe { pt.unmap_pages(iova_usize, pgsize as usize, pgcount as usize) }; + + if unmapped == 0 { + dev_err!(dev, "Failed to unmap any bytes at iova {:#x}", iova_usize); + return Err(EINVAL); + } + + bytes_left_to_unmap -= unmapped as u64; + iova += unmapped as u64; + } + + Ok(()) +} -- cgit v1.3.1