summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/tyr
diff options
context:
space:
mode:
authorUlises Mendez Martinez <umendez@google.com>2026-09-04 15:07:09 +0000
committerJosh Poimboeuf <jpoimboe@kernel.org>2026-09-13 18:15:43 -0700
commit7e61560628d17ea6b1d8ee370f6d42694cff8758 (patch)
treee6080164b2f4c4c419894bdb3ce4d956ca9a98c5 /drivers/gpu/drm/tyr
downloadlinux-stable-7e61560628d17ea6b1d8ee370f6d42694cff8758.tar.gz
linux-stable-7e61560628d17ea6b1d8ee370f6d42694cff8758.zip
objtool: Validate disassembler headers in libopcodes probegrafted
commit 3f2de814c059 ("objtool: Fix libopcodes linking with static libraries") tested for libopcodes availability by linking a test snippet with a forward declaration of disassemble_init_for_target(). However, testing symbol linkage with an extern declaration only verifies the presence of the library (.so/.a) and bypasses checking for development headers (binutils-dev). On systems where libopcodes is present without development headers installed, the probe succeeds, enabling BUILD_DISAS. Subsequent compilation of objtool then fails: fatal error: 'bfd.h' file not found 113 | #include <bfd.h> Additionally, the probe invokes $(HOSTCC) without $(HOSTCFLAGS), ignoring any sysroot or include flags specified for the host compiler. Fix this by including <bfd.h> and <dis-asm.h> directly in the test snippet, passing $(HOSTCFLAGS) so host compiler options are respected, and defining PACKAGE="objtool" to satisfy the configuration check in <bfd.h>. Fixes: 3f2de814c059 ("objtool: Fix libopcodes linking with static libraries") Fixes: 436326bc525d ("objtool: fix build failure due to missing libopcodes check") Reported-by: Alice Ryhl <aliceryhl@google.com> Assisted-by: Antigravity:Gemini-Next Signed-off-by: Ulises Mendez Martinez <umendez@google.com> Link: https://patch.msgid.link/20260904150710.2997558-1-umendez@google.com Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
Diffstat (limited to 'drivers/gpu/drm/tyr')
-rw-r--r--drivers/gpu/drm/tyr/Kconfig26
-rw-r--r--drivers/gpu/drm/tyr/Makefile3
-rw-r--r--drivers/gpu/drm/tyr/driver.rs237
-rw-r--r--drivers/gpu/drm/tyr/file.rs65
-rw-r--r--drivers/gpu/drm/tyr/fw.rs321
-rw-r--r--drivers/gpu/drm/tyr/fw/parser.rs588
-rw-r--r--drivers/gpu/drm/tyr/gem.rs160
-rw-r--r--drivers/gpu/drm/tyr/gpu.rs175
-rw-r--r--drivers/gpu/drm/tyr/mmu.rs120
-rw-r--r--drivers/gpu/drm/tyr/mmu/address_space.rs511
-rw-r--r--drivers/gpu/drm/tyr/regs.rs1779
-rw-r--r--drivers/gpu/drm/tyr/slot.rs404
-rw-r--r--drivers/gpu/drm/tyr/tyr.rs26
-rw-r--r--drivers/gpu/drm/tyr/vm.rs950
14 files changed, 5365 insertions, 0 deletions
diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig
new file mode 100644
index 000000000..8f13e49f1
--- /dev/null
+++ b/drivers/gpu/drm/tyr/Kconfig
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: GPL-2.0 or MIT
+
+config DRM_TYR
+ tristate "Tyr (Rust DRM support for ARM Mali CSF-based GPUs)"
+ depends on DRM=y
+ depends on RUST
+ depends on ARM || ARM64 || COMPILE_TEST
+ depends on MMU
+ depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE
+ depends on COMMON_CLK
+ depends on IOMMU_SUPPORT
+ default n
+ select IOMMU_IO_PGTABLE_LPAE
+ select RUST_DRM_GEM_SHMEM_HELPER
+ select RUST_DRM_GPUVM
+ select RUST_FW_LOADER_ABSTRACTIONS
+ help
+ Rust DRM driver for ARM Mali CSF-based GPUs.
+
+ This driver is for Mali (or Immortalis) Valhall Gxxx GPUs.
+
+ Note that the Mali-G68 and Mali-G78, while Valhall architecture, will
+ be supported with the panfrost driver as they are not CSF GPUs.
+
+ if M is selected, the module will be called tyr. This driver is work
+ in progress and may not be functional.
diff --git a/drivers/gpu/drm/tyr/Makefile b/drivers/gpu/drm/tyr/Makefile
new file mode 100644
index 000000000..ba545f65f
--- /dev/null
+++ b/drivers/gpu/drm/tyr/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0 or MIT
+
+obj-$(CONFIG_DRM_TYR) += tyr.o
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs
new file mode 100644
index 000000000..bfb0ba19c
--- /dev/null
+++ b/drivers/gpu/drm/tyr/driver.rs
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+use kernel::{
+ clk::{
+ Clk,
+ OptionalClk, //
+ },
+ device::{
+ Bound,
+ Core,
+ Device,
+ DeviceContext, //
+ },
+ dma::{
+ Device as DmaDevice,
+ DmaMask, //
+ },
+ drm,
+ drm::ioctl,
+ io::{
+ poll,
+ Io, //
+ },
+ new_mutex,
+ of,
+ platform,
+ prelude::*,
+ regulator,
+ regulator::Regulator,
+ sizes::SZ_2M,
+ sync::{
+ Arc,
+ Mutex, //
+ },
+ time, //
+};
+
+use crate::{
+ file::TyrDrmFileData,
+ fw::Firmware,
+ gem::Bo,
+ gpu,
+ gpu::GpuInfo,
+ mmu::Mmu,
+ regs::gpu_control::*, //
+};
+
+pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>;
+
+pub(crate) struct TyrDrmDriver;
+
+/// Convenience type alias for the DRM device type for this driver.
+pub(crate) type TyrDrmDevice<Ctx = drm::Normal> = drm::Device<TyrDrmDriver, Ctx>;
+
+pub(crate) struct TyrPlatformDriver;
+
+#[pin_data(PinnedDrop)]
+pub(crate) struct TyrPlatformDriverData<'bound> {
+ _reg: drm::Registration<'bound, TyrDrmDriver>,
+}
+
+/// Data owned by the DRM [`Registration`].
+///
+/// This data can have references tied to the parent platform device binding scope
+/// and is accessible only while the DRM device is registered with userspace.
+#[pin_data]
+pub(crate) struct TyrDrmRegistrationData<'drm> {
+ /// Parent platform device.
+ pub(crate) pdev: &'drm platform::Device<Bound>,
+
+ /// Firmware sections.
+ pub(crate) fw: Firmware<'drm>,
+
+ #[pin]
+ clks: Mutex<Clocks>,
+
+ #[pin]
+ regulators: Mutex<Regulators>,
+
+ /// GPU MMIO register mapping.
+ pub(crate) iomem: Arc<IoMem<'drm>>,
+
+ /// GPU information read from hardware during probe.
+ pub(crate) gpu_info: GpuInfo,
+}
+
+fn issue_soft_reset(dev: &Device, iomem: &IoMem<'_>) -> Result {
+ iomem.write_reg(GPU_COMMAND::reset(ResetMode::SoftReset));
+
+ poll::read_poll_timeout(
+ || Ok(iomem.read(GPU_IRQ_RAWSTAT)),
+ |status| status.reset_completed(),
+ time::Delta::from_millis(1),
+ time::Delta::from_millis(100),
+ )
+ .inspect_err(|_| dev_err!(dev, "GPU reset failed."))?;
+
+ Ok(())
+}
+
+kernel::of_device_table!(
+ OF_TABLE,
+ <TyrPlatformDriver as platform::Driver>::IdInfo,
+ [
+ (of::DeviceId::new(c"rockchip,rk3588-mali"), ()),
+ (of::DeviceId::new(c"arm,mali-valhall-csf"), ())
+ ]
+);
+
+impl platform::Driver for TyrPlatformDriver {
+ type IdInfo = ();
+ type Data<'bound> = TyrPlatformDriverData<'bound>;
+ const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE);
+
+ fn probe<'bound>(
+ pdev: &'bound platform::Device<Core<'_>>,
+ _info: Option<&'bound Self::IdInfo>,
+ ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
+ let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?;
+ let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?;
+ let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?;
+
+ core_clk.prepare_enable()?;
+ stacks_clk.prepare_enable()?;
+ coregroup_clk.prepare_enable()?;
+
+ let mali_regulator = Regulator::<regulator::Enabled>::get(pdev.as_ref(), c"mali")?;
+ let sram_regulator = Regulator::<regulator::Enabled>::get(pdev.as_ref(), c"sram")?;
+
+ let request = pdev.io_request_by_index(0).ok_or(ENODEV)?;
+
+ let iomem = Arc::new(request.iomap_sized::<SZ_2M>()?, GFP_KERNEL)?;
+
+ issue_soft_reset(pdev.as_ref(), &iomem)?;
+ gpu::l2_power_on(pdev.as_ref(), &iomem)?;
+
+ let gpu_info = GpuInfo::new(&iomem);
+ gpu_info.log(pdev.as_ref());
+
+ let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features)
+ .pa_bits()
+ .get();
+ // SAFETY: No concurrent DMA allocations or mappings can be made because
+ // the device is still being probed and therefore isn't being used by
+ // other threads of execution.
+ unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? };
+
+ let unreg_dev = drm::UnregisteredDevice::<TyrDrmDriver>::new(pdev, Ok(()))?;
+
+ let mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?;
+
+ let firmware = Firmware::new(
+ pdev.as_ref(),
+ iomem.clone(),
+ &unreg_dev,
+ mmu.as_arc_borrow(),
+ &gpu_info,
+ )?;
+
+ firmware.boot()?;
+
+ let reg_data = pin_init!(TyrDrmRegistrationData {
+ pdev,
+ fw: firmware,
+ clks <- new_mutex!(Clocks {
+ core: core_clk,
+ stacks: stacks_clk,
+ coregroup: coregroup_clk,
+ }),
+ regulators <- new_mutex!(Regulators {
+ _mali: mali_regulator,
+ _sram: sram_regulator,
+ }),
+ iomem,
+ gpu_info,
+ });
+
+ // SAFETY: `reg` is stored in `TyrPlatformDriverData` and dropped when the driver is
+ // unbound; it is never forgotten.
+ let reg = unsafe { drm::Registration::new(pdev.as_ref(), unreg_dev, reg_data, 0)? };
+
+ let driver = TyrPlatformDriverData { _reg: reg };
+
+ dev_dbg!(pdev, "Tyr initialized correctly.");
+ Ok(driver)
+ }
+}
+
+#[pinned_drop]
+impl PinnedDrop for TyrPlatformDriverData<'_> {
+ fn drop(self: Pin<&mut Self>) {}
+}
+
+// We need to retain the name "panthor" to achieve drop-in compatibility with
+// the C driver in the userspace stack.
+const INFO: drm::DriverInfo = drm::DriverInfo {
+ major: 1,
+ minor: 5,
+ patchlevel: 0,
+ name: c"panthor",
+ desc: c"ARM Mali Tyr DRM driver",
+};
+
+#[vtable]
+impl drm::Driver for TyrDrmDriver {
+ type Data = ();
+ type RegistrationData<'drm> = TyrDrmRegistrationData<'drm>;
+ type File = TyrDrmFileData;
+ type Object = Bo;
+ type ParentDevice<Ctx: DeviceContext> = platform::Device<Ctx>;
+
+ const INFO: drm::DriverInfo = INFO;
+ const FEAT_RENDER: bool = true;
+
+ kernel::declare_drm_ioctls! {
+ (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, TyrDrmFileData::dev_query),
+ }
+}
+
+struct Clocks {
+ core: Clk,
+ stacks: OptionalClk,
+ coregroup: OptionalClk,
+}
+
+impl Drop for Clocks {
+ fn drop(&mut self) {
+ self.core.disable_unprepare();
+ self.stacks.disable_unprepare();
+ self.coregroup.disable_unprepare();
+ }
+}
+
+struct Regulators {
+ _mali: Regulator<regulator::Enabled>,
+ _sram: Regulator<regulator::Enabled>,
+}
diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs
new file mode 100644
index 000000000..9f60a90d4
--- /dev/null
+++ b/drivers/gpu/drm/tyr/file.rs
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+use kernel::{
+ drm::{
+ self,
+ Registered, //
+ },
+ prelude::*,
+ uaccess::UserSlice,
+ uapi, //
+};
+
+use crate::driver::{
+ TyrDrmDevice,
+ TyrDrmDriver,
+ TyrDrmRegistrationData, //
+};
+
+#[pin_data]
+pub(crate) struct TyrDrmFileData {}
+
+/// Convenience type alias for our DRM `File` type
+pub(crate) type TyrDrmFile = drm::file::File<TyrDrmFileData>;
+
+impl drm::file::DriverFile for TyrDrmFileData {
+ type Driver = TyrDrmDriver;
+
+ fn open(_dev: &drm::Device<Self::Driver>) -> Result<Pin<KBox<Self>>> {
+ KBox::try_pin_init(try_pin_init!(Self {}), GFP_KERNEL)
+ }
+}
+
+impl TyrDrmFileData {
+ pub(crate) fn dev_query(
+ _ddev: &TyrDrmDevice<Registered>,
+ reg_data: &TyrDrmRegistrationData<'_>,
+ devquery: &mut uapi::drm_panthor_dev_query,
+ _file: &TyrDrmFile,
+ ) -> Result<u32> {
+ if devquery.pointer == 0 {
+ match devquery.type_ {
+ uapi::drm_panthor_dev_query_type_DRM_PANTHOR_DEV_QUERY_GPU_INFO => {
+ devquery.size = core::mem::size_of_val(&reg_data.gpu_info) as u32;
+ Ok(0)
+ }
+ _ => Err(EINVAL),
+ }
+ } else {
+ match devquery.type_ {
+ uapi::drm_panthor_dev_query_type_DRM_PANTHOR_DEV_QUERY_GPU_INFO => {
+ let mut writer = UserSlice::new(
+ UserPtr::from_addr(devquery.pointer as usize),
+ devquery.size as usize,
+ )
+ .writer();
+
+ writer.write(&reg_data.gpu_info)?;
+
+ Ok(0)
+ }
+ _ => Err(EINVAL),
+ }
+ }
+ }
+}
diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs
new file mode 100644
index 000000000..47d25c901
--- /dev/null
+++ b/drivers/gpu/drm/tyr/fw.rs
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! Firmware loading and management for Mali CSF GPU.
+//!
+//! This module handles loading the Mali GPU firmware binary, parsing it into sections,
+//! and mapping those sections into the MCU's virtual address space. Each firmware section
+//! has specific properties (read/write/execute permissions, cache modes) and must be loaded
+//! at specific virtual addresses expected by the MCU.
+//!
+//! See [`Firmware`] for the main firmware management interface and [`Section`] for
+//! individual firmware sections.
+//!
+//! [`Firmware`]: crate::fw::Firmware
+//! [`Section`]: crate::fw::Section
+
+use kernel::{
+ device::{
+ Bound,
+ Device, //
+ },
+ drm::{
+ gem::BaseObject, //
+ },
+ io::{
+ poll,
+ Io, //
+ },
+ num::Bounded,
+ prelude::*,
+ register,
+ str::CString,
+ sync::{
+ Arc,
+ ArcBorrow, //
+ },
+ time, //
+};
+
+use crate::{
+ driver::{
+ IoMem,
+ TyrDrmDevice, //
+ },
+ fw::parser::{
+ FwParser,
+ ParsedSection, //
+ },
+ gem,
+ gem::{
+ KernelBo,
+ KernelBoVaAlloc, //
+ },
+ gpu::GpuInfo,
+
+ mmu::Mmu,
+ regs::{
+ gpu_control::{
+ McuControlMode,
+ McuStatus,
+ GPU_ID,
+ MCU_CONTROL,
+ MCU_STATUS, //
+ }, //
+ job_control::{
+ JOB_IRQ_CLEAR,
+ JOB_IRQ_RAWSTAT, //
+ }, //
+ },
+ vm::Vm, //
+};
+
+mod parser;
+
+pub(super) const CSF_MCU_SHARED_REGION_START: u32 = 0x04000000;
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[repr(u8)]
+pub(super) enum CacheMode {
+ None = 0,
+ Cached = 1,
+ UncachedCoherent = 2,
+ CachedCoherent = 3,
+}
+
+impl From<Bounded<u32, 2>> for CacheMode {
+ fn from(value: Bounded<u32, 2>) -> Self {
+ match value.get() {
+ 0 => Self::None,
+ 1 => Self::Cached,
+ 2 => Self::UncachedCoherent,
+ 3 => Self::CachedCoherent,
+ _ => unreachable!(),
+ }
+ }
+}
+
+impl From<CacheMode> for Bounded<u32, 2> {
+ fn from(value: CacheMode) -> Self {
+ Bounded::try_new(value as u32).unwrap()
+ }
+}
+
+register! {
+ #[allow(non_upper_case_globals)]
+ pub(super) SectionFlags(u32) @ 0x0 {
+ 0:0 read => bool;
+ 1:1 write => bool;
+ 2:2 exec => bool;
+ 4:3 cache_mode => CacheMode;
+ 5:5 prot => bool;
+ 30:30 shared => bool;
+ 31:31 zero => bool;
+ }
+}
+
+impl SectionFlags {
+ const VALID_MASK: u32 = Self::READ_MASK
+ | Self::WRITE_MASK
+ | Self::EXEC_MASK
+ | Self::CACHE_MODE_MASK
+ | Self::PROT_MASK
+ | Self::SHARED_MASK
+ | Self::ZERO_MASK;
+
+ fn try_from_fw(value: u32) -> Result<Self> {
+ if value & !Self::VALID_MASK != 0 {
+ Err(EINVAL)
+ } else {
+ Ok(Self::from_raw(value))
+ }
+ }
+}
+
+/// A parsed section of the firmware binary.
+struct Section<'drm> {
+ // Raw firmware section data for reset purposes
+ #[expect(dead_code)]
+ data: KVec<u8>,
+
+ // Keep the BO backing this firmware section so that both the
+ // GPU mapping and CPU mapping remain valid until the Section is dropped.
+ #[expect(dead_code)]
+ mem: gem::KernelBo<'drm>,
+}
+
+/// Loaded firmware with sections mapped into MCU VM.
+pub(crate) struct Firmware<'drm> {
+ /// Iomem need to access registers.
+ iomem: Arc<IoMem<'drm>>,
+
+ /// MCU VM.
+ vm: Arc<Vm<'drm>>,
+
+ /// List of firmware sections.
+ #[expect(dead_code)]
+ sections: KVec<Section<'drm>>,
+}
+
+impl<'drm> Drop for Firmware<'drm> {
+ fn drop(&mut self) {
+ // Stop the MCU before releasing its firmware mappings and memory.
+ let _ = self.stop();
+
+ // AS slots retain a VM ref, we need to kill the circular ref manually.
+ self.vm.kill();
+ }
+}
+
+impl<'drm> Firmware<'drm> {
+ fn init_section_mem(dev: &Device, mem: &mut KernelBo<'drm>, data: &KVec<u8>) -> Result {
+ if data.is_empty() {
+ return Ok(());
+ }
+
+ let vmap = mem.bo().vmap::<0>()?;
+ let size = mem.bo().size();
+
+ if data.len() > size {
+ dev_err!(dev, "fw section {} bigger than BO {}", data.len(), size);
+ return Err(EINVAL);
+ }
+
+ for (i, &byte) in data.iter().enumerate() {
+ vmap.try_write8(byte, i)?;
+ }
+
+ Ok(())
+ }
+
+ fn request(ddev: &TyrDrmDevice, gpu_info: &GpuInfo) -> Result<kernel::firmware::Firmware> {
+ let gpu_id = GPU_ID::from_raw(gpu_info.gpu_id);
+
+ let path = CString::try_from_fmt(fmt!(
+ "arm/mali/arch{}.{}/mali_csffw.bin",
+ gpu_id.arch_major().get(),
+ gpu_id.arch_minor().get()
+ ))?;
+
+ kernel::firmware::Firmware::request(&path, ddev.as_ref().as_ref())
+ }
+
+ fn load(
+ dev: &Device,
+ ddev: &TyrDrmDevice,
+ gpu_info: &GpuInfo,
+ ) -> Result<(kernel::firmware::Firmware, KVec<ParsedSection>)> {
+ let fw = Self::request(ddev, gpu_info)?;
+ let mut parser = FwParser::new(dev, fw.data());
+
+ let parsed_sections = parser.parse()?;
+
+ Ok((fw, parsed_sections))
+ }
+
+ /// Load firmware and map sections into MCU VM.
+ pub(crate) fn new(
+ dev: &'drm Device<Bound>,
+ iomem: Arc<IoMem<'drm>>,
+ ddev: &TyrDrmDevice,
+ mmu: ArcBorrow<'_, Mmu<'drm>>,
+ gpu_info: &GpuInfo,
+ ) -> Result<Firmware<'drm>> {
+ let vm = Vm::new(dev, ddev, mmu, gpu_info)?;
+ vm.activate()?;
+
+ let result = (|| {
+ let (fw, parsed_sections) = Self::load(dev, ddev, gpu_info)?;
+ let mut sections = KVec::new();
+ for parsed in parsed_sections {
+ let size = u64::from(parsed.va.end.checked_sub(parsed.va.start).ok_or(EINVAL)?);
+
+ let va = u64::from(parsed.va.start);
+
+ let mut mem = KernelBo::new(
+ ddev,
+ vm.clone(),
+ size,
+ KernelBoVaAlloc::Explicit(va),
+ parsed.vm_map_flags,
+ )?;
+
+ let section_start = parsed.data_range.start as usize;
+ let section_end = parsed.data_range.end as usize;
+ let mut data = KVec::new();
+
+ // Ensure that the firmware slice is not out of bounds.
+ let fw_data = fw.data();
+ let bytes = fw_data.get(section_start..section_end).ok_or(EINVAL)?;
+ data.extend_from_slice(bytes, GFP_KERNEL)?;
+
+ Self::init_section_mem(dev, &mut mem, &data)?;
+
+ sections.push(Section { data, mem }, GFP_KERNEL)?;
+ }
+
+ Ok(Firmware {
+ iomem,
+ vm: vm.clone(),
+ sections,
+ })
+ })();
+
+ if result.is_err() {
+ vm.kill();
+ }
+
+ result
+ }
+
+ pub(crate) fn boot(&self) -> Result {
+ let io = &self.iomem;
+
+ // Discard any stale global interrupt.
+ io.write_reg(JOB_IRQ_CLEAR::zeroed().with_glb(true));
+
+ io.write_reg(MCU_CONTROL::zeroed().with_req(McuControlMode::Auto));
+
+ if let Err(e) = poll::read_poll_timeout(
+ || Ok((io.read(MCU_STATUS), io.read(JOB_IRQ_RAWSTAT))),
+ |(mcu_status, irq_rawstat)| {
+ mcu_status.value() == McuStatus::Enabled && irq_rawstat.glb()
+ },
+ time::Delta::from_millis(1),
+ time::Delta::from_millis(100),
+ ) {
+ let status = io.read(MCU_STATUS);
+ dev_err!(
+ self.vm.dev(),
+ "MCU failed to boot, status: {:?}",
+ status.value()
+ );
+ return Err(e);
+ }
+
+ io.write_reg(JOB_IRQ_CLEAR::zeroed().with_glb(true));
+
+ Ok(())
+ }
+
+ fn stop(&self) -> Result {
+ let io = &self.iomem;
+ io.write_reg(MCU_CONTROL::zeroed().with_req(McuControlMode::Disable));
+
+ if let Err(e) = poll::read_poll_timeout(
+ || Ok(io.read(MCU_STATUS)),
+ |status| status.value() == McuStatus::Disabled,
+ time::Delta::from_micros(10),
+ time::Delta::from_millis(100),
+ ) {
+ let status = io.read(MCU_STATUS);
+ dev_err!(
+ self.vm.dev(),
+ "MCU failed to stop, status: {:?}",
+ status.value()
+ );
+ return Err(e);
+ }
+
+ Ok(())
+ }
+}
diff --git a/drivers/gpu/drm/tyr/fw/parser.rs b/drivers/gpu/drm/tyr/fw/parser.rs
new file mode 100644
index 000000000..c4d0ad1d7
--- /dev/null
+++ b/drivers/gpu/drm/tyr/fw/parser.rs
@@ -0,0 +1,588 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! Firmware binary parser for Mali CSF (Command Stream Frontend) GPU.
+//!
+//! This module implements a parser for the Mali GPU firmware binary format. The firmware
+//! file contains a header followed by a sequence of entries, each describing how to load
+//! firmware sections into the MCU (Microcontroller Unit) memory. The parser extracts section
+//! metadata including:
+//! - Virtual address ranges where sections should be mapped
+//! - Data ranges (byte offsets) within the firmware binary
+//! - Section flags (permissions, cache modes)
+
+use core::{
+ mem::size_of,
+ ops::Range, //
+};
+
+use kernel::{
+ bits::bit_u32,
+ device::Device,
+ prelude::*,
+ sizes::SZ_4K, //
+};
+
+use crate::{
+ fw::{
+ CacheMode,
+ SectionFlags,
+ CSF_MCU_SHARED_REGION_START, //
+ },
+ vm::{
+ VmFlag,
+ VmMapFlags, //
+ }, //
+};
+
+/// A parsed firmware section ready for loading into MCU memory.
+///
+/// Represents a single firmware section extracted from the firmware binary, containing
+/// all information needed to map the section's data into the MCU's virtual address space.
+pub(super) struct ParsedSection {
+ /// Byte offset range within the firmware binary where this section's data resides.
+ pub(super) data_range: Range<u32>,
+ /// MCU virtual address range where this section should be mapped.
+ pub(super) va: Range<u32>,
+ /// Memory protection and caching flags for the mapping.
+ pub(super) vm_map_flags: VmMapFlags,
+}
+
+/// A bare-bones `std::io::Cursor<[u8]>` clone to keep track of the current position in the
+/// firmware binary.
+///
+/// Provides methods to sequentially read primitive types and byte arrays from the firmware
+/// binary while maintaining the current read position.
+struct Cursor<'a> {
+ dev: &'a Device,
+ data: &'a [u8],
+ pos: usize,
+}
+
+impl<'a> Cursor<'a> {
+ fn new(dev: &'a Device, data: &'a [u8]) -> Self {
+ Self { dev, data, pos: 0 }
+ }
+
+ fn len(&self) -> usize {
+ self.data.len()
+ }
+
+ fn pos(&self) -> usize {
+ self.pos
+ }
+
+ /// Returns a view into the cursor's data.
+ ///
+ /// This spawns a new cursor, leaving the current cursor unchanged.
+ fn view(&self, range: Range<usize>) -> Result<Cursor<'_>> {
+ if range.start < self.pos || range.end > self.data.len() {
+ dev_err!(
+ self.dev,
+ "Invalid cursor range {:?} for data of length {}",
+ range,
+ self.data.len()
+ );
+
+ Err(EINVAL)
+ } else {
+ Ok(Self {
+ dev: self.dev,
+ data: &self.data[range],
+ pos: 0,
+ })
+ }
+ }
+
+ /// Reads a slice of bytes from the current position and advances the cursor.
+ ///
+ /// Returns an error if the read would exceed the data bounds.
+ fn read(&mut self, nbytes: usize) -> Result<&[u8]> {
+ let start = self.pos;
+ let end = start + nbytes;
+
+ if end > self.data.len() {
+ dev_err!(
+ self.dev,
+ "Invalid firmware file: read of size {} at position {} is out of bounds",
+ nbytes,
+ start,
+ );
+ return Err(EINVAL);
+ }
+
+ self.pos += nbytes;
+ Ok(&self.data[start..end])
+ }
+
+ /// Reads a little-endian `u8` from the current position and advances the cursor.
+ fn read_u8(&mut self) -> Result<u8> {
+ let bytes = self.read(size_of::<u8>())?;
+ Ok(bytes[0])
+ }
+
+ /// Reads a little-endian `u16` from the current position and advances the cursor.
+ fn read_u16(&mut self) -> Result<u16> {
+ let bytes: [u8; 2] = self
+ .read(size_of::<u16>())?
+ .try_into()
+ .map_err(|_| EINVAL)?;
+
+ Ok(u16::from_le_bytes(bytes))
+ }
+
+ /// Reads a little-endian `u32` from the current position and advances the cursor.
+ fn read_u32(&mut self) -> Result<u32> {
+ let bytes: [u8; 4] = self
+ .read(size_of::<u32>())?
+ .try_into()
+ .map_err(|_| EINVAL)?;
+
+ Ok(u32::from_le_bytes(bytes))
+ }
+
+ /// Advances the cursor position by the specified number of bytes.
+ ///
+ /// Returns an error if the advance would exceed the data bounds.
+ fn advance(&mut self, nbytes: usize) -> Result {
+ if self.pos + nbytes > self.data.len() {
+ dev_err!(
+ self.dev,
+ "Invalid firmware file: advance of size {} at position {} is out of bounds",
+ nbytes,
+ self.pos,
+ );
+ return Err(EINVAL);
+ }
+ self.pos += nbytes;
+ Ok(())
+ }
+}
+
+/// Parser for Mali CSF GPU firmware binaries.
+///
+/// Parses the firmware binary format, extracting section metadata including virtual
+/// address ranges, data offsets, and memory protection flags needed to load firmware
+/// into the MCU's memory.
+pub(super) struct FwParser<'a> {
+ cursor: Cursor<'a>,
+}
+
+impl<'a> FwParser<'a> {
+ /// Creates a new firmware parser for the given firmware binary data.
+ pub(super) fn new(dev: &'a Device, data: &'a [u8]) -> Self {
+ Self {
+ cursor: Cursor::new(dev, data),
+ }
+ }
+
+ /// Parses the firmware binary and returns a collection of parsed sections.
+ ///
+ /// This method validates the firmware header and iterates through all entries
+ /// in the binary, extracting section information needed for loading.
+ pub(super) fn parse(&mut self) -> Result<KVec<ParsedSection>> {
+ let fw_header = self.parse_fw_header()?;
+ let header_end = fw_header.size as usize;
+
+ let mut parsed_sections = KVec::new();
+ while self.cursor.pos() < header_end {
+ let entry_section = self.parse_entry(header_end)?;
+
+ if let Some(inner) = entry_section.inner {
+ parsed_sections.push(inner, GFP_KERNEL)?;
+ }
+ }
+
+ if parsed_sections.is_empty() {
+ dev_err!(self.cursor.dev, "Firmware contains no loadable sections");
+ return Err(EINVAL);
+ }
+
+ Ok(parsed_sections)
+ }
+
+ fn parse_fw_header(&mut self) -> Result<FirmwareHeader> {
+ let fw_header: FirmwareHeader = match FirmwareHeader::new(&mut self.cursor) {
+ Ok(fw_header) => fw_header,
+ Err(e) => {
+ dev_err!(self.cursor.dev, "Invalid firmware file: {}", e.to_errno());
+ return Err(e);
+ }
+ };
+
+ if fw_header.size as usize > self.cursor.len() {
+ dev_err!(self.cursor.dev, "Firmware image is truncated");
+ return Err(EINVAL);
+ }
+ Ok(fw_header)
+ }
+
+ fn parse_entry(&mut self, header_end: usize) -> Result<EntrySection> {
+ let entry_start = self.cursor.pos();
+
+ let entry_header_end = entry_start
+ .checked_add(size_of::<EntryHeader>())
+ .ok_or(EINVAL)?;
+
+ if entry_header_end > header_end {
+ dev_err!(
+ self.cursor.dev,
+ "Firmware entry header at {:#x} exceeds header region ending at {:#x}",
+ entry_start,
+ header_end
+ );
+ return Err(EINVAL);
+ }
+
+ let entry_section = EntrySection {
+ entry_hdr: EntryHeader(self.cursor.read_u32()?),
+ inner: None,
+ };
+
+ let firmware_size = self.cursor.len();
+ let entry_size = entry_section.entry_hdr.size() as usize;
+
+ if self.cursor.pos() % size_of::<u32>() != 0
+ || entry_size % size_of::<u32>() != 0
+ || entry_size < size_of::<EntryHeader>()
+ {
+ dev_err!(
+ self.cursor.dev,
+ "Firmware entry isn't 32 bit aligned, offset={:#x} size={:#x}",
+ self.cursor.pos() - size_of::<u32>(),
+ entry_size
+ );
+ return Err(EINVAL);
+ }
+
+ let entry_end = entry_start.checked_add(entry_size).ok_or(EINVAL)?;
+
+ if entry_end > header_end {
+ dev_err!(
+ self.cursor.dev,
+ "Firmware entry at {:#x} extends beyond header region ending at {:#x}",
+ entry_start,
+ header_end
+ );
+ return Err(EINVAL);
+ }
+
+ let section_hdr_size = entry_size - size_of::<EntryHeader>();
+
+ let entry_section = {
+ let mut entry_cursor = self.cursor.view(self.cursor.pos()..entry_end)?;
+
+ match entry_section.entry_hdr.entry_type() {
+ Ok(EntryType::Iface) => Ok(EntrySection {
+ entry_hdr: entry_section.entry_hdr,
+ inner: Self::parse_section_entry(&mut entry_cursor, firmware_size)?,
+ }),
+ Ok(
+ EntryType::Config
+ | EntryType::FutfTest
+ | EntryType::TraceBuffer
+ | EntryType::TimelineMetadata
+ | EntryType::BuildInfoMetadata,
+ ) => Ok(entry_section),
+
+ Err(_) => {
+ if entry_section.entry_hdr.optional() {
+ Ok(entry_section)
+ } else {
+ dev_err!(
+ self.cursor.dev,
+ "Failed to handle firmware entry type: {}",
+ entry_section.entry_hdr.entry_type_raw()
+ );
+ Err(EINVAL)
+ }
+ }
+ }
+ };
+
+ if entry_section.is_ok() {
+ self.cursor.advance(section_hdr_size)?;
+ }
+
+ entry_section
+ }
+
+ fn parse_section_entry(
+ entry_cursor: &mut Cursor<'_>,
+ firmware_size: usize,
+ ) -> Result<Option<ParsedSection>> {
+ let section_hdr: SectionHeader = SectionHeader::new(entry_cursor)?;
+
+ if section_hdr.data.end < section_hdr.data.start {
+ dev_err!(
+ entry_cursor.dev,
+ "Firmware corrupted, data.end < data.start (0x{:x} < 0x{:x})",
+ section_hdr.data.end,
+ section_hdr.data.start
+ );
+ return Err(EINVAL);
+ }
+
+ if section_hdr.data.end as usize > firmware_size {
+ dev_err!(
+ entry_cursor.dev,
+ "Firmware data range {:#x}..{:#x} exceeds firmware size {:#x}",
+ section_hdr.data.start,
+ section_hdr.data.end,
+ firmware_size,
+ );
+ return Err(EINVAL);
+ }
+
+ if section_hdr.va.start as usize % SZ_4K != 0 || section_hdr.va.end as usize % SZ_4K != 0 {
+ dev_err!(
+ entry_cursor.dev,
+ "Firmware virtual address range {:#x}..{:#x} is not page aligned",
+ section_hdr.va.start,
+ section_hdr.va.end
+ );
+ return Err(EINVAL);
+ }
+
+ if section_hdr.section_flags.prot() {
+ dev_dbg!(
+ entry_cursor.dev,
+ "Firmware protected mode entry not supported, ignoring"
+ );
+ return Ok(None);
+ }
+
+ if section_hdr.va.start == CSF_MCU_SHARED_REGION_START
+ && !section_hdr.section_flags.shared()
+ {
+ dev_err!(
+ entry_cursor.dev,
+ "Interface at 0x{:x} must be shared",
+ CSF_MCU_SHARED_REGION_START
+ );
+ return Err(EINVAL);
+ }
+
+ if section_hdr.va.is_empty() {
+ return Ok(None);
+ }
+
+ let mut vm_map_flags = VmMapFlags::empty();
+
+ if !section_hdr.section_flags.write() {
+ vm_map_flags |= VmFlag::Readonly;
+ }
+
+ if !section_hdr.section_flags.exec() {
+ vm_map_flags |= VmFlag::Noexec;
+ }
+
+ // TODO: As in Panthor, map coherent firmware sections uncached until the VM
+ // supports a coherent mapping attribute.
+ if section_hdr.section_flags.cache_mode() != CacheMode::Cached {
+ vm_map_flags |= VmFlag::Uncached;
+ }
+
+ Ok(Some(ParsedSection {
+ data_range: section_hdr.data.clone(),
+ va: section_hdr.va,
+ vm_map_flags,
+ }))
+ }
+}
+
+/// Firmware binary header containing version and size information.
+///
+/// The header is located at the beginning of the firmware binary and contains
+/// a magic value for validation, version information, and the total size of
+/// all structured headers that follow.
+#[expect(dead_code)]
+struct FirmwareHeader {
+ /// Magic value to check binary validity.
+ magic: u32,
+
+ /// Minor firmware version.
+ minor: u8,
+
+ /// Major firmware version.
+ major: u8,
+
+ /// Padding. Must be set to zero.
+ _padding1: u16,
+
+ /// Firmware version hash.
+ version_hash: u32,
+
+ /// Padding. Must be set to zero.
+ _padding2: u32,
+
+ /// Total size of all the structured data headers at beginning of firmware binary.
+ size: u32,
+}
+
+impl FirmwareHeader {
+ const FW_BINARY_MAGIC: u32 = 0xc3f13a6e;
+ const FW_BINARY_MAJOR_MAX: u8 = 0;
+
+ /// Reads and validates a firmware header from the cursor.
+ ///
+ /// Verifies the magic value, version compatibility, and padding fields.
+ fn new(cursor: &mut Cursor<'_>) -> Result<Self> {
+ let magic = cursor.read_u32()?;
+ if magic != Self::FW_BINARY_MAGIC {
+ dev_err!(cursor.dev, "Invalid firmware magic");
+ return Err(EINVAL);
+ }
+
+ let minor = cursor.read_u8()?;
+ let major = cursor.read_u8()?;
+
+ if major > Self::FW_BINARY_MAJOR_MAX {
+ dev_err!(
+ cursor.dev,
+ "Unsupported firmware binary header version {}.{} (expected {}.x)",
+ major,
+ minor,
+ Self::FW_BINARY_MAJOR_MAX
+ );
+ return Err(EINVAL);
+ }
+
+ let padding1 = cursor.read_u16()?;
+ let version_hash = cursor.read_u32()?;
+ let padding2 = cursor.read_u32()?;
+ let size = cursor.read_u32()?;
+
+ if padding1 != 0 || padding2 != 0 {
+ dev_err!(
+ cursor.dev,
+ "Invalid firmware file: header padding is not zero"
+ );
+ return Err(EINVAL);
+ }
+
+ let fw_header = Self {
+ magic,
+ minor,
+ major,
+ _padding1: padding1,
+ version_hash,
+ _padding2: padding2,
+ size,
+ };
+
+ Ok(fw_header)
+ }
+}
+
+/// Firmware section header for loading binary sections into MCU memory.
+#[derive(Debug)]
+struct SectionHeader {
+ section_flags: SectionFlags,
+ /// MCU virtual range to map this binary section to.
+ va: Range<u32>,
+ /// References the data in the FW binary.
+ data: Range<u32>,
+}
+
+impl SectionHeader {
+ /// Reads and validates a section header from the cursor.
+ ///
+ /// Parses section flags, virtual address range, and data range from the firmware binary.
+ fn new(cursor: &mut Cursor<'_>) -> Result<Self> {
+ let section_flags = SectionFlags::try_from_fw(cursor.read_u32()?)?;
+
+ let va_start = cursor.read_u32()?;
+ let va_end = cursor.read_u32()?;
+
+ let va = va_start..va_end;
+
+ if va.end < va.start {
+ dev_err!(
+ cursor.dev,
+ "Invalid firmware file: VA end precedes start at pos {}",
+ cursor.pos(),
+ );
+ return Err(EINVAL);
+ }
+
+ let data_start = cursor.read_u32()?;
+ let data_end = cursor.read_u32()?;
+ let data = data_start..data_end;
+
+ Ok(Self {
+ section_flags,
+ va,
+ data,
+ })
+ }
+}
+
+/// A firmware entry containing a header and optional parsed section data.
+///
+/// Represents a single entry in the firmware binary, which may contain loadable
+/// section data or metadata that doesn't require loading.
+struct EntrySection {
+ entry_hdr: EntryHeader,
+ inner: Option<ParsedSection>,
+}
+
+/// Header for a firmware entry, packed into a single u32.
+///
+/// The entry header encodes the entry type, size, and optional flag in a
+/// 32-bit value with the following layout:
+/// - Bits 0-7: Entry type
+/// - Bits 8-15: Size in bytes
+/// - Bit 31: Optional flag
+struct EntryHeader(u32);
+
+impl EntryHeader {
+ fn entry_type_raw(&self) -> u8 {
+ (self.0 & 0xff) as u8
+ }
+
+ fn entry_type(&self) -> Result<EntryType> {
+ let v = self.entry_type_raw();
+ EntryType::try_from(v)
+ }
+
+ fn optional(&self) -> bool {
+ self.0 & bit_u32(31) != 0
+ }
+
+ fn size(&self) -> u32 {
+ self.0 >> 8 & 0xff
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+enum EntryType {
+ /// Host <-> FW interface.
+ Iface = 0,
+ /// FW config.
+ Config = 1,
+ /// Unit tests.
+ FutfTest = 2,
+ /// Trace buffer interface.
+ TraceBuffer = 3,
+ /// Timeline metadata interface.
+ TimelineMetadata = 4,
+ /// Metadata about how the FW binary was built.
+ BuildInfoMetadata = 6,
+}
+
+impl TryFrom<u8> for EntryType {
+ type Error = Error;
+
+ fn try_from(value: u8) -> Result<Self, Self::Error> {
+ match value {
+ 0 => Ok(EntryType::Iface),
+ 1 => Ok(EntryType::Config),
+ 2 => Ok(EntryType::FutfTest),
+ 3 => Ok(EntryType::TraceBuffer),
+ 4 => Ok(EntryType::TimelineMetadata),
+ 6 => Ok(EntryType::BuildInfoMetadata),
+ _ => Err(EINVAL),
+ }
+ }
+}
diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs
new file mode 100644
index 000000000..3bf3787f5
--- /dev/null
+++ b/drivers/gpu/drm/tyr/gem.rs
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+//! GEM buffer object management for the Tyr driver.
+//!
+//! This module provides buffer object (BO) management functionality using
+//! DRM's GEM subsystem with shmem backing.
+
+use core::ops::Range;
+
+use kernel::{
+ drm::gem::{
+ self,
+ shmem, //
+ },
+ prelude::*,
+ sync::{
+ aref::ARef,
+ Arc, //
+ }, //
+};
+
+use crate::{
+ driver::{
+ TyrDrmDevice,
+ TyrDrmDriver, //
+ },
+ vm::{
+ Vm,
+ VmMapFlags, //
+ },
+};
+
+/// Tyr's DriverObject type for GEM objects.
+#[pin_data]
+pub(crate) struct BoData {
+ flags: u32,
+}
+
+/// Provides a way to pass arguments when creating BoData
+/// as required by the gem::DriverObject trait.
+pub(crate) struct BoCreateArgs {
+ flags: u32,
+}
+
+impl gem::DriverObject for BoData {
+ type Driver = TyrDrmDriver;
+ type Args = BoCreateArgs;
+
+ fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit<Self, Error> {
+ try_pin_init!(Self { flags: args.flags })
+ }
+}
+
+/// Type alias for Tyr GEM buffer objects.
+pub(crate) type Bo = gem::shmem::Object<BoData>;
+
+/// Creates a dummy GEM object to serve as the root of a GPUVM.
+pub(crate) fn new_dummy_object(ddev: &TyrDrmDevice) -> Result<ARef<Bo>> {
+ let bo = Bo::new(
+ ddev,
+ 4096,
+ shmem::ObjectConfig {
+ map_wc: true,
+ parent_resv_obj: None,
+ },
+ BoCreateArgs { flags: 0 },
+ )?;
+
+ Ok(bo)
+}
+
+/// Specifies how to choose a GPU virtual address for a [`KernelBo`].
+/// An automatic VA allocation strategy will be added in the future.
+pub(crate) enum KernelBoVaAlloc {
+ /// Explicit VA address specified by the caller.
+ Explicit(u64),
+}
+
+/// A kernel-owned buffer object with automatic GPU virtual address mapping.
+///
+/// This structure represents a buffer object that is created and managed entirely
+/// by the kernel driver, as opposed to userspace-created GEM objects. It combines
+/// a GEM object with automatic GPU virtual address (VA) space mapping and cleanup.
+///
+/// When dropped, the buffer is automatically unmapped from the GPU VA space.
+pub(crate) struct KernelBo<'drm> {
+ /// The underlying GEM buffer object.
+ bo: ARef<Bo>,
+ /// The GPU VM this buffer is mapped into.
+ vm: Arc<Vm<'drm>>,
+ /// The GPU VA range occupied by this buffer.
+ va_range: Range<u64>,
+}
+
+impl<'drm> KernelBo<'drm> {
+ /// Creates a new kernel-owned buffer object and maps it into GPU VA space.
+ ///
+ /// This function allocates a new shmem-backed GEM object and immediately maps
+ /// it into the specified GPU virtual memory space. The mapping is automatically
+ /// cleaned up when the [`KernelBo`] is dropped.
+ pub(crate) fn new(
+ ddev: &TyrDrmDevice,
+ vm: Arc<Vm<'drm>>,
+ size: u64,
+ va_alloc: KernelBoVaAlloc,
+ flags: VmMapFlags,
+ ) -> Result<Self> {
+ if size == 0 {
+ dev_err!(vm.dev(), "Cannot create KernelBo with size 0");
+ return Err(EINVAL);
+ }
+
+ let KernelBoVaAlloc::Explicit(va) = va_alloc;
+
+ let bo_size = usize::try_from(size).map_err(|_| EOVERFLOW)?;
+ let va_end = va.checked_add(size).ok_or(EINVAL)?;
+
+ let bo = Bo::new(
+ ddev,
+ bo_size,
+ shmem::ObjectConfig {
+ map_wc: true,
+ parent_resv_obj: None,
+ },
+ BoCreateArgs { flags: 0 },
+ )?;
+
+ vm.map_bo_range(&bo, 0, size, va, flags)?;
+
+ Ok(KernelBo {
+ bo,
+ vm,
+ va_range: va..va_end,
+ })
+ }
+
+ pub(crate) fn bo(&self) -> &Bo {
+ &self.bo
+ }
+}
+
+impl Drop for KernelBo<'_> {
+ fn drop(&mut self) {
+ let va = self.va_range.start;
+ let size = self.va_range.end - self.va_range.start;
+
+ if let Err(e) = self.vm.unmap_range(va, size) {
+ // If unmap_range fails, it is still safe to drop the
+ // KernelBo and its ARef to the GEM buffer object because
+ // GPUVM also holds a reference to the GEM buffer object.
+ // The physical pages won't be freed or reallocated.
+ dev_err!(
+ self.vm.dev(),
+ "Failed to unmap KernelBo range {:#x}..{:#x}: {:?}",
+ self.va_range.start,
+ self.va_range.end,
+ e
+ );
+ }
+ }
+}
diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs
new file mode 100644
index 000000000..592b8bb16
--- /dev/null
+++ b/drivers/gpu/drm/tyr/gpu.rs
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+use core::ops::{
+ Deref,
+ DerefMut, //
+};
+use kernel::{
+ device::{
+ Bound,
+ Device, //
+ },
+ io::{
+ poll,
+ register::Array,
+ Io, //
+ },
+ prelude::*,
+ time::Delta,
+ transmute::AsBytes,
+ uapi, //
+};
+
+use crate::{
+ driver::IoMem,
+ regs::{
+ gpu_control::*,
+ join_u64, //
+ }, //
+};
+
+/// Struct containing information that can be queried by userspace. This is read from
+/// the GPU's registers.
+///
+/// # Invariants
+///
+/// - The layout of this struct is identical to the C `struct drm_panthor_gpu_info`.
+#[repr(transparent)]
+#[derive(Clone, Copy)]
+pub(crate) struct GpuInfo(pub(crate) uapi::drm_panthor_gpu_info);
+
+impl GpuInfo {
+ pub(crate) fn new(io: &IoMem<'_>) -> Self {
+ Self(uapi::drm_panthor_gpu_info {
+ gpu_id: io.read(GPU_ID).into_raw(),
+ gpu_rev: io.read(REVIDR).into_raw(),
+ csf_id: io.read(CSF_ID).into_raw(),
+ l2_features: io.read(L2_FEATURES).into_raw(),
+ tiler_features: io.read(TILER_FEATURES).into_raw(),
+ mem_features: io.read(MEM_FEATURES).into_raw(),
+ mmu_features: io.read(MMU_FEATURES).into_raw(),
+ thread_features: io.read(THREAD_FEATURES).into_raw(),
+ max_threads: io.read(THREAD_MAX_THREADS).into_raw(),
+ thread_max_workgroup_size: io.read(THREAD_MAX_WORKGROUP_SIZE).into_raw(),
+ thread_max_barrier_size: io.read(THREAD_MAX_BARRIER_SIZE).into_raw(),
+ coherency_features: io.read(COHERENCY_FEATURES).into_raw(),
+ texture_features: [
+ io.read(TEXTURE_FEATURES::at(0)).supported_formats().get(),
+ io.read(TEXTURE_FEATURES::at(1)).supported_formats().get(),
+ io.read(TEXTURE_FEATURES::at(2)).supported_formats().get(),
+ io.read(TEXTURE_FEATURES::at(3)).supported_formats().get(),
+ ],
+ as_present: io.read(AS_PRESENT).into_raw(),
+ selected_coherency: uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE,
+ shader_present: join_u64(
+ io.read(SHADER_PRESENT_LO).into_raw(),
+ io.read(SHADER_PRESENT_HI).into_raw(),
+ ),
+ l2_present: join_u64(
+ io.read(L2_PRESENT_LO).into_raw(),
+ io.read(L2_PRESENT_HI).into_raw(),
+ ),
+ tiler_present: join_u64(
+ io.read(TILER_PRESENT_LO).into_raw(),
+ io.read(TILER_PRESENT_HI).into_raw(),
+ ),
+ core_features: io.read(CORE_FEATURES).into_raw(),
+ // Padding must be zero.
+ pad: 0,
+ //GPU_FEATURES register is not available; it was introduced in arch 11.x.
+ gpu_features: 0,
+ })
+ }
+
+ pub(crate) fn log(&self, dev: &Device<Bound>) {
+ let gpu_id = GPU_ID::from_raw(self.gpu_id);
+
+ let model_name = if let Some(model) = GPU_MODELS.iter().find(|&f| {
+ f.arch_major == gpu_id.arch_major().get() && f.prod_major == gpu_id.prod_major().get()
+ }) {
+ model.name
+ } else {
+ "unknown"
+ };
+
+ dev_info!(
+ dev,
+ "mali-{} GPU_ID 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}",
+ model_name,
+ gpu_id.into_raw(),
+ gpu_id.ver_major().get(),
+ gpu_id.ver_minor().get(),
+ gpu_id.ver_status().get()
+ );
+
+ dev_info!(
+ dev,
+ "Features: L2:{:#x} Tiler:{:#x} Mem:{:#x} MMU:{:#x} AS:{:#x}",
+ self.l2_features,
+ self.tiler_features,
+ self.mem_features,
+ self.mmu_features,
+ self.as_present,
+ );
+
+ dev_info!(
+ dev,
+ "shader_present=0x{:016x} l2_present=0x{:016x} tiler_present=0x{:016x}",
+ self.shader_present,
+ self.l2_present,
+ self.tiler_present,
+ );
+ }
+}
+
+impl Deref for GpuInfo {
+ type Target = uapi::drm_panthor_gpu_info;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+impl DerefMut for GpuInfo {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.0
+ }
+}
+
+// SAFETY: `GpuInfo`'s invariant guarantees that it is the same type that is
+// already exposed to userspace by the C driver. This implies that it fulfills
+// the requirements for `AsBytes`.
+//
+// This means:
+//
+// - No implicit padding,
+// - No kernel pointers,
+// - No interior mutability.
+unsafe impl AsBytes for GpuInfo {}
+
+struct GpuModels {
+ name: &'static str,
+ arch_major: u32,
+ prod_major: u32,
+}
+
+const GPU_MODELS: [GpuModels; 1] = [GpuModels {
+ name: "g610",
+ arch_major: 10,
+ prod_major: 7,
+}];
+
+/// Powers on the l2 block.
+pub(crate) fn l2_power_on(dev: &Device, io: &IoMem<'_>) -> Result {
+ io.write_reg(L2_PWRON_LO::zeroed().with_const_request::<1>());
+
+ poll::read_poll_timeout(
+ || Ok(io.read(L2_READY_LO)),
+ |status| status.ready() == 1,
+ Delta::from_millis(1),
+ Delta::from_millis(100),
+ )
+ .inspect_err(|_| dev_err!(dev, "Failed to power on the GPU."))?;
+
+ Ok(())
+}
diff --git a/drivers/gpu/drm/tyr/mmu.rs b/drivers/gpu/drm/tyr/mmu.rs
new file mode 100644
index 000000000..5e88c9e6a
--- /dev/null
+++ b/drivers/gpu/drm/tyr/mmu.rs
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! Memory Management Unit (MMU) module.
+//!
+//! The GPU MMU provides a limited number of memory address spaces for use by command streams.
+//! The MMU translates virtual addresses to physical addresses and manages memory configuration
+//! and access permissions.
+//!
+//! This MMU module is essentially a locked wrapper around a [`SlotManager`] instance.
+//! The [`SlotManager`] manages the assignment of virtual address spaces to hardware address-space
+//! (AS) slots. MMU commands such as updates and flushes are carried out by the
+//! [`AddressSpaceManager`] which actually writes to the MMU registers.
+
+use core::ops::Range;
+
+use kernel::{
+ device::{
+ Bound,
+ Device, //
+ },
+ new_mutex,
+ prelude::*,
+ sync::{
+ Arc,
+ ArcBorrow,
+ Mutex, //
+ }, //
+};
+
+use crate::{
+ driver::IoMem,
+ gpu::GpuInfo,
+ mmu::address_space::{
+ AddressSpaceManager,
+ VmAsData, //
+ },
+ regs::{
+ gpu_control::AS_PRESENT,
+ MAX_AS, //
+ },
+ slot::SlotManager, //
+};
+
+pub(crate) mod address_space;
+
+pub(crate) type AsSlotManager<'drm> = SlotManager<AddressSpaceManager<'drm>, MAX_AS>;
+
+/// Locked wrapper for carrying out virtual memory (VM) operations on the MMU.
+#[pin_data]
+pub(crate) struct Mmu<'drm> {
+ /// Slot Manager instance used to allocate hardware slots and write to MMU registers.
+ #[pin]
+ pub(crate) as_manager: Mutex<AsSlotManager<'drm>>,
+}
+
+impl<'drm> Mmu<'drm> {
+ /// Create an MMU component for this device.
+ pub(crate) fn new(
+ dev: &'drm Device<Bound>,
+ iomem: ArcBorrow<'_, IoMem<'drm>>,
+ gpu_info: &GpuInfo,
+ ) -> Result<Arc<Mmu<'drm>>> {
+ let present = AS_PRESENT::from_raw(gpu_info.as_present).present().get();
+ let slot_count = present.count_ones().try_into()?;
+
+ let address_space_manager = AddressSpaceManager::new(dev, iomem.into(), present)?;
+ let as_slot_manager =
+ SlotManager::new(address_space_manager, slot_count).inspect_err(|e| {
+ dev_err!(
+ dev,
+ "Failed to initialize MMU slot manager with {} slots: {:?}",
+ slot_count,
+ e
+ );
+ })?;
+ let mmu_init = try_pin_init!(Self{
+ as_manager <- new_mutex!(as_slot_manager),
+ });
+ Arc::pin_init(mmu_init, GFP_KERNEL)
+ }
+
+ /// Assign a VM to an AS slot, provide a translation table,
+ /// and update the MMU to make the VM resident.
+ pub(crate) fn activate_vm(&self, vm_as_data: ArcBorrow<'_, VmAsData<'drm>>) -> Result {
+ self.as_manager.lock().activate_vm(vm_as_data)
+ }
+
+ /// Evict a VM from its AS slot and flush the MMU.
+ pub(crate) fn deactivate_vm(&self, vm_as_data: &VmAsData<'drm>) -> Result {
+ self.as_manager.lock().deactivate_vm(vm_as_data)
+ }
+
+ /// Flush MMU translation caches after a VM update.
+ pub(crate) fn flush_vm(&self, vm_as_data: &VmAsData<'drm>) -> Result {
+ self.as_manager.lock().flush_vm(vm_as_data)
+ }
+
+ /// Flags the start of a VM update.
+ ///
+ /// If the VM is resident, any GPU access on the memory range being
+ /// updated will be blocked until `Mmu::end_vm_update()` is called.
+ /// This guarantees the atomicity of a VM update.
+ /// If the VM is not resident, this is a NOP.
+ pub(crate) fn start_vm_update(
+ &self,
+ vm_as_data: &VmAsData<'drm>,
+ region: &Range<u64>,
+ ) -> Result {
+ self.as_manager.lock().start_vm_update(vm_as_data, region)
+ }
+
+ /// Flags the end of a VM update.
+ ///
+ /// If the VM is resident, this will let GPU accesses on the updated
+ /// range go through, in case any of them were blocked.
+ /// If the VM is not resident, this is a NOP.
+ pub(crate) fn end_vm_update(&self, vm_as_data: &VmAsData<'drm>) -> Result {
+ self.as_manager.lock().end_vm_update(vm_as_data)
+ }
+}
diff --git a/drivers/gpu/drm/tyr/mmu/address_space.rs b/drivers/gpu/drm/tyr/mmu/address_space.rs
new file mode 100644
index 000000000..76e3d2df6
--- /dev/null
+++ b/drivers/gpu/drm/tyr/mmu/address_space.rs
@@ -0,0 +1,511 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! Address space module.
+//!
+//! This module handles the hardware interaction for MMU operations through
+//! MMIO register access.
+//!
+
+use core::ops::Range;
+
+use kernel::{
+ device::{
+ Bound,
+ Device, //
+ }, //
+ error::Result,
+ io::{
+ poll,
+ register::Array,
+ Io, //
+ },
+ iommu::pgtable::{
+ Config,
+ IoPageTable,
+ ARM64LPAES1, //
+ },
+ num::Bounded,
+ prelude::*,
+ sizes::{
+ SZ_2M,
+ SZ_4K, //
+ },
+ sync::{
+ Arc,
+ ArcBorrow,
+ LockedBy, //
+ },
+ time::Delta, //
+};
+
+use crate::{
+ driver::IoMem,
+ mmu::{
+ AsSlotManager,
+ Mmu, //
+ },
+ regs::{
+ mmu_control::mmu_as_control,
+ mmu_control::mmu_as_control::*,
+ MAX_AS, //
+ },
+ slot::{
+ LockedSeat,
+ Seat,
+ SlotOperations, //
+ }, //
+};
+
+/// Address space configuration values to be written to MMU registers.
+#[derive(Clone, Copy)]
+struct AddressSpaceConfig {
+ /// Translation configuration. Configures how the MMU walks the page table for this
+ /// address space.
+ transcfg: u64,
+
+ /// Translation table base address. The address of the page table.
+ transtab: u64,
+
+ /// Memory attributes such as cacheability.
+ memattr: u64,
+}
+
+/// Virtual memory (VM) address space data for use in MMU operations.
+#[pin_data]
+pub(crate) struct VmAsData<'drm> {
+ /// This address-space seat tracks this VM's binding to a hardware address space slot.
+ /// It can only be accessed when holding the `Mmu::as_manager` lock.
+ as_seat: LockedSeat<AddressSpaceManager<'drm>, MAX_AS>,
+
+ /// Virtual address bits for this address space.
+ va_bits: u8,
+
+ /// The page table which maps GPU virtual addresses to physical addresses for this VM.
+ #[pin]
+ pub(crate) page_table: IoPageTable<'drm, ARM64LPAES1>,
+}
+
+impl<'drm> VmAsData<'drm> {
+ /// Creates VM address space data by initializing all of its fields.
+ pub(crate) fn new<'a>(
+ mmu: &'a Mmu<'drm>,
+ dev: &'drm Device<Bound>,
+ va_bits: u32,
+ pa_bits: u32,
+ ) -> impl pin_init::PinInit<VmAsData<'drm>, Error> + 'a {
+ let pt_config = Config {
+ quirks: 0,
+ pgsize_bitmap: SZ_4K | SZ_2M,
+ ias: va_bits,
+ oas: pa_bits,
+ coherent_walk: false,
+ };
+
+ let page_table_init = IoPageTable::new(dev, pt_config);
+
+ try_pin_init!(Self {
+ as_seat: LockedBy::new(&mmu.as_manager, Seat::NoSeat),
+ va_bits: va_bits as u8,
+ page_table <- page_table_init,
+ }? Error)
+ }
+
+ /// Computes the hardware configuration for this address space.
+ fn as_config(&self) -> Result<AddressSpaceConfig> {
+ let pt = &self.page_table;
+ // The hardware computes the valid input address range as:
+ // INA_BITS_VALID = min(HW_INA_BITS, 55 - INA_BITS)
+ // To configure our desired va_bits, we solve for INA_BITS:
+ // INA_BITS = 55 - va_bits
+ // This assumes HW_INA_BITS (hardware capability) >= va_bits.
+ let field = 55u64.checked_sub(self.va_bits.into()).ok_or(EINVAL)?;
+ let ina_bits =
+ match mmu_as_control::InaBits::try_from(Bounded::try_new(field).ok_or(EINVAL)?)? {
+ mmu_as_control::InaBits::Reset => return Err(EINVAL),
+ bits => bits,
+ };
+
+ let transcfg = mmu_as_control::TRANSCFG::zeroed()
+ .with_ptw_memattr(mmu_as_control::PtwMemattr::WriteBack)
+ .with_r_allocate(true)
+ .with_mode(mmu_as_control::AddressSpaceMode::Aarch64_4K)
+ .with_ina_bits(ina_bits)
+ .into_raw();
+
+ Ok(AddressSpaceConfig {
+ transcfg,
+ // SAFETY: The SlotManager holds an `Arc<VmAsData>` as SlotData while this
+ // TTBR is programmed and stores that Arc in the active slot before
+ // returning. Eviction flushes and disables the slot before releasing
+ // the Arc; if eviction fails, the slot retains it. Therefore the page
+ // table cannot be dropped while the GPU is using it.
+ transtab: unsafe { pt.ttbr() },
+ memattr: MEMATTR::from_mair(pt.mair()).into_raw(),
+ })
+ }
+}
+
+/// Coordinates all hardware-level address space operations through MMIO register
+/// operations including enabling, disabling, flushing, and updating address spaces.
+pub(crate) struct AddressSpaceManager<'drm> {
+ /// Parent device used for logging.
+ dev: &'drm Device<Bound>,
+
+ /// Memory-mapped I/O region for GPU register access.
+ iomem: Arc<IoMem<'drm>>,
+
+ /// Bitmask of present address space slots from GPU_AS_PRESENT register.
+ as_present: u32,
+}
+
+impl<'drm> AddressSpaceManager<'drm> {
+ /// Creates a new address space manager.
+ ///
+ /// Initializes the manager with references to the platform device and
+ /// I/O memory region, along with the bitmask of available AS slots.
+ pub(super) fn new(
+ dev: &'drm Device<Bound>,
+ iomem: Arc<IoMem<'drm>>,
+ as_present: u32,
+ ) -> Result<AddressSpaceManager<'drm>> {
+ if as_present.trailing_ones() != as_present.count_ones() {
+ dev_err!(
+ dev,
+ "Sparse AS_PRESENT mask is unsupported: {:#x}",
+ as_present
+ );
+ return Err(EINVAL);
+ }
+ Ok(Self {
+ dev,
+ iomem,
+ as_present,
+ })
+ }
+
+ /// Validates that an AS slot number is within range and present in hardware.
+ ///
+ /// Checks that the slot index is less than [`MAX_AS`] and that
+ /// the corresponding bit is set in the `as_present` mask read from the GPU.
+ ///
+ /// Returns [`EINVAL`] if the slot is out of range or not present in hardware.
+ fn validate_as_slot(&self, as_nr: usize) -> Result {
+ if as_nr >= MAX_AS {
+ dev_err!(
+ self.dev,
+ "AS slot {} out of valid range (max {})",
+ as_nr,
+ MAX_AS
+ );
+ return Err(EINVAL);
+ }
+
+ if (self.as_present & (1 << as_nr)) == 0 {
+ dev_err!(
+ self.dev,
+ "AS slot {} not present in hardware (AS_PRESENT={:#x})",
+ as_nr,
+ self.as_present
+ );
+ return Err(EINVAL);
+ }
+ Ok(())
+ }
+
+ /// Waits for an AS slot to become ready (not active).
+ ///
+ /// Returns an error if polling times out after 10ms or if register access fails.
+ fn as_wait_ready(&self, as_nr: usize) -> Result {
+ let io = &*self.iomem;
+ let op = || {
+ let status_reg = STATUS::try_at(as_nr).ok_or(EINVAL)?;
+ Ok(io.read(status_reg))
+ };
+ let cond = |status: &STATUS| -> bool { !status.active_ext() };
+ poll::read_poll_timeout(op, cond, Delta::from_micros(50), Delta::from_millis(10))?;
+
+ Ok(())
+ }
+
+ /// Sends a command to an AS slot.
+ ///
+ /// Returns an error if waiting for ready times out or if register write fails.
+ fn as_send_cmd(&mut self, as_nr: usize, cmd: MmuCommand) -> Result {
+ self.as_wait_ready(as_nr)?;
+ let io = &*self.iomem;
+ let command_reg = COMMAND::try_at(as_nr).ok_or(EINVAL)?;
+ io.write(command_reg, COMMAND::zeroed().with_command(cmd));
+ Ok(())
+ }
+
+ /// Sends a command to an AS slot and waits for completion.
+ ///
+ /// Returns an error if sending the command fails or if waiting for completion times out.
+ fn as_send_cmd_and_wait(&mut self, as_nr: usize, cmd: MmuCommand) -> Result {
+ self.as_send_cmd(as_nr, cmd)?;
+ self.as_wait_ready(as_nr)?;
+ Ok(())
+ }
+
+ /// Enables an AS slot with the provided configuration.
+ ///
+ /// Returns an error if the slot is invalid or if register writes/commands fail.
+ fn as_enable(&mut self, as_nr: usize, as_config: &AddressSpaceConfig) -> Result {
+ self.validate_as_slot(as_nr)?;
+
+ let io = &*self.iomem;
+
+ let transtab = as_config.transtab;
+ io.write(
+ TRANSTAB_LO::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSTAB_LO::from_raw(transtab as u32),
+ );
+ io.write(
+ TRANSTAB_HI::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSTAB_HI::from_raw((transtab >> 32) as u32),
+ );
+
+ let transcfg = as_config.transcfg;
+ io.write(
+ TRANSCFG_LO::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSCFG_LO::from_raw(transcfg as u32),
+ );
+ io.write(
+ TRANSCFG_HI::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSCFG_HI::from_raw((transcfg >> 32) as u32),
+ );
+
+ let memattr = as_config.memattr;
+ io.write(
+ MEMATTR_LO::try_at(as_nr).ok_or(EINVAL)?,
+ MEMATTR_LO::from_raw(memattr as u32),
+ );
+ io.write(
+ MEMATTR_HI::try_at(as_nr).ok_or(EINVAL)?,
+ MEMATTR_HI::from_raw((memattr >> 32) as u32),
+ );
+
+ self.as_send_cmd_and_wait(as_nr, MmuCommand::Update)?;
+
+ Ok(())
+ }
+
+ /// Disables an AS slot and clears its configuration.
+ ///
+ /// Returns an error if the slot is invalid or if register writes/commands fail.
+ fn as_disable(&mut self, as_nr: usize) -> Result {
+ self.validate_as_slot(as_nr)?;
+
+ // Flush AS before disabling
+ self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushMem)?;
+
+ let io = &*self.iomem;
+
+ io.write(
+ TRANSTAB_LO::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSTAB_LO::from_raw(0),
+ );
+ io.write(
+ TRANSTAB_HI::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSTAB_HI::from_raw(0),
+ );
+
+ io.write(
+ MEMATTR_LO::try_at(as_nr).ok_or(EINVAL)?,
+ MEMATTR_LO::from_raw(0),
+ );
+ io.write(
+ MEMATTR_HI::try_at(as_nr).ok_or(EINVAL)?,
+ MEMATTR_HI::from_raw(0),
+ );
+
+ let transcfg = TRANSCFG::zeroed()
+ .with_mode(AddressSpaceMode::Unmapped)
+ .into_raw();
+
+ io.write(
+ TRANSCFG_LO::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSCFG_LO::from_raw(transcfg as u32),
+ );
+ io.write(
+ TRANSCFG_HI::try_at(as_nr).ok_or(EINVAL)?,
+ TRANSCFG_HI::from_raw((transcfg >> 32) as u32),
+ );
+
+ self.as_send_cmd_and_wait(as_nr, MmuCommand::Update)?;
+
+ Ok(())
+ }
+
+ /// Locks a region of the translation tables for an atomic update.
+ ///
+ /// Programs the MMU [`LOCKADDR`] register for the given address space and issues
+ /// the lock command. The hardware rounds the requested range up to a
+ /// power-of-two region aligned to its size.
+ ///
+ /// Returns an error if the slot is invalid or if register writes/commands fail.
+ fn as_start_update(&mut self, as_nr: usize, region: &Range<u64>) -> Result {
+ self.validate_as_slot(as_nr)?;
+
+ // Avoid both an empty range and an inverted range.
+ if region.start >= region.end {
+ return Err(EINVAL);
+ }
+
+ // The lock operates on full 64-byte cache lines of translation table entries.
+ // Since each translation table entry (TTE) is 8 bytes, a cache line has 8 TTEs.
+ // Since each TTE maps one page, the minimum locked region size will be 8 pages.
+ //
+ // With 4KiB pages (Aarch64_4K mode), the minimum locked region is 32KiB.
+ let lock_region_min_size: u64 = 4096 * 8;
+
+ // Count the number of trailing zero bits (zeros at the right/least-significant
+ // end of the binary representation). For a power-of-two value, this equals the
+ // base-2 exponent (e.g., 32 KiB = 2^15 → 15).
+ let lock_region_min_size_log2 = lock_region_min_size.trailing_zeros() as u8;
+
+ // XOR the first and last addresses to identify which bits differ between them.
+ // The highest set bit in the result determines the exponent of the smallest
+ // power-of-two region that can contain both addresses.
+ //
+ // Example:
+ // addr_xor = 0x1000 ^ 0x2FFF = 0x3FFF
+ // highest set bit in 0x3FFF is bit 13
+ // minimum region size = 2^(13 + 1) = 16 KiB
+ let addr_xor = region.start ^ (region.end - 1);
+ let region_size_log2 = 64 - addr_xor.leading_zeros() as u8;
+
+ let lock_region_log2 = core::cmp::max(region_size_log2, lock_region_min_size_log2);
+
+ let lock_region_size = 1u64.checked_shl(lock_region_log2.into()).ok_or(EINVAL)?;
+ // Align the LOCKADDR base address down to the lock region size (1 << lock_region_log2).
+ //
+ // The MMU ignores the low lock_region_log2 bits of LOCKADDR base, so ensure
+ // they are cleared in software to avoid ambiguity.
+ //
+ // Example:
+ // lock_region_log2 = 14 (16 KiB)
+ // region.start = 0x1000
+ // lockaddr_base = 0x1000 & ~(0x3FFF) = 0x0000
+ let lockaddr_base = region.start & !(lock_region_size - 1);
+
+ // The LOCKADDR size field encodes the lock region size as log2(size) - 1,
+ // per the hardware definition. For example, a 32 KiB region is encoded as 14
+ // because log2(32 KiB) = 15.
+ let lockaddr_size = lock_region_log2 - 1;
+
+ let io = &*self.iomem;
+
+ // The LOCKADDR base field stores address bits 63:12, so remove the low 12 bits
+ // before passing this value to the register macro helper.
+ // These bits are guaranteed to be zero anyway because of the minimum
+ // size of the locked region.
+ let lockaddr_base_field = lockaddr_base >> 12;
+ let lockaddr_val = LOCKADDR::zeroed()
+ .try_with_size(lockaddr_size)?
+ .try_with_base(lockaddr_base_field)?
+ .into_raw();
+
+ io.write(
+ LOCKADDR_LO::try_at(as_nr).ok_or(EINVAL)?,
+ LOCKADDR_LO::from_raw(lockaddr_val as u32),
+ );
+ io.write(
+ LOCKADDR_HI::try_at(as_nr).ok_or(EINVAL)?,
+ LOCKADDR_HI::from_raw((lockaddr_val >> 32) as u32),
+ );
+
+ self.as_send_cmd_and_wait(as_nr, MmuCommand::Lock)
+ }
+
+ /// Completes an atomic translation table update.
+ ///
+ /// Returns an error if the slot is invalid or if the flush command fails.
+ fn as_end_update(&mut self, as_nr: usize) -> Result {
+ self.validate_as_slot(as_nr)?;
+ self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushPt)?;
+ Ok(())
+ }
+
+ /// Flushes the translation table cache for an AS slot.
+ ///
+ /// Returns an error if the slot is invalid or if the flush command fails.
+ fn as_flush(&mut self, as_nr: usize) -> Result {
+ self.validate_as_slot(as_nr)?;
+ self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushPt)
+ }
+}
+
+impl<'drm> SlotOperations<MAX_AS> for AddressSpaceManager<'drm> {
+ /// VM address space data associated with a hardware slot.
+ type SlotData = Arc<VmAsData<'drm>>;
+
+ fn seat(slot_data: &Self::SlotData) -> &LockedSeat<Self, MAX_AS> {
+ &slot_data.as_seat
+ }
+
+ /// Activates a VM in a hardware slot.
+ fn activate(&mut self, slot_idx: usize, slot_data: &Self::SlotData) -> Result {
+ let as_config = slot_data.as_config()?;
+ self.as_enable(slot_idx, &as_config)
+ }
+
+ /// Evicts a VM from a hardware slot.
+ fn evict(&mut self, slot_idx: usize, _slot_data: &Self::SlotData) -> Result {
+ self.as_flush(slot_idx)?;
+ self.as_disable(slot_idx)?;
+ Ok(())
+ }
+}
+
+impl<'drm> AsSlotManager<'drm> {
+ /// Locks a region for translation table updates if the VM has an active slot.
+ pub(super) fn start_vm_update(
+ &mut self,
+ vm_as_data: &VmAsData<'drm>,
+ region: &Range<u64>,
+ ) -> Result {
+ let seat = vm_as_data.as_seat.access(self);
+ match seat.slot() {
+ Some(slot) => {
+ let as_nr = slot as usize;
+ self.as_start_update(as_nr, region)
+ }
+ _ => Ok(()),
+ }
+ }
+
+ /// Completes translation table updates and unlocks the region.
+ pub(super) fn end_vm_update(&mut self, vm_as_data: &VmAsData<'drm>) -> Result {
+ let seat = vm_as_data.as_seat.access(self);
+ match seat.slot() {
+ Some(slot) => {
+ let as_nr = slot as usize;
+ self.as_end_update(as_nr)
+ }
+ _ => Ok(()),
+ }
+ }
+
+ /// Flushes the translation table cache if the VM has an active slot.
+ pub(super) fn flush_vm(&mut self, vm_as_data: &VmAsData<'drm>) -> Result {
+ let seat = vm_as_data.as_seat.access(self);
+ match seat.slot() {
+ Some(slot) => {
+ let as_nr = slot as usize;
+ self.as_flush(as_nr)
+ }
+ _ => Ok(()),
+ }
+ }
+
+ /// Activates a VM by assigning it to a hardware slot.
+ pub(super) fn activate_vm(&mut self, vm_as_data: ArcBorrow<'_, VmAsData<'drm>>) -> Result {
+ self.activate(vm_as_data.into())
+ }
+
+ /// Deactivates a VM by evicting it from its hardware slot.
+ pub(super) fn deactivate_vm(&mut self, vm_as_data: &VmAsData<'drm>) -> Result {
+ self.evict(&vm_as_data.as_seat)
+ }
+}
diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs
new file mode 100644
index 000000000..a62724378
--- /dev/null
+++ b/drivers/gpu/drm/tyr/regs.rs
@@ -0,0 +1,1779 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! # Definitions
+//!
+//! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions)
+//! from the command stream.
+//! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular
+//! job or sequence of jobs. The instructions exist in one or more command buffers.
+//! - **CSF**: Command Stream Frontend - The interface and implementation for job submission
+//! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS
+//! interfaces.
+//! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple
+//! CSGs, and each CSG contains multiple CSs.
+//! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the
+//! iterator control aspects. Implements the CSF in conjunction with the MCU.
+//! - **GLB**: Global - Prefix for global interface registers that control operations common to
+//! all CSs.
+//! - **JASID**: Job Address Space ID - Identifies the address space for a job.
+//! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream
+//! hardware.
+//! - **MMU**: Memory Management Unit - Handles address translation and memory access protection.
+
+// We don't expect that all the registers and fields will be used, even in the
+// future.
+//
+// Nevertheless, it is useful to have most of them defined, like the C driver
+// does.
+#![expect(dead_code)]
+
+/// Combine two 32-bit values into a single 64-bit value.
+pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 {
+ (u64::from(lo)) | ((u64::from(hi)) << 32)
+}
+
+/// Read a logical 64-bit value from split 32-bit registers without tearing.
+pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 {
+ loop {
+ let hi1 = hi_read();
+ let lo = lo_read();
+ let hi2 = hi_read();
+
+ if hi1 == hi2 {
+ return join_u64(lo, hi1);
+ }
+ }
+}
+
+pub(crate) use mmu_control::mmu_as_control::MAX_AS;
+
+/// These registers correspond to the GPU_CONTROL register page.
+/// They are involved in GPU configuration and control.
+pub(crate) mod gpu_control {
+ use kernel::{
+ num::Bounded,
+ prelude::*,
+ register,
+ uapi, //
+ };
+
+ register! {
+ /// GPU identification register.
+ pub(crate) GPU_ID(u32) @ 0x0 {
+ /// Status of the GPU release.
+ 3:0 ver_status;
+ /// Minor release version number.
+ 11:4 ver_minor;
+ /// Major release version number.
+ 15:12 ver_major;
+ /// Product identifier.
+ 19:16 prod_major;
+ /// Architecture patch revision.
+ 23:20 arch_rev;
+ /// Architecture minor revision.
+ 27:24 arch_minor;
+ /// Architecture major revision.
+ 31:28 arch_major;
+ }
+
+ /// Level 2 cache features register.
+ pub(crate) L2_FEATURES(u32) @ 0x4 {
+ /// Cache line size.
+ 7:0 line_size;
+ /// Cache associativity.
+ 15:8 associativity;
+ /// Cache slice size.
+ 23:16 cache_size;
+ /// External bus width.
+ 31:24 bus_width;
+ }
+
+ /// Shader core features.
+ pub(crate) CORE_FEATURES(u32) @ 0x8 {
+ /// Shader core variant.
+ 7:0 core_variant;
+ }
+
+ /// Tiler features.
+ pub(crate) TILER_FEATURES(u32) @ 0xc {
+ /// Log of the tiler's bin size.
+ 5:0 bin_size;
+ /// Maximum number of active levels.
+ 11:8 max_levels;
+ }
+
+ /// Memory system features.
+ pub(crate) MEM_FEATURES(u32) @ 0x10 {
+ 0:0 coherent_core_group => bool;
+ 1:1 coherent_super_group => bool;
+ 11:8 l2_slices;
+ }
+
+ /// Memory management unit features.
+ pub(crate) MMU_FEATURES(u32) @ 0x14 {
+ /// Number of bits supported in virtual addresses.
+ 7:0 va_bits;
+ /// Number of bits supported in physical addresses.
+ 15:8 pa_bits;
+ }
+
+ /// Address spaces present.
+ pub(crate) AS_PRESENT(u32) @ 0x18 {
+ 31:0 present;
+ }
+
+ /// CSF version information.
+ pub(crate) CSF_ID(u32) @ 0x1c {
+ /// MCU revision ID.
+ 3:0 mcu_rev;
+ /// MCU minor revision number.
+ 9:4 mcu_minor;
+ /// MCU major revision number.
+ 15:10 mcu_major;
+ /// CSHW revision ID.
+ 19:16 cshw_rev;
+ /// CSHW minor revision number.
+ 25:20 cshw_minor;
+ /// CSHW major revision number.
+ 31:26 cshw_major;
+ }
+
+ /// IRQ sources raw status.
+ /// Writing to this register forces bits on, but does not clear them.
+ pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 {
+ /// A GPU fault has occurred, a 1-bit boolean flag.
+ 0:0 gpu_fault => bool;
+ /// A GPU fault has occurred, a 1-bit boolean flag.
+ 1:1 gpu_protected_fault => bool;
+ /// Reset has completed, a 1-bit boolean flag.
+ 8:8 reset_completed => bool;
+ /// Set when a single power domain has powered up or down, a 1-bit boolean flag.
+ 9:9 power_changed_single => bool;
+ /// Set when the all pending power domain changes are completed, a 1-bit boolean flag.
+ 10:10 power_changed_all => bool;
+ /// Set when cache cleaning has completed, a 1-bit boolean flag.
+ 17:17 clean_caches_completed => bool;
+ /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag.
+ 18:18 doorbell_mirror => bool;
+ /// MCU requires attention, a 1-bit boolean flag.
+ 19:19 mcu_status => bool;
+ }
+
+ /// IRQ sources to clear. Write only.
+ pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 {
+ /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag.
+ 0:0 gpu_fault => bool;
+ /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag.
+ 1:1 gpu_protected_fault => bool;
+ /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag.
+ 8:8 reset_completed => bool;
+ /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag.
+ 9:9 power_changed_single => bool;
+ /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag.
+ 10:10 power_changed_all => bool;
+ /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag.
+ 17:17 clean_caches_completed => bool;
+ /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag.
+ 19:19 mcu_status => bool;
+ }
+
+ /// IRQ sources enabled.
+ pub(crate) GPU_IRQ_MASK(u32) @ 0x28 {
+ /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag.
+ 0:0 gpu_fault => bool;
+ /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag.
+ 1:1 gpu_protected_fault => bool;
+ /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag.
+ 8:8 reset_completed => bool;
+ /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag.
+ 9:9 power_changed_single => bool;
+ /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag.
+ 10:10 power_changed_all => bool;
+ /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag.
+ 17:17 clean_caches_completed => bool;
+ /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag.
+ 18:18 doorbell_mirror => bool;
+ /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag.
+ 19:19 mcu_status => bool;
+ }
+
+ /// IRQ status for enabled sources. Read only.
+ pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c {
+ /// GPU_FAULT interrupt status, a 1-bit boolean flag.
+ 0:0 gpu_fault => bool;
+ /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag.
+ 1:1 gpu_protected_fault => bool;
+ /// RESET_COMPLETED interrupt status, a 1-bit boolean flag.
+ 8:8 reset_completed => bool;
+ /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag.
+ 9:9 power_changed_single => bool;
+ /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag.
+ 10:10 power_changed_all => bool;
+ /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag.
+ 17:17 clean_caches_completed => bool;
+ /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag.
+ 18:18 doorbell_mirror => bool;
+ /// MCU_STATUS interrupt status, a 1-bit boolean flag.
+ 19:19 mcu_status => bool;
+ }
+ }
+
+ /// Helpers for GPU_COMMAND Register
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum GpuCommand {
+ /// No operation. This is the default value.
+ Nop = 0,
+ /// Reset the GPU.
+ Reset = 1,
+ /// Flush caches.
+ FlushCaches = 4,
+ /// Clear GPU faults.
+ ClearFault = 7,
+ }
+
+ impl TryFrom<Bounded<u32, 8>> for GpuCommand {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(GpuCommand::Nop),
+ 1 => Ok(GpuCommand::Reset),
+ 4 => Ok(GpuCommand::FlushCaches),
+ 7 => Ok(GpuCommand::ClearFault),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<GpuCommand> for Bounded<u32, 8> {
+ fn from(cmd: GpuCommand) -> Self {
+ (cmd as u8).into()
+ }
+ }
+
+ /// Reset mode for [`GPU_COMMAND::reset()`].
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum ResetMode {
+ /// Stop all external bus interfaces, then reset the entire GPU.
+ SoftReset = 1,
+ /// Force a full GPU reset.
+ HardReset = 2,
+ }
+
+ impl TryFrom<Bounded<u32, 4>> for ResetMode {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 1 => Ok(ResetMode::SoftReset),
+ 2 => Ok(ResetMode::HardReset),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<ResetMode> for Bounded<u32, 4> {
+ fn from(mode: ResetMode) -> Self {
+ Bounded::try_new(mode as u32).unwrap()
+ }
+ }
+
+ /// Cache flush mode for [`GPU_COMMAND::flush_caches()`].
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum FlushMode {
+ /// No flush.
+ None = 0,
+ /// Clean the caches.
+ Clean = 1,
+ /// Invalidate the caches.
+ Invalidate = 2,
+ /// Clean and invalidate the caches.
+ CleanInvalidate = 3,
+ }
+
+ impl TryFrom<Bounded<u32, 4>> for FlushMode {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(FlushMode::None),
+ 1 => Ok(FlushMode::Clean),
+ 2 => Ok(FlushMode::Invalidate),
+ 3 => Ok(FlushMode::CleanInvalidate),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<FlushMode> for Bounded<u32, 4> {
+ fn from(mode: FlushMode) -> Self {
+ Bounded::try_new(mode as u32).unwrap()
+ }
+ }
+
+ register! {
+ /// GPU command register.
+ ///
+ /// Use the constructor methods to create commands:
+ /// - [`GPU_COMMAND::nop()`]
+ /// - [`GPU_COMMAND::reset()`]
+ /// - [`GPU_COMMAND::flush_caches()`]
+ /// - [`GPU_COMMAND::clear_fault()`]
+ pub(crate) GPU_COMMAND (u32) @ 0x30 {
+ 7:0 command ?=> GpuCommand;
+ }
+ /// Internal alias for GPU_COMMAND in reset mode.
+ /// Use [`GPU_COMMAND::reset()`] instead.
+ GPU_COMMAND_RESET (u32) => GPU_COMMAND {
+ 7:0 command ?=> GpuCommand;
+ 11:8 reset_mode ?=> ResetMode;
+ }
+
+ /// Internal alias for GPU_COMMAND in cache flush mode.
+ /// Use [`GPU_COMMAND::flush_caches()`] instead.
+ GPU_COMMAND_FLUSH (u32) => GPU_COMMAND {
+ 7:0 command ?=> GpuCommand;
+ /// L2 cache flush mode.
+ 11:8 l2_flush ?=> FlushMode;
+ /// Shader core load/store cache flush mode.
+ 15:12 lsc_flush ?=> FlushMode;
+ /// Shader core other caches flush mode.
+ 19:16 other_flush ?=> FlushMode;
+ }
+ }
+
+ impl GPU_COMMAND {
+ /// Create a NOP command.
+ pub(crate) fn nop() -> Self {
+ Self::zeroed()
+ }
+
+ /// Create a reset command with the specified reset mode.
+ pub(crate) fn reset(mode: ResetMode) -> Self {
+ Self::from_raw(
+ GPU_COMMAND_RESET::zeroed()
+ .with_command(GpuCommand::Reset)
+ .with_reset_mode(mode)
+ .into_raw(),
+ )
+ }
+
+ /// Create a cache flush command with the specified flush modes.
+ pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self {
+ Self::from_raw(
+ GPU_COMMAND_FLUSH::zeroed()
+ .with_command(GpuCommand::FlushCaches)
+ .with_l2_flush(l2)
+ .with_lsc_flush(lsc)
+ .with_other_flush(other)
+ .into_raw(),
+ )
+ }
+
+ /// Create a clear fault command.
+ pub(crate) fn clear_fault() -> Self {
+ Self::zeroed().with_command(GpuCommand::ClearFault)
+ }
+ }
+
+ register! {
+ /// GPU status register. Read only.
+ pub(crate) GPU_STATUS(u32) @ 0x34 {
+ /// GPU active, a 1-bit boolean flag.
+ 0:0 gpu_active => bool;
+ /// Power manager active, a 1-bit boolean flag
+ 1:1 pwr_active => bool;
+ /// Page fault active, a 1-bit boolean flag.
+ 4:4 page_fault => bool;
+ /// Protected mode active, a 1-bit boolean flag.
+ 7:7 protected_mode_active => bool;
+ /// Debug mode active, a 1-bit boolean flag.
+ 8:8 gpu_dbg_enabled => bool;
+ }
+ }
+
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum ExceptionType {
+ /// Exception type: No error.
+ Ok = 0x00,
+ /// Exception type: GPU external bus error.
+ GpuBusFault = 0x80,
+ /// Exception type: GPU shareability error.
+ GpuShareabilityFault = 0x88,
+ /// Exception type: System shareability error.
+ SystemShareabilityFault = 0x89,
+ /// Exception type: GPU cacheability error.
+ GpuCacheabilityFault = 0x8A,
+ }
+
+ impl TryFrom<Bounded<u32, 8>> for ExceptionType {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0x00 => Ok(ExceptionType::Ok),
+ 0x80 => Ok(ExceptionType::GpuBusFault),
+ 0x88 => Ok(ExceptionType::GpuShareabilityFault),
+ 0x89 => Ok(ExceptionType::SystemShareabilityFault),
+ 0x8A => Ok(ExceptionType::GpuCacheabilityFault),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<ExceptionType> for Bounded<u32, 8> {
+ fn from(exc: ExceptionType) -> Self {
+ (exc as u8).into()
+ }
+ }
+
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum AccessType {
+ /// Access type: An atomic (read/write) transaction.
+ Atomic = 0,
+ /// Access type: An execute transaction.
+ Execute = 1,
+ /// Access type: A read transaction.
+ Read = 2,
+ /// Access type: A write transaction.
+ Write = 3,
+ }
+
+ impl From<Bounded<u32, 2>> for AccessType {
+ fn from(val: Bounded<u32, 2>) -> Self {
+ match val.get() {
+ 0 => AccessType::Atomic,
+ 1 => AccessType::Execute,
+ 2 => AccessType::Read,
+ 3 => AccessType::Write,
+ _ => unreachable!(),
+ }
+ }
+ }
+
+ impl From<AccessType> for Bounded<u32, 2> {
+ fn from(access: AccessType) -> Self {
+ Bounded::try_new(access as u32).unwrap()
+ }
+ }
+
+ register! {
+ /// GPU fault status register. Read only.
+ pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c {
+ /// Exception type.
+ 7:0 exception_type ?=> ExceptionType;
+ /// Access type.
+ 9:8 access_type => AccessType;
+ /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag.
+ 10:10 address_valid => bool;
+ /// The JASID field is valid, a 1-bit boolean flag.
+ 11:11 jasid_valid => bool;
+ /// JASID of the fault, if known.
+ 15:12 jasid;
+ /// ID of the source that triggered the fault.
+ 31:16 source_id;
+ }
+
+ /// GPU fault address. Read only.
+ /// Once a fault is reported, it must be manually cleared by issuing a
+ /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU
+ /// faults will be reported until the previous fault has been cleared.
+ pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 {
+ 31:0 pointer;
+ }
+
+ pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 {
+ 31:0 pointer;
+ }
+
+ /// Level 2 cache configuration.
+ pub(crate) L2_CONFIG(u32) @ 0x48 {
+ /// Requested cache size.
+ 23:16 cache_size;
+ /// Requested hash function index.
+ 31:24 hash_function;
+ }
+
+ /// Global time stamp offset.
+ pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 {
+ 31:0 offset;
+ }
+
+ pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c {
+ 31:0 offset;
+ }
+
+ /// GPU cycle counter. Read only.
+ pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 {
+ 31:0 count;
+ }
+
+ pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 {
+ 31:0 count;
+ }
+
+ /// Global time stamp. Read only.
+ pub(crate) TIMESTAMP_LO(u32) @ 0x98 {
+ 31:0 timestamp;
+ }
+
+ pub(crate) TIMESTAMP_HI(u32) @ 0x9c {
+ 31:0 timestamp;
+ }
+
+ /// Maximum number of threads per core. Read only constant.
+ pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 {
+ 31:0 threads;
+ }
+
+ /// Maximum number of threads per workgroup. Read only constant.
+ pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 {
+ 31:0 threads;
+ }
+
+ /// Maximum number of threads per barrier. Read only constant.
+ pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 {
+ 31:0 threads;
+ }
+
+ /// Thread features. Read only constant.
+ pub(crate) THREAD_FEATURES(u32) @ 0xac {
+ /// Total number of registers per core.
+ 21:0 max_registers;
+ /// Implementation technology type.
+ 23:22 implementation_technology;
+ /// Maximum number of compute tasks waiting.
+ 31:24 max_task_queue;
+ }
+
+ /// Support flags for compressed texture formats. Read only constant.
+ ///
+ /// A bitmap where each bit indicates support for a specific compressed texture format.
+ /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec).
+ pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 {
+ 31:0 supported_formats;
+ }
+
+ /// Shader core present bitmap. Read only constant.
+ pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 {
+ 31:0 value;
+ }
+
+ pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 {
+ 31:0 value;
+ }
+
+ /// Tiler present bitmap. Read only constant.
+ pub(crate) TILER_PRESENT_LO(u32) @ 0x110 {
+ 31:0 present;
+ }
+
+ pub(crate) TILER_PRESENT_HI(u32) @ 0x114 {
+ 31:0 present;
+ }
+
+ /// L2 cache present bitmap. Read only constant.
+ pub(crate) L2_PRESENT_LO(u32) @ 0x120 {
+ 31:0 present;
+ }
+
+ pub(crate) L2_PRESENT_HI(u32) @ 0x124 {
+ 31:0 present;
+ }
+
+ /// Shader core ready bitmap. Read only.
+ pub(crate) SHADER_READY_LO(u32) @ 0x140 {
+ 31:0 ready;
+ }
+
+ pub(crate) SHADER_READY_HI(u32) @ 0x144 {
+ 31:0 ready;
+ }
+
+ /// Tiler ready bitmap. Read only.
+ pub(crate) TILER_READY_LO(u32) @ 0x150 {
+ 31:0 ready;
+ }
+
+ pub(crate) TILER_READY_HI(u32) @ 0x154 {
+ 31:0 ready;
+ }
+
+ /// L2 ready bitmap. Read only.
+ pub(crate) L2_READY_LO(u32) @ 0x160 {
+ 31:0 ready;
+ }
+
+ pub(crate) L2_READY_HI(u32) @ 0x164 {
+ 31:0 ready;
+ }
+
+ /// Shader core power up bitmap.
+ pub(crate) SHADER_PWRON_LO(u32) @ 0x180 {
+ 31:0 request;
+ }
+
+ pub(crate) SHADER_PWRON_HI(u32) @ 0x184 {
+ 31:0 request;
+ }
+
+ /// Tiler power up bitmap.
+ pub(crate) TILER_PWRON_LO(u32) @ 0x190 {
+ 31:0 request;
+ }
+
+ pub(crate) TILER_PWRON_HI(u32) @ 0x194 {
+ 31:0 request;
+ }
+
+ /// L2 power up bitmap.
+ pub(crate) L2_PWRON_LO(u32) @ 0x1a0 {
+ 31:0 request;
+ }
+
+ pub(crate) L2_PWRON_HI(u32) @ 0x1a4 {
+ 31:0 request;
+ }
+
+ /// Shader core power down bitmap.
+ pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 {
+ 31:0 request;
+ }
+
+ pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 {
+ 31:0 request;
+ }
+
+ /// Tiler power down bitmap.
+ pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 {
+ 31:0 request;
+ }
+
+ pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 {
+ 31:0 request;
+ }
+
+ /// L2 power down bitmap.
+ pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 {
+ 31:0 request;
+ }
+
+ pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 {
+ 31:0 request;
+ }
+
+ /// Shader core power transition bitmap. Read-only.
+ pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 {
+ 31:0 changing;
+ }
+
+ pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 {
+ 31:0 changing;
+ }
+
+ /// Tiler power transition bitmap. Read-only.
+ pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 {
+ 31:0 changing;
+ }
+
+ pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 {
+ 31:0 changing;
+ }
+
+ /// L2 power transition bitmap. Read-only.
+ pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 {
+ 31:0 changing;
+ }
+
+ pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 {
+ 31:0 changing;
+ }
+
+ /// Shader core active bitmap. Read-only.
+ pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 {
+ 31:0 active;
+ }
+
+ pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 {
+ 31:0 active;
+ }
+
+ /// Tiler active bitmap. Read-only.
+ pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 {
+ 31:0 active;
+ }
+
+ pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 {
+ 31:0 active;
+ }
+
+ /// L2 active bitmap. Read-only.
+ pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 {
+ 31:0 active;
+ }
+
+ pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 {
+ 31:0 active;
+ }
+
+ /// Revision ID. Read only constant.
+ pub(crate) REVIDR(u32) @ 0x280 {
+ 31:0 revision;
+ }
+
+ /// Coherency features present. Read only constant.
+ /// Supported protocols on the interconnect between the GPU and the
+ /// system into which it is integrated.
+ pub(crate) COHERENCY_FEATURES(u32) @ 0x300 {
+ /// ACE-Lite protocol supported, a 1-bit boolean flag.
+ 0:0 ace_lite => bool;
+ /// ACE protocol supported, a 1-bit boolean flag.
+ 1:1 ace => bool;
+ }
+ }
+
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum CoherencyMode {
+ /// ACE-Lite coherency protocol.
+ AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8,
+ /// ACE coherency protocol.
+ Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8,
+ /// No coherency protocol.
+ None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8,
+ }
+
+ impl TryFrom<Bounded<u32, 32>> for CoherencyMode {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 32>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(CoherencyMode::AceLite),
+ 1 => Ok(CoherencyMode::Ace),
+ 31 => Ok(CoherencyMode::None),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<CoherencyMode> for Bounded<u32, 32> {
+ fn from(mode: CoherencyMode) -> Self {
+ (mode as u8).into()
+ }
+ }
+
+ register! {
+ /// Coherency enable. An index of which coherency protocols should be used.
+ /// This register only selects the protocol for coherency messages on the
+ /// interconnect. This is not to enable or disable coherency controlled by MMU.
+ pub(crate) COHERENCY_ENABLE(u32) @ 0x304 {
+ 31:0 l2_cache_protocol_select ?=> CoherencyMode;
+ }
+ }
+
+ /// Helpers for MCU_CONTROL register
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum McuControlMode {
+ /// Disable the MCU.
+ Disable = 0,
+ /// Enable the MCU.
+ Enable = 1,
+ /// Enable the MCU to execute and automatically reboot after a fast reset.
+ Auto = 2,
+ }
+
+ impl TryFrom<Bounded<u32, 2>> for McuControlMode {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 2>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(McuControlMode::Disable),
+ 1 => Ok(McuControlMode::Enable),
+ 2 => Ok(McuControlMode::Auto),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<McuControlMode> for Bounded<u32, 2> {
+ fn from(mode: McuControlMode) -> Self {
+ Bounded::try_new(mode as u32).unwrap()
+ }
+ }
+
+ register! {
+ /// MCU control.
+ pub(crate) MCU_CONTROL(u32) @ 0x700 {
+ /// Request MCU state change.
+ 1:0 req ?=> McuControlMode;
+ }
+ }
+
+ /// Helpers for MCU_STATUS register
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum McuStatus {
+ /// MCU is disabled.
+ Disabled = 0,
+ /// MCU is enabled.
+ Enabled = 1,
+ /// The MCU has halted by itself in an orderly manner to enable the core group to be
+ /// powered down.
+ Halt = 2,
+ /// The MCU has encountered an error that prevents it from continuing.
+ Fatal = 3,
+ }
+
+ impl From<Bounded<u32, 2>> for McuStatus {
+ fn from(val: Bounded<u32, 2>) -> Self {
+ match val.get() {
+ 0 => McuStatus::Disabled,
+ 1 => McuStatus::Enabled,
+ 2 => McuStatus::Halt,
+ 3 => McuStatus::Fatal,
+ _ => unreachable!(),
+ }
+ }
+ }
+
+ impl From<McuStatus> for Bounded<u32, 2> {
+ fn from(status: McuStatus) -> Self {
+ Bounded::try_new(status as u32).unwrap()
+ }
+ }
+
+ register! {
+ /// MCU status. Read only.
+ pub(crate) MCU_STATUS(u32) @ 0x704 {
+ /// Read current state of MCU.
+ 1:0 value => McuStatus;
+ }
+ }
+}
+
+/// These registers correspond to the JOB_CONTROL register page.
+/// They are involved in communication between the firmware running on the MCU and the host.
+pub(crate) mod job_control {
+ use kernel::register;
+
+ register! {
+ /// Raw status of job interrupts.
+ ///
+ /// Write to this register to trigger these interrupts.
+ /// Writing a 1 to a bit forces that bit on.
+ pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 {
+ /// CSG request. These bits indicate that CSGn requires attention from the host.
+ 30:0 csg;
+ /// GLB request. Indicates that the GLB interface requires attention from the host.
+ 31:31 glb => bool;
+ }
+
+ /// Clear job interrupts. Write only.
+ ///
+ /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`].
+ pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 {
+ /// Clear CSG request interrupts.
+ 30:0 csg;
+ /// Clear GLB request interrupt.
+ 31:31 glb => bool;
+ }
+
+ /// Mask for job interrupts.
+ ///
+ /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it.
+ pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 {
+ /// Enable CSG request interrupts.
+ 30:0 csg;
+ /// Enable GLB request interrupt.
+ 31:31 glb => bool;
+ }
+
+ /// Active job interrupts. Read only.
+ ///
+ /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and
+ /// [`JOB_IRQ_MASK`].
+ pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c {
+ /// CSG request interrupt status.
+ 30:0 csg;
+ /// GLB request interrupt status.
+ 31:31 glb => bool;
+ }
+ }
+}
+
+/// These registers correspond to the MMU_CONTROL register page.
+/// They are involved in MMU configuration and control.
+pub(crate) mod mmu_control {
+ use kernel::register;
+
+ register! {
+ /// IRQ sources raw status.
+ ///
+ /// This register contains the raw unmasked interrupt sources for MMU status and exception
+ /// handling.
+ ///
+ /// Writing to this register forces bits on.
+ /// Use [`IRQ_CLEAR`] to clear interrupts.
+ pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 {
+ /// Page fault for address spaces.
+ 15:0 page_fault;
+ /// Command completed in address spaces.
+ 31:16 command_completed;
+ }
+
+ /// IRQ sources to clear.
+ /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`].
+ pub(crate) IRQ_CLEAR(u32) @ 0x2004 {
+ /// Clear the PAGE_FAULT interrupt.
+ 15:0 page_fault;
+ /// Clear the COMMAND_COMPLETED interrupt.
+ 31:16 command_completed;
+ }
+
+ /// IRQ sources enabled.
+ ///
+ /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it.
+ pub(crate) IRQ_MASK(u32) @ 0x2008 {
+ /// Enable the PAGE_FAULT interrupt.
+ 15:0 page_fault;
+ /// Enable the COMMAND_COMPLETED interrupt.
+ 31:16 command_completed;
+ }
+
+ /// IRQ status for enabled sources. Read only.
+ ///
+ /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`].
+ pub(crate) IRQ_STATUS(u32) @ 0x200c {
+ /// PAGE_FAULT interrupt status.
+ 15:0 page_fault;
+ /// COMMAND_COMPLETED interrupt status.
+ 31:16 command_completed;
+ }
+ }
+
+ /// Per-address space registers ASn [0..15] within the MMU_CONTROL page.
+ ///
+ /// This array contains 16 instances of the MMU_AS_CONTROL register page.
+ pub(crate) mod mmu_as_control {
+ use kernel::{
+ num::Bounded,
+ prelude::*,
+ register, //
+ };
+
+ use pin_init::Zeroable;
+
+ /// Maximum number of hardware address space slots.
+ /// The actual number of slots available is usually lower.
+ pub(crate) const MAX_AS: usize = 16;
+
+ /// Address space register stride. The elements in the array are spaced 64B apart.
+ const STRIDE: usize = 0x40;
+
+ register! {
+ /// Translation table base address. A 64-bit pointer.
+ ///
+ /// This field contains the address of the top level of a translation table structure.
+ /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero.
+ pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 {
+ /// Base address of the translation table.
+ 63:0 base;
+ }
+
+ // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two
+ // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+ // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+ pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 {
+ 31:0 value;
+ }
+
+ pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 {
+ 31:0 value;
+ }
+ }
+
+ /// Helpers for MEMATTR Register.
+
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum AllocPolicySelect {
+ /// Ignore ALLOC_R/ALLOC_W fields.
+ Impl = 2,
+ /// Use ALLOC_R/ALLOC_W fields for allocation policy.
+ Alloc = 3,
+ }
+
+ impl TryFrom<Bounded<u8, 2>> for AllocPolicySelect {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 2 => Ok(Self::Impl),
+ 3 => Ok(Self::Alloc),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<AllocPolicySelect> for Bounded<u8, 2> {
+ fn from(val: AllocPolicySelect) -> Self {
+ Bounded::try_new(val as u8).unwrap()
+ }
+ }
+
+ /// Coherency policy for memory attributes. Indicates the shareability of cached accesses.
+ ///
+ /// The hardware spec defines different interpretations of these values depending on
+ /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation
+ /// tables (all input addresses map to the same output address); it is deprecated and not
+ /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum Coherency {
+ /// Midgard inner domain coherency.
+ ///
+ /// Most flexible mode - can map non-coherent, internally coherent, and system/IO
+ /// coherent memory. Used for non-cacheable memory in MAIR conversion.
+ MidgardInnerDomain = 0,
+ /// CPU inner domain coherency.
+ ///
+ /// Can map non-coherent and system/IO coherent memory. Used for write-back
+ /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency.
+ CpuInnerDomain = 1,
+ /// CPU inner domain with shader coherency.
+ ///
+ /// Can map internally coherent and system/IO coherent memory. Used for
+ /// GPU-internal shared buffers requiring shader coherency.
+ CpuInnerDomainShaderCoh = 2,
+ }
+
+ impl TryFrom<Bounded<u8, 2>> for Coherency {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(Self::MidgardInnerDomain),
+ 1 => Ok(Self::CpuInnerDomain),
+ 2 => Ok(Self::CpuInnerDomainShaderCoh),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<Coherency> for Bounded<u8, 2> {
+ fn from(val: Coherency) -> Self {
+ Bounded::try_new(val as u8).unwrap()
+ }
+ }
+
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum MemoryType {
+ /// Normal memory (shared).
+ Shared = 0,
+ /// Normal memory, inner/outer non-cacheable.
+ NonCacheable = 1,
+ /// Normal memory, inner/outer write-back cacheable.
+ WriteBack = 2,
+ /// Triggers MEMORY_ATTRIBUTE_FAULT.
+ Fault = 3,
+ }
+
+ impl From<Bounded<u8, 2>> for MemoryType {
+ fn from(val: Bounded<u8, 2>) -> Self {
+ match val.get() {
+ 0 => Self::Shared,
+ 1 => Self::NonCacheable,
+ 2 => Self::WriteBack,
+ 3 => Self::Fault,
+ _ => unreachable!(),
+ }
+ }
+ }
+
+ impl From<MemoryType> for Bounded<u8, 2> {
+ fn from(val: MemoryType) -> Self {
+ Bounded::try_new(val as u8).unwrap()
+ }
+ }
+
+ register! {
+ /// Stage 1 memory attributes (8-bit bitfield).
+ ///
+ /// This is not an actual register, but a bitfield definition used by the MEMATTR
+ /// register. Each of the 8 bytes in MEMATTR follows this layout.
+ MMU_MEMATTR_STAGE1(u8) @ 0x0 {
+ /// Inner cache write allocation policy.
+ 0:0 alloc_w => bool;
+ /// Inner cache read allocation policy.
+ 1:1 alloc_r => bool;
+ /// Inner allocation policy select.
+ 3:2 alloc_sel ?=> AllocPolicySelect;
+ /// Coherency policy.
+ 5:4 coherency ?=> Coherency;
+ /// Memory type.
+ 7:6 memory_type => MemoryType;
+ }
+ }
+
+ impl TryFrom<Bounded<u64, 8>> for MMU_MEMATTR_STAGE1 {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u64, 8>) -> Result<Self, Self::Error> {
+ Ok(Self::from_raw(val.get() as u8))
+ }
+ }
+
+ impl From<MMU_MEMATTR_STAGE1> for Bounded<u64, 8> {
+ fn from(val: MMU_MEMATTR_STAGE1) -> Self {
+ Bounded::try_new(u64::from(val.into_raw())).unwrap()
+ }
+ }
+
+ register! {
+ /// Memory attributes.
+ ///
+ /// Each address space can configure up to 8 different memory attribute profiles.
+ /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout.
+ pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 {
+ 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1;
+ 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1;
+ 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1;
+ 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1;
+ 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1;
+ 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1;
+ 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1;
+ 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1;
+ }
+
+ // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two
+ // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+ // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+ pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 {
+ 31:0 value;
+ }
+
+ pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c {
+ 31:0 value;
+ }
+ }
+
+ impl MEMATTR {
+ /// Outer cache-policy nibble indicating device memory.
+ const ARM_MAIR_DEVICE_MEMORY: u8 = 0x0;
+
+ /// In the ARM Architecture Reference Manual, the MAIR encoding for Normal memory
+ /// uses the format `0bxxRW` where:
+ /// - `W` (bit 0) = Write-Allocate policy
+ /// - `R` (bit 1) = Read-Allocate policy
+ /// E.g., `0b0011` would allow both read and write allocation on a cache miss.
+ ///
+ /// ARM MAIR Write-Allocate bit (bit 0 of a cache policy nibble).
+ const ARM_MAIR_WRITE_ALLOCATE: u8 = 0x1;
+ /// ARM MAIR Read-Allocate bit (bit 1 of a cache policy nibble).
+ const ARM_MAIR_READ_ALLOCATE: u8 = 0x2;
+
+ /// Write-back policy bit. For cacheable encodings, it is necessary but not
+ /// sufficient to set bit 2 of the cache policy nibble. Bit 2 does not
+ /// definitively determine write back because bit 2 is also set in `0b0100`
+ /// which encodes Normal non-cacheable memory.
+ const ARM_MAIR_WRITE_BACK_BIT: u8 = 0x4;
+
+ /// Complete cache-policy nibble encoding for Normal Non-cacheable memory.
+ const ARM_MAIR_NON_CACHEABLE: u8 = 0x4;
+
+ /// Mask for the inner cache policy nibble in MAIR attribute bytes.
+ const ARM_MAIR_INNER_MASK: u8 = 0x0f;
+
+ /// Check if a MAIR attribute byte represents device memory.
+ ///
+ /// Device memory (memory-mapped I/O, registers) cannot be cached because
+ /// reading and writing to this memory may have side effects.
+ fn is_device_memory(mair_attr: u8) -> bool {
+ // In AArch64 MAIR, outer nibble only is 0 for device memory.
+ (mair_attr >> 4) == Self::ARM_MAIR_DEVICE_MEMORY
+ }
+
+ /// Check if normal memory is fully write-back cacheable.
+ ///
+ /// ARM MAIR has two cache policy levels (outer [7:4] and inner [3:0]).
+ /// For memory to be truly write-back, BOTH levels must have the write-back bit set.
+ /// If only one level is write-back, treat it as non-cacheable for GPU purposes.
+ fn is_writeback_cacheable(mair_attr: u8) -> bool {
+ let outer = mair_attr >> 4;
+ let inner = mair_attr & Self::ARM_MAIR_INNER_MASK;
+
+ outer != Self::ARM_MAIR_NON_CACHEABLE
+ && inner != Self::ARM_MAIR_NON_CACHEABLE
+ && (outer & Self::ARM_MAIR_WRITE_BACK_BIT) != 0
+ && (inner & Self::ARM_MAIR_WRITE_BACK_BIT) != 0
+ }
+
+ // Helper to encode a MEMATTR attribute from its individual fields.
+ fn encode_attribute(
+ alloc_w: bool,
+ alloc_r: bool,
+ alloc_sel: AllocPolicySelect,
+ coherency: Coherency,
+ memory_type: MemoryType,
+ ) -> MMU_MEMATTR_STAGE1 {
+ MMU_MEMATTR_STAGE1::zeroed()
+ .with_alloc_w(alloc_w)
+ .with_alloc_r(alloc_r)
+ .with_alloc_sel(alloc_sel)
+ .with_coherency(coherency)
+ .with_memory_type(memory_type)
+ }
+
+ /// Convert one MAIR attribute byte into a MEMATTR attribute.
+ // TODO: Add a `coherent` parameter like panthor's mair_to_memattr().
+ // For now, assume a non-coherent system and always encode write-back
+ // memory with MidgardInnerDomain coherency.
+ fn attribute_from_mair(mair_attr: u8) -> MMU_MEMATTR_STAGE1 {
+ // Device memory or non-write-back normal memory
+ if Self::is_device_memory(mair_attr) || !Self::is_writeback_cacheable(mair_attr) {
+ return Self::encode_attribute(
+ false,
+ false,
+ AllocPolicySelect::Alloc,
+ Coherency::MidgardInnerDomain,
+ MemoryType::NonCacheable,
+ );
+ }
+
+ // Write-back cacheable normal memory
+ let inner: u8 = mair_attr & Self::ARM_MAIR_INNER_MASK;
+ Self::encode_attribute(
+ (inner & Self::ARM_MAIR_WRITE_ALLOCATE) != 0,
+ (inner & Self::ARM_MAIR_READ_ALLOCATE) != 0,
+ AllocPolicySelect::Alloc,
+ Coherency::MidgardInnerDomain,
+ MemoryType::WriteBack,
+ )
+ }
+
+ /// Write one converted MAIR attribute into a corresponding MEMATTR slot.
+ fn with_encoded_attribute(self, index: usize, attr: MMU_MEMATTR_STAGE1) -> Self {
+ debug_assert!(index < 8);
+
+ let shift = index * 8;
+ let mask = !(0xffu64 << shift);
+ let raw = (self.into_raw() & mask) | ((u64::from(attr.into_raw())) << shift);
+
+ Self::from_raw(raw)
+ }
+
+ /// Convert an AArch64 MAIR value into the GPU MEMATTR register encoding.
+ ///
+ /// Both MAIR and MEMATTR are 64-bit values with eight 8-bit memory
+ /// attribute entries, but the bits do not map directly. The GPU MEMATTR encoding
+ /// is less detailed than the MAIR encoding, so MAIR is converted to MEMATTR
+ /// conservatively as follows:
+ ///
+ /// 1. Device memory, or Normal Memory that is not write-back cacheable, is encoded
+ /// as GPU `NonCacheable`
+ ///
+ /// 2. Normal memory that is write-back cacheable is encoded as GPU `WriteBack`,
+ /// and the inner allocation hints are preserved.
+ pub(crate) fn from_mair(mair: u64) -> Self {
+ mair.to_le_bytes()
+ .into_iter()
+ .enumerate()
+ .fold(Self::zeroed(), |acc, (i, attr)| {
+ acc.with_encoded_attribute(i, Self::attribute_from_mair(attr))
+ })
+ }
+ }
+
+ register! {
+ /// Lock region address for each address space.
+ pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 {
+ /// Lock region size.
+ 5:0 size;
+ /// Lock region base address.
+ 63:12 base;
+ }
+
+ // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two
+ // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+ // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+ pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 {
+ 31:0 value;
+ }
+
+ pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 {
+ 31:0 value;
+ }
+ }
+
+ /// Helpers for MMU COMMAND register.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum MmuCommand {
+ /// No operation, nothing happens.
+ Nop = 0,
+ /// Propagate settings to the MMU.
+ Update = 1,
+ /// Lock an address region.
+ Lock = 2,
+ /// Unlock an address region.
+ Unlock = 3,
+ /// Clean and invalidate the L2 cache, then unlock.
+ FlushPt = 4,
+ /// Clean and invalidate all caches, then unlock.
+ FlushMem = 5,
+ }
+
+ impl TryFrom<Bounded<u32, 8>> for MmuCommand {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(MmuCommand::Nop),
+ 1 => Ok(MmuCommand::Update),
+ 2 => Ok(MmuCommand::Lock),
+ 3 => Ok(MmuCommand::Unlock),
+ 4 => Ok(MmuCommand::FlushPt),
+ 5 => Ok(MmuCommand::FlushMem),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<MmuCommand> for Bounded<u32, 8> {
+ fn from(cmd: MmuCommand) -> Self {
+ (cmd as u8).into()
+ }
+ }
+
+ register! {
+ /// MMU command register for each address space. Write only.
+ pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 {
+ 7:0 command ?=> MmuCommand;
+ }
+ }
+
+ /// MMU exception types for FAULTSTATUS register.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum MmuExceptionType {
+ /// No error.
+ Ok = 0x00,
+ /// Invalid translation table entry, level 0.
+ TranslationFault0 = 0xC0,
+ /// Invalid translation table entry, level 1.
+ TranslationFault1 = 0xC1,
+ /// Invalid translation table entry, level 2.
+ TranslationFault2 = 0xC2,
+ /// Invalid translation table entry, level 3.
+ TranslationFault3 = 0xC3,
+ /// Invalid block descriptor.
+ TranslationFault4 = 0xC4,
+ /// Page permission error, level 0.
+ PermissionFault0 = 0xC8,
+ /// Page permission error, level 1.
+ PermissionFault1 = 0xC9,
+ /// Page permission error, level 2.
+ PermissionFault2 = 0xCA,
+ /// Page permission error, level 3.
+ PermissionFault3 = 0xCB,
+ /// Access flag not set, level 1.
+ AccessFlag1 = 0xD9,
+ /// Access flag not set, level 2.
+ AccessFlag2 = 0xDA,
+ /// Access flag not set, level 3.
+ AccessFlag3 = 0xDB,
+ /// Virtual address out of range.
+ AddressSizeFaultIn = 0xE0,
+ /// Physical address out of range, level 0.
+ AddressSizeFaultOut0 = 0xE4,
+ /// Physical address out of range, level 1.
+ AddressSizeFaultOut1 = 0xE5,
+ /// Physical address out of range, level 2.
+ AddressSizeFaultOut2 = 0xE6,
+ /// Physical address out of range, level 3.
+ AddressSizeFaultOut3 = 0xE7,
+ /// Page attribute error, level 0.
+ MemoryAttributeFault0 = 0xE8,
+ /// Page attribute error, level 1.
+ MemoryAttributeFault1 = 0xE9,
+ /// Page attribute error, level 2.
+ MemoryAttributeFault2 = 0xEA,
+ /// Page attribute error, level 3.
+ MemoryAttributeFault3 = 0xEB,
+ }
+
+ impl TryFrom<Bounded<u32, 8>> for MmuExceptionType {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0x00 => Ok(MmuExceptionType::Ok),
+ 0xC0 => Ok(MmuExceptionType::TranslationFault0),
+ 0xC1 => Ok(MmuExceptionType::TranslationFault1),
+ 0xC2 => Ok(MmuExceptionType::TranslationFault2),
+ 0xC3 => Ok(MmuExceptionType::TranslationFault3),
+ 0xC4 => Ok(MmuExceptionType::TranslationFault4),
+ 0xC8 => Ok(MmuExceptionType::PermissionFault0),
+ 0xC9 => Ok(MmuExceptionType::PermissionFault1),
+ 0xCA => Ok(MmuExceptionType::PermissionFault2),
+ 0xCB => Ok(MmuExceptionType::PermissionFault3),
+ 0xD9 => Ok(MmuExceptionType::AccessFlag1),
+ 0xDA => Ok(MmuExceptionType::AccessFlag2),
+ 0xDB => Ok(MmuExceptionType::AccessFlag3),
+ 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn),
+ 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0),
+ 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1),
+ 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2),
+ 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3),
+ 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0),
+ 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1),
+ 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2),
+ 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<MmuExceptionType> for Bounded<u32, 8> {
+ fn from(exc: MmuExceptionType) -> Self {
+ (exc as u8).into()
+ }
+ }
+
+ /// Access type for MMU faults.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum MmuAccessType {
+ /// An atomic (read/write) transaction.
+ Atomic = 0,
+ /// An execute transaction.
+ Execute = 1,
+ /// A read transaction.
+ Read = 2,
+ /// A write transaction.
+ Write = 3,
+ }
+
+ impl From<Bounded<u32, 2>> for MmuAccessType {
+ fn from(val: Bounded<u32, 2>) -> Self {
+ match val.get() {
+ 0 => MmuAccessType::Atomic,
+ 1 => MmuAccessType::Execute,
+ 2 => MmuAccessType::Read,
+ 3 => MmuAccessType::Write,
+ _ => unreachable!(),
+ }
+ }
+ }
+
+ impl From<MmuAccessType> for Bounded<u32, 2> {
+ fn from(access: MmuAccessType) -> Self {
+ Bounded::try_new(access as u32).unwrap()
+ }
+ }
+
+ register! {
+ /// Fault status register for each address space. Read only.
+ pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c {
+ /// Exception type.
+ 7:0 exception_type ?=> MmuExceptionType;
+ /// Access type.
+ 9:8 access_type => MmuAccessType;
+ /// ID of the source that triggered the fault.
+ 31:16 source_id;
+ }
+
+ /// Fault address for each address space. Read only.
+ pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 {
+ 31:0 pointer;
+ }
+
+ pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 {
+ 31:0 pointer;
+ }
+
+ /// MMU status register for each address space. Read only.
+ pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 {
+ /// External address space command is active, a 1-bit boolean flag.
+ 0:0 active_ext => bool;
+ /// Internal address space command is active, a 1-bit boolean flag.
+ 1:1 active_int => bool;
+ }
+ }
+
+ /// Helpers for TRANSCFG register.
+ ///
+ /// Address space mode for TRANSCFG register.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum AddressSpaceMode {
+ /// The MMU forces all memory access to fail with a decode fault.
+ Unmapped = 1,
+ /// All input addresses map to the same output address (deprecated).
+ Identity = 2,
+ /// Translation tables interpreted according to AArch64 4kB granule specification.
+ Aarch64_4K = 6,
+ /// Translation tables interpreted according to AArch64 64kB granule specification.
+ Aarch64_64K = 8,
+ }
+
+ impl TryFrom<Bounded<u64, 4>> for AddressSpaceMode {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u64, 4>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 1 => Ok(AddressSpaceMode::Unmapped),
+ 2 => Ok(AddressSpaceMode::Identity),
+ 6 => Ok(AddressSpaceMode::Aarch64_4K),
+ 8 => Ok(AddressSpaceMode::Aarch64_64K),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<AddressSpaceMode> for Bounded<u64, 4> {
+ fn from(mode: AddressSpaceMode) -> Self {
+ Bounded::try_new(mode as u64).unwrap()
+ }
+ }
+
+ /// Input address range restriction for TRANSCFG register.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum InaBits {
+ /// Invalid VA range (reset value).
+ Reset = 0,
+ /// 48-bit VA range.
+ Bits48 = 7,
+ /// 47-bit VA range.
+ Bits47 = 8,
+ /// 46-bit VA range.
+ Bits46 = 9,
+ /// 45-bit VA range.
+ Bits45 = 10,
+ /// 44-bit VA range.
+ Bits44 = 11,
+ /// 43-bit VA range.
+ Bits43 = 12,
+ /// 42-bit VA range.
+ Bits42 = 13,
+ /// 41-bit VA range.
+ Bits41 = 14,
+ /// 40-bit VA range.
+ Bits40 = 15,
+ /// 39-bit VA range.
+ Bits39 = 16,
+ /// 38-bit VA range.
+ Bits38 = 17,
+ /// 37-bit VA range.
+ Bits37 = 18,
+ /// 36-bit VA range.
+ Bits36 = 19,
+ /// 35-bit VA range.
+ Bits35 = 20,
+ /// 34-bit VA range.
+ Bits34 = 21,
+ /// 33-bit VA range.
+ Bits33 = 22,
+ /// 32-bit VA range.
+ Bits32 = 23,
+ /// 31-bit VA range.
+ Bits31 = 24,
+ /// 30-bit VA range.
+ Bits30 = 25,
+ /// 29-bit VA range.
+ Bits29 = 26,
+ /// 28-bit VA range.
+ Bits28 = 27,
+ /// 27-bit VA range.
+ Bits27 = 28,
+ /// 26-bit VA range.
+ Bits26 = 29,
+ /// 25-bit VA range.
+ Bits25 = 30,
+ }
+
+ impl TryFrom<Bounded<u64, 5>> for InaBits {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u64, 5>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(InaBits::Reset),
+ 7 => Ok(InaBits::Bits48),
+ 8 => Ok(InaBits::Bits47),
+ 9 => Ok(InaBits::Bits46),
+ 10 => Ok(InaBits::Bits45),
+ 11 => Ok(InaBits::Bits44),
+ 12 => Ok(InaBits::Bits43),
+ 13 => Ok(InaBits::Bits42),
+ 14 => Ok(InaBits::Bits41),
+ 15 => Ok(InaBits::Bits40),
+ 16 => Ok(InaBits::Bits39),
+ 17 => Ok(InaBits::Bits38),
+ 18 => Ok(InaBits::Bits37),
+ 19 => Ok(InaBits::Bits36),
+ 20 => Ok(InaBits::Bits35),
+ 21 => Ok(InaBits::Bits34),
+ 22 => Ok(InaBits::Bits33),
+ 23 => Ok(InaBits::Bits32),
+ 24 => Ok(InaBits::Bits31),
+ 25 => Ok(InaBits::Bits30),
+ 26 => Ok(InaBits::Bits29),
+ 27 => Ok(InaBits::Bits28),
+ 28 => Ok(InaBits::Bits27),
+ 29 => Ok(InaBits::Bits26),
+ 30 => Ok(InaBits::Bits25),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<InaBits> for Bounded<u64, 5> {
+ fn from(bits: InaBits) -> Self {
+ Bounded::try_new(bits as u64).unwrap()
+ }
+ }
+
+ /// Translation table memory attributes for TRANSCFG register.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ pub(crate) enum PtwMemattr {
+ /// Invalid (reset value, not valid for enabled address space).
+ Invalid = 0,
+ /// Normal memory, inner/outer non-cacheable.
+ NonCacheable = 1,
+ /// Normal memory, inner/outer write-back cacheable.
+ WriteBack = 2,
+ }
+
+ impl TryFrom<Bounded<u64, 2>> for PtwMemattr {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(PtwMemattr::Invalid),
+ 1 => Ok(PtwMemattr::NonCacheable),
+ 2 => Ok(PtwMemattr::WriteBack),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<PtwMemattr> for Bounded<u64, 2> {
+ fn from(attr: PtwMemattr) -> Self {
+ Bounded::try_new(attr as u64).unwrap()
+ }
+ }
+
+ /// Translation table memory shareability for TRANSCFG register.
+ #[derive(Copy, Clone, Debug, PartialEq)]
+ #[repr(u8)]
+ #[allow(clippy::enum_variant_names)]
+ pub(crate) enum PtwShareability {
+ /// Non-shareable.
+ NonShareable = 0,
+ /// Outer shareable.
+ OuterShareable = 2,
+ /// Inner shareable.
+ InnerShareable = 3,
+ }
+
+ impl TryFrom<Bounded<u64, 2>> for PtwShareability {
+ type Error = Error;
+
+ fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> {
+ match val.get() {
+ 0 => Ok(PtwShareability::NonShareable),
+ 2 => Ok(PtwShareability::OuterShareable),
+ 3 => Ok(PtwShareability::InnerShareable),
+ _ => Err(EINVAL),
+ }
+ }
+ }
+
+ impl From<PtwShareability> for Bounded<u64, 2> {
+ fn from(sh: PtwShareability) -> Self {
+ Bounded::try_new(sh as u64).unwrap()
+ }
+ }
+
+ register! {
+ /// Translation configuration and control.
+ pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 {
+ /// Address space mode.
+ 3:0 mode ?=> AddressSpaceMode;
+ /// Address input restriction.
+ 10:6 ina_bits ?=> InaBits;
+ /// Address output restriction.
+ 18:14 outa_bits;
+ /// Translation table concatenation enable, a 1-bit boolean flag.
+ 22:22 sl_concat_en => bool;
+ /// Translation table memory attributes.
+ 25:24 ptw_memattr ?=> PtwMemattr;
+ /// Translation table memory shareability.
+ 29:28 ptw_sh ?=> PtwShareability;
+ /// Inner read allocation hint for translation table walks, a 1-bit boolean flag.
+ 30:30 r_allocate => bool;
+ /// Disable hierarchical access permissions.
+ 33:33 disable_hier_ap => bool;
+ /// Disable access fault checking.
+ 34:34 disable_af_fault => bool;
+ /// Disable execution on all writable pages.
+ 35:35 wxn => bool;
+ /// Enable execution on readable pages.
+ 36:36 xreadable => bool;
+ /// Page-based hardware attributes for translation table walks.
+ 63:60 ptw_pbha;
+ }
+
+ // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two
+ // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+ // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+ pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 {
+ 31:0 value;
+ }
+
+ pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 {
+ 31:0 value;
+ }
+
+ /// Extra fault information for each address space. Read only.
+ pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 {
+ 31:0 value;
+ }
+
+ pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c {
+ 31:0 value;
+ }
+ }
+ }
+}
+
+/// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages.
+pub(crate) mod doorbell_block {
+ use kernel::register;
+
+ /// Number of doorbells available.
+ pub(crate) const NUM_DOORBELLS: usize = 64;
+
+ /// Doorbell block stride (64KiB).
+ ///
+ /// Each block occupies a full page, allowing it to be mapped
+ /// separately into a virtual address space.
+ const STRIDE: usize = 0x10000;
+
+ register! {
+ /// Doorbell request register. Write-only.
+ pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 {
+ /// Doorbell set. Writing 1 triggers the doorbell.
+ 0:0 ring => bool;
+ }
+ }
+}
diff --git a/drivers/gpu/drm/tyr/slot.rs b/drivers/gpu/drm/tyr/slot.rs
new file mode 100644
index 000000000..d194ead53
--- /dev/null
+++ b/drivers/gpu/drm/tyr/slot.rs
@@ -0,0 +1,404 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! Slot management abstraction for limited hardware resources.
+//!
+//! This module provides a generic [`SlotManager`] that assigns limited hardware
+//! slots to logical "seats". A seat represents an entity (such as a virtual memory
+//! (VM) address space) that needs access to a hardware slot.
+//!
+//! The [`SlotManager`] tracks slot allocation using sequence numbers (seqno) to detect
+//! when a seat's binding has been invalidated. When a seat requests activation,
+//! the manager will either reuse the seat's existing slot (if still valid),
+//! allocate a free slot (if any are available), or evict the oldest idle slot if any
+//! slots are idle.
+//!
+//! Hardware-specific behavior is customized by implementing the [`SlotOperations`]
+//! trait, which allows callbacks when slots are activated or evicted.
+//!
+//! This is currently used for managing address space slots in the GPU, and it will
+//! also be used to manage Command Stream Group (CSG) interface slots in the future.
+//!
+//! [SlotOperations]: crate::slot::SlotOperations
+//! [SlotManager]: crate::slot::SlotManager
+
+use core::{
+ mem,
+ ops::{
+ Deref,
+ DerefMut, //
+ }, //
+};
+
+use kernel::{
+ prelude::*,
+ sync::LockedBy, //
+};
+
+/// Seat information.
+///
+/// This can't be accessed directly by the element embedding a `Seat`,
+/// but is used by the generic slot manager logic to control residency
+/// of a certain object on a hardware slot.
+pub(crate) struct SeatInfo {
+ /// Slot used by this seat.
+ ///
+ /// This index is only valid if the slot pointed to by this index
+ /// has its `SlotInfo::seqno` match `SeatInfo::seqno`. Otherwise,
+ /// it means the object has been evicted from the hardware slot,
+ /// and a new slot needs to be acquired to make this object
+ /// resident again.
+ slot: u8,
+
+ /// Sequence number encoding the last time this seat was active.
+ /// We also use it to check if a slot is still bound to a seat.
+ seqno: u64,
+}
+
+/// Seat state.
+///
+/// This is meant to be embedded in the object that wants to acquire
+/// hardware slots. It also starts in the `Seat::NoSeat` state, and
+/// the slot manager will change the object value when an active/evict
+/// request is issued.
+#[derive(Default)]
+pub(crate) enum Seat {
+ #[expect(clippy::enum_variant_names)]
+ /// Resource is not resident.
+ ///
+ /// All objects start with a seat in the `Seat::NoSeat` state. The seat also
+ /// gets back to that state if the user requests eviction. It
+ /// can also end up in that state next time an operation is done
+ /// on a `Seat::Idle` seat and the slot manager finds out this
+ /// object has been evicted from the slot.
+ #[default]
+ NoSeat,
+
+ /// Resource is actively used and resident.
+ ///
+ /// When a seat is in the `Seat::Active` state, it can't be evicted, and the
+ /// slot pointed to by `SeatInfo::slot` is guaranteed to be reserved
+ /// for this object as long as the seat stays active.
+ Active(SeatInfo),
+
+ /// Resource is idle and might or might not be resident.
+ ///
+ /// When a seat is in the`Seat::Idle` state, we can't know for sure if the
+ /// object is resident or evicted until the next request we issue
+ /// to the slot manager. This tells the slot manager it can
+ /// reclaim the underlying slot if needed.
+ /// In order for the hardware to use this object again, the seat
+ /// needs to be turned into an `Seat::Active` state again
+ /// with a `SlotManager::activate()` call.
+ Idle(SeatInfo),
+}
+
+impl Seat {
+ /// Get the slot index this seat is pointing to.
+ ///
+ /// If the seat is not `Seat::Active` we can't trust the
+ /// `SeatInfo`. In that case `None` is returned, otherwise
+ /// `Some(SeatInfo::slot)` is returned.
+ pub(crate) fn slot(&self) -> Option<u8> {
+ match self {
+ Self::Active(info) => Some(info.slot),
+ _ => None,
+ }
+ }
+}
+
+/// Information related to a slot.
+struct SlotInfo<D> {
+ /// Type specific data attached to a slot.
+ slot_data: D,
+
+ /// Sequence number from when this slot was last activated.
+ seqno: u64,
+}
+
+/// Slot state.
+#[derive(Default)]
+enum Slot<D> {
+ /// Slot is free.
+ #[default]
+ Free,
+
+ /// Slot is active.
+ Active(SlotInfo<D>),
+
+ /// Slot is idle.
+ Idle(SlotInfo<D>),
+}
+
+pub(crate) type LockedSeat<T, const MAX_SLOTS: usize> = LockedBy<Seat, SlotManager<T, MAX_SLOTS>>;
+
+/// Trait describing the slot-related operations.
+pub(crate) trait SlotOperations<const MAX_SLOTS: usize>: Sized {
+ /// Implementation-specific data associated with each slot.
+ type SlotData;
+
+ /// Returns the seat belonging to this slot data.
+ fn seat(slot_data: &Self::SlotData) -> &LockedSeat<Self, MAX_SLOTS>;
+
+ /// Called when a slot is being activated for a seat.
+ fn activate(&mut self, _slot_idx: usize, _slot_data: &Self::SlotData) -> Result {
+ Ok(())
+ }
+
+ /// Called when a slot is being evicted and freed.
+ fn evict(&mut self, _slot_idx: usize, _slot_data: &Self::SlotData) -> Result {
+ Ok(())
+ }
+}
+
+/// A generic slot manager that provides access to a limited number of hardware slots.
+pub(crate) struct SlotManager<T: SlotOperations<MAX_SLOTS>, const MAX_SLOTS: usize> {
+ /// A specific implementation of the generic slot manager.
+ manager: T,
+
+ /// Number of slots actually available.
+ slot_count: usize,
+
+ /// Slot array used to track the state of each slot.
+ slots: [Slot<T::SlotData>; MAX_SLOTS],
+
+ /// Sequence number incremented each time a Seat is successfully activated
+ use_seqno: u64,
+}
+
+impl<T: SlotOperations<MAX_SLOTS>, const MAX_SLOTS: usize> SlotManager<T, MAX_SLOTS> {
+ /// Creates a specific instance of a slot manager.
+ pub(crate) fn new(manager: T, slot_count: usize) -> Result<Self> {
+ if slot_count == 0 {
+ return Err(EINVAL);
+ }
+ if slot_count > MAX_SLOTS {
+ return Err(EINVAL);
+ }
+ // Since the slot index is stored in SeatInfo as a u8, the maximum number of slots is 256.
+ if slot_count > u8::MAX as usize + 1 {
+ return Err(EINVAL);
+ }
+
+ Ok(Self {
+ manager,
+ slot_count,
+ slots: [const { Slot::Free }; MAX_SLOTS],
+ use_seqno: 1,
+ })
+ }
+
+ /// Records a newly activated slot for the given seat.
+ /// The slot manager takes ownership of the hardware-specific slot data.
+ fn record_active_slot(&mut self, slot_idx: usize, slot_data: T::SlotData) {
+ let cur_seqno = self.use_seqno;
+
+ *T::seat(&slot_data).access_mut(self) = Seat::Active(SeatInfo {
+ slot: slot_idx as u8,
+ seqno: cur_seqno,
+ });
+
+ self.slots[slot_idx] = Slot::Active(SlotInfo {
+ slot_data,
+ seqno: cur_seqno,
+ });
+
+ self.use_seqno += 1;
+ }
+
+ /// Reactivates an active/idle slot for a given seat without reprogramming the hardware.
+ /// The SlotManager reuses the existing slot_data. This ensures that the hardware-specific
+ /// information is not changed between subsequent uses. It also ensures that resources
+ /// owned by the existing slot_data remain alive while the hardware is configured to use them.
+ fn reactivate_slot(&mut self, slot_idx: usize, slot_data: &T::SlotData) -> Result {
+ let cur_seqno = self.use_seqno;
+
+ let mut slot_info = match mem::take(&mut self.slots[slot_idx]) {
+ Slot::Active(slot_info) | Slot::Idle(slot_info) => slot_info,
+ Slot::Free => {
+ *T::seat(slot_data).access_mut(self) = Seat::NoSeat;
+ return Err(EINVAL);
+ }
+ };
+
+ *T::seat(slot_data).access_mut(self) = Seat::Active(SeatInfo {
+ slot: slot_idx as u8,
+ seqno: cur_seqno,
+ });
+
+ slot_info.seqno = cur_seqno;
+ self.slots[slot_idx] = Slot::Active(slot_info);
+
+ self.use_seqno += 1;
+
+ Ok(())
+ }
+
+ /// Activates a slot for the given seat.
+ fn activate_slot(&mut self, slot_idx: usize, slot_data: T::SlotData) -> Result {
+ self.manager.activate(slot_idx, &slot_data)?;
+ self.record_active_slot(slot_idx, slot_data);
+ Ok(())
+ }
+
+ /// Finds a slot for the given seat. A free slot is preferred, but if none
+ /// are available, the oldest idle slot is evicted and reused. Otherwise, if
+ /// there are no free or idle slots, return [`EBUSY`].
+ fn allocate_slot(&mut self, slot_data: T::SlotData) -> Result {
+ let slots = &self.slots[..self.slot_count];
+
+ let mut idle_slot_idx = None;
+ let mut idle_slot_seqno: u64 = 0;
+
+ for (slot_idx, slot) in slots.iter().enumerate() {
+ match slot {
+ Slot::Free => {
+ return self.activate_slot(slot_idx, slot_data);
+ }
+ Slot::Idle(slot_info) => {
+ if idle_slot_idx.is_none() || slot_info.seqno < idle_slot_seqno {
+ idle_slot_idx = Some(slot_idx);
+ idle_slot_seqno = slot_info.seqno;
+ }
+ }
+ Slot::Active(_) => (),
+ }
+ }
+
+ match idle_slot_idx {
+ Some(slot_idx) => {
+ // Lazily evict idle slot just before it is reused.
+ if let Slot::Idle(slot_info) = &self.slots[slot_idx] {
+ self.manager.evict(slot_idx, &slot_info.slot_data)?;
+ mem::take(&mut self.slots[slot_idx]);
+ }
+ self.activate_slot(slot_idx, slot_data)
+ }
+ None => Err(EBUSY),
+ }
+ }
+
+ /// Converts an active slot and its seat to idle state.
+ fn idle_slot(&mut self, slot_idx: usize, locked_seat: &LockedSeat<T, MAX_SLOTS>) -> Result {
+ let slot = mem::take(&mut self.slots[slot_idx]);
+
+ self.slots[slot_idx] = match slot {
+ // If the slot was active, make it idle.
+ Slot::Active(slot_info) => Slot::Idle(slot_info),
+
+ // Preserve an already-idle slot.
+ Slot::Idle(slot_info) => Slot::Idle(slot_info),
+
+ // A free slot remains free.
+ Slot::Free => Slot::Free,
+ };
+
+ // If the seat was active, make it idle, or keep it idle if it was already idle.
+ *locked_seat.access_mut(self) = match locked_seat.access(self) {
+ Seat::Active(seat_info) | Seat::Idle(seat_info) => Seat::Idle(SeatInfo {
+ slot: seat_info.slot,
+ seqno: seat_info.seqno,
+ }),
+ Seat::NoSeat => Seat::NoSeat,
+ };
+ Ok(())
+ }
+
+ /// Evicts an active or idle slot: calls the eviction callback and marks the slot as free
+ /// and the seat as NoSeat.
+ fn evict_slot(&mut self, slot_idx: usize, locked_seat: &LockedSeat<T, MAX_SLOTS>) -> Result {
+ match &self.slots[slot_idx] {
+ Slot::Active(slot_info) | Slot::Idle(slot_info) => {
+ // If hardware eviction fails (e.g. times out), the slot retains
+ // its SlotData so that any resources still referenced by the hardware
+ // will remain alive. This prevents use-after-free errors.
+ self.manager.evict(slot_idx, &slot_info.slot_data)?;
+ mem::take(&mut self.slots[slot_idx]);
+ }
+ _ => (),
+ }
+
+ *locked_seat.access_mut(self) = Seat::NoSeat;
+ Ok(())
+ }
+
+ /// Checks that the seat state matches the slot's state.
+ /// If they don't match, the seat is stale and is reset to `NoSeat`.
+ fn check_seat(&mut self, locked_seat: &LockedSeat<T, MAX_SLOTS>) {
+ let (slot_idx, seat_seqno, is_active) = match locked_seat.access(self) {
+ Seat::Active(seat_info) => (seat_info.slot as usize, seat_info.seqno, true),
+ Seat::Idle(seat_info) => (seat_info.slot as usize, seat_info.seqno, false),
+ _ => return,
+ };
+
+ let valid = if is_active {
+ !kernel::warn_on!(!matches!(
+ &self.slots[slot_idx],
+ Slot::Active(slot_info) if slot_info.seqno == seat_seqno
+ ))
+ } else {
+ matches!(
+ &self.slots[slot_idx],
+ Slot::Idle(slot_info) if slot_info.seqno == seat_seqno
+ )
+ };
+
+ if !valid {
+ *locked_seat.access_mut(self) = Seat::NoSeat;
+ }
+ }
+
+ /// Activates a resource on any available/reclaimable slot.
+ pub(crate) fn activate(&mut self, slot_data: T::SlotData) -> Result {
+ self.check_seat(T::seat(&slot_data));
+
+ // Copy out only the slot index so the borrow of slot_data ends here.
+ let slot_idx = match T::seat(&slot_data).access(self) {
+ Seat::Active(seat_info) | Seat::Idle(seat_info) => Some(seat_info.slot as usize),
+ Seat::NoSeat => None,
+ };
+
+ match slot_idx {
+ Some(slot_idx) => self.reactivate_slot(slot_idx, &slot_data),
+ None => self.allocate_slot(slot_data),
+ }
+ }
+
+ /// Flag a resource as idle. This method will be used for user VM support.
+ #[expect(dead_code)]
+ pub(crate) fn idle(&mut self, locked_seat: &LockedSeat<T, MAX_SLOTS>) -> Result {
+ self.check_seat(locked_seat);
+ if let Seat::Active(seat_info) = locked_seat.access(self) {
+ self.idle_slot(seat_info.slot as usize, locked_seat)?;
+ }
+ Ok(())
+ }
+
+ /// Evict a resource from its slot.
+ pub(crate) fn evict(&mut self, locked_seat: &LockedSeat<T, MAX_SLOTS>) -> Result {
+ self.check_seat(locked_seat);
+
+ match locked_seat.access(self) {
+ Seat::Active(seat_info) | Seat::Idle(seat_info) => {
+ let slot_idx = seat_info.slot as usize;
+ self.evict_slot(slot_idx, locked_seat)?;
+ }
+ _ => (),
+ }
+
+ Ok(())
+ }
+}
+
+impl<T: SlotOperations<MAX_SLOTS>, const MAX_SLOTS: usize> Deref for SlotManager<T, MAX_SLOTS> {
+ type Target = T;
+
+ fn deref(&self) -> &Self::Target {
+ &self.manager
+ }
+}
+
+impl<T: SlotOperations<MAX_SLOTS>, const MAX_SLOTS: usize> DerefMut for SlotManager<T, MAX_SLOTS> {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.manager
+ }
+}
diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs
new file mode 100644
index 000000000..e7ec450bd
--- /dev/null
+++ b/drivers/gpu/drm/tyr/tyr.rs
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! Arm Mali Tyr DRM driver.
+//!
+//! The name "Tyr" is inspired by Norse mythology, reflecting Arm's tradition of
+//! naming their GPUs after Nordic mythological figures and places.
+
+use crate::driver::TyrPlatformDriver;
+
+mod driver;
+mod file;
+mod fw;
+mod gem;
+mod gpu;
+mod mmu;
+mod regs;
+mod slot;
+mod vm;
+
+kernel::module_platform_driver! {
+ type: TyrPlatformDriver,
+ name: "tyr",
+ authors: ["The Tyr driver authors"],
+ description: "Arm Mali Tyr DRM driver",
+ license: "Dual MIT/GPL",
+}
diff --git a/drivers/gpu/drm/tyr/vm.rs b/drivers/gpu/drm/tyr/vm.rs
new file mode 100644
index 000000000..c5e307b1e
--- /dev/null
+++ b/drivers/gpu/drm/tyr/vm.rs
@@ -0,0 +1,950 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+//! GPU virtual memory management using the DRM GPUVM framework.
+//!
+//! This module manages GPU virtual address spaces, providing memory isolation and
+//! the illusion of owning the entire virtual address (VA) range, similar to CPU virtual memory.
+//! Each virtual memory (VM) area is backed by ARM64 LPAE Stage 1 page tables and can be
+//! mapped into hardware address space (AS) slots for GPU execution.
+
+use core::marker::PhantomData;
+use core::ops::Range;
+
+use kernel::{
+ device::{
+ Bound,
+ Device, //
+ },
+ drm::{
+ gem::BaseObject,
+ gpuvm::{
+ DriverGpuVm,
+ GpuVaAlloc,
+ GpuVm,
+ GpuVmBo,
+ OpMap,
+ OpMapRequest,
+ OpMapped,
+ OpRemap,
+ OpRemapped,
+ OpUnmap,
+ OpUnmapped,
+ UniqueRefGpuVm, //
+ }, //
+ },
+ fmt,
+ impl_flags,
+ io::PhysAddr,
+ iommu::pgtable::{
+ prot,
+ IoPageTable,
+ ARM64LPAES1, //
+ },
+ new_mutex,
+ prelude::*,
+ sizes::{
+ SZ_1G,
+ SZ_2M,
+ SZ_4K, //
+ },
+ sync::{
+ aref::ARef,
+ Arc,
+ ArcBorrow,
+ Mutex, //
+ },
+ uapi, //
+};
+
+use crate::{
+ driver::{
+ TyrDrmDevice,
+ TyrDrmDriver, //
+ },
+ gem,
+ gem::Bo,
+ gpu::GpuInfo,
+ mmu::{
+ address_space::VmAsData,
+ Mmu, //
+ },
+ regs::gpu_control::MMU_FEATURES,
+};
+
+impl_flags!(
+ /// Flags controlling virtual memory mapping behavior.
+ ///
+ /// These flags control access permissions and caching behavior for GPU virtual
+ /// memory mappings.
+ #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)]
+ pub(crate) struct VmMapFlags(u32);
+
+ /// Individual flags that can be combined in [`VmMapFlags`].
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+ pub(crate) enum VmFlag {
+ /// Map as read-only.
+ Readonly = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_READONLY as u32,
+ /// Map as non-executable.
+ Noexec = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC as u32,
+ /// Map as uncached.
+ Uncached = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED as u32,
+ }
+);
+
+impl VmMapFlags {
+ /// Convert the flags to `pgtable::prot`.
+ fn to_prot(self) -> u32 {
+ let mut prot = 0;
+
+ if self.contains(VmFlag::Readonly) {
+ prot |= prot::READ;
+ } else {
+ prot |= prot::READ | prot::WRITE;
+ }
+
+ if self.contains(VmFlag::Noexec) {
+ prot |= prot::NOEXEC;
+ }
+
+ if !self.contains(VmFlag::Uncached) {
+ prot |= prot::CACHE;
+ }
+
+ prot
+ }
+}
+
+impl fmt::Display for VmMapFlags {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let mut first = true;
+
+ if self.contains(VmFlag::Readonly) {
+ write!(f, "READONLY")?;
+ first = false;
+ }
+ if self.contains(VmFlag::Noexec) {
+ if !first {
+ write!(f, " | ")?;
+ }
+ write!(f, "NOEXEC")?;
+ first = false;
+ }
+
+ if self.contains(VmFlag::Uncached) {
+ if !first {
+ write!(f, " | ")?;
+ }
+ write!(f, "UNCACHED")?;
+ }
+
+ Ok(())
+ }
+}
+
+impl TryFrom<u32> for VmMapFlags {
+ type Error = Error;
+
+ fn try_from(value: u32) -> Result<Self, Self::Error> {
+ let valid = VmFlag::Readonly as u32 | VmFlag::Noexec as u32 | VmFlag::Uncached as u32;
+
+ if value & !valid != 0 {
+ return Err(EINVAL);
+ }
+ Ok(Self(value))
+ }
+}
+
+/// Arguments for a virtual memory map operation.
+struct VmMapArgs<'drm> {
+ /// Access permissions and caching behavior for the mapping.
+ flags: VmMapFlags,
+ /// GEM buffer object registered with the GPUVM framework.
+ vm_bo: ARef<GpuVmBo<GpuVmData<'drm>>>,
+ /// Offset in bytes from the start of the buffer object.
+ bo_offset: u64,
+}
+
+/// Type of virtual memory operation.
+enum VmOpType<'drm> {
+ /// Map a GEM buffer object into the virtual address space.
+ Map(VmMapArgs<'drm>),
+ /// Unmap a region from the virtual address space.
+ Unmap,
+}
+
+/// Preallocated resources needed to execute a VM operation.
+///
+/// VM operations may require allocating new GPUVA objects to track mappings.
+/// To avoid allocation failures during the operation, preallocate the
+/// maximum number of GPUVAs that might be needed.
+struct VmOpResources<'drm> {
+ /// Preallocated GPUVA objects for remap operations.
+ ///
+ /// Partial unmap requests or map requests overlapping existing mappings
+ /// will trigger a remap call, which needs to register up to three VA
+ /// objects (one for the new mapping, and two for the previous and next
+ /// mappings).
+ preallocated_gpuvas: [Option<GpuVaAlloc<GpuVmData<'drm>>>; 3],
+}
+
+/// Request to execute a virtual memory operation.
+struct VmOpRequest<'drm> {
+ /// Request type.
+ op_type: VmOpType<'drm>,
+
+ /// Region of the virtual address space covered by this request.
+ region: Range<u64>,
+}
+
+/// Arguments for a page table map operation.
+struct PtMapArgs {
+ /// Memory protection flags describing allowed accesses for this mapping.
+ ///
+ /// This is directly derived from [`VmMapFlags`] via [`VmMapFlags::to_prot`].
+ prot: u32,
+}
+
+/// Type of page table operation.
+enum PtOpType {
+ /// Map pages into the page table.
+ Map(PtMapArgs),
+ /// Unmap pages from the page table.
+ Unmap,
+}
+
+/// Context for updating the GPU page table.
+///
+/// This context is created when beginning a page table update operation and
+/// automatically flushes changes when dropped. It ensures that the
+/// Memory Management Unit (MMU) state is properly managed and Translation
+/// Lookaside Buffer (TLB) entries are flushed.
+pub(crate) struct PtUpdateContext<'ctx, 'drm> {
+ /// Device used for DMA-mapping GEM shmem SG tables.
+ dev: &'ctx Device<Bound>,
+
+ /// Page table.
+ pt: &'ctx IoPageTable<'drm, ARM64LPAES1>,
+
+ /// MMU manager.
+ mmu: &'ctx Mmu<'drm>,
+
+ /// Reference to the address space data to pass to the MMU functions.
+ as_data: &'ctx VmAsData<'drm>,
+
+ /// Region of the virtual address space covered by this request.
+ region: Range<u64>,
+
+ /// Operation type.
+ op_type: PtOpType,
+
+ /// Preallocated resources that can be used when executing the request.
+ resources: &'ctx mut VmOpResources<'drm>,
+}
+
+impl<'ctx, 'drm> PtUpdateContext<'ctx, 'drm> {
+ /// Creates a new page table update context.
+ ///
+ /// This prepares the MMU for a page table update.
+ /// The context will automatically flush the TLB and
+ /// complete the update when dropped.
+ fn new(
+ dev: &'ctx Device<Bound>,
+ pt: &'ctx IoPageTable<'drm, ARM64LPAES1>,
+ mmu: &'ctx Mmu<'drm>,
+ as_data: &'ctx VmAsData<'drm>,
+ region: Range<u64>,
+ op_type: PtOpType,
+ resources: &'ctx mut VmOpResources<'drm>,
+ ) -> Result<PtUpdateContext<'ctx, 'drm>> {
+ mmu.start_vm_update(as_data, &region)?;
+
+ Ok(Self {
+ dev,
+ pt,
+ mmu,
+ as_data,
+ region,
+ op_type,
+ resources,
+ })
+ }
+
+ /// Finds one of our pre-allocated VAs.
+ fn preallocated_gpuva(&mut self) -> Result<GpuVaAlloc<GpuVmData<'drm>>> {
+ self.resources
+ .preallocated_gpuvas
+ .iter_mut()
+ .find_map(|f| f.take())
+ .ok_or(EINVAL)
+ }
+
+ /// Returns an unused GPUVA object to the preallocated pool.
+ /// If the pool is already full, the unused allocation is simply dropped.
+ fn return_preallocated_gpuva(&mut self, gpuva: GpuVaAlloc<GpuVmData<'drm>>) {
+ if let Some(slot) = self
+ .resources
+ .preallocated_gpuvas
+ .iter_mut()
+ .find(|slot| slot.is_none())
+ {
+ *slot = Some(gpuva);
+ }
+ }
+}
+
+impl Drop for PtUpdateContext<'_, '_> {
+ fn drop(&mut self) {
+ if let Err(e) = self.mmu.end_vm_update(self.as_data) {
+ dev_err!(self.dev, "Failed to end VM update {:?}", e);
+ }
+
+ if let Err(e) = self.mmu.flush_vm(self.as_data) {
+ dev_err!(self.dev, "Failed to flush VM {:?}", e);
+ }
+ }
+}
+
+/// Driver implementation for the GPUVM framework.
+///
+/// Implements [`DriverGpuVm`] to provide VM operation callbacks (map, unmap, remap)
+/// and associated types for buffer objects, virtual addresses, and contexts.
+pub(crate) struct GpuVmData<'drm> {
+ _phantom: PhantomData<&'drm ()>,
+}
+
+/// GPU virtual address space.
+///
+/// Each VM can be mapped into a hardware address space slot.
+#[pin_data]
+pub(crate) struct Vm<'drm> {
+ /// Data referenced by an AS when the VM is active
+ as_data: Arc<VmAsData<'drm>>,
+ /// MMU manager.
+ mmu: Arc<Mmu<'drm>>,
+ /// Parent device used for DMA mapping and page-table operations.
+ dev: &'drm Device<Bound>,
+ /// DRM GPUVM core for managing virtual address space.
+ #[pin]
+ gpuvm_unique: Mutex<UniqueRefGpuVm<GpuVmData<'drm>>>,
+ /// Non-core part of the GPUVM. Can be used for stuff that doesn't modify the
+ /// internal mapping tree, like GpuVm::obtain()
+ gpuvm: ARef<GpuVm<GpuVmData<'drm>>>,
+ /// VA range for this VM.
+ va_range: Range<u64>,
+}
+
+impl<'drm> Vm<'drm> {
+ /// Creates a new GPU virtual address space.
+ ///
+ /// The VM is initialized with a page table configured according to the GPU's
+ /// address translation capabilities and registered with the GPUVM framework.
+ pub(crate) fn new(
+ dev: &'drm Device<Bound>,
+ ddev: &TyrDrmDevice,
+ mmu: ArcBorrow<'_, Mmu<'drm>>,
+ gpu_info: &GpuInfo,
+ ) -> Result<Arc<Vm<'drm>>> {
+ let mmu_features = MMU_FEATURES::from_raw(gpu_info.mmu_features);
+ let va_bits = mmu_features.va_bits().get();
+ let pa_bits = mmu_features.pa_bits().get();
+
+ let range = 0..(1u64 << va_bits);
+ let reserve_range = 0..0u64;
+
+ // dummy_obj is used to initialize the GPUVM tree.
+ let dummy_obj = gem::new_dummy_object(ddev).inspect_err(|e| {
+ dev_err!(dev, "Failed to create dummy GEM object: {:?}", e);
+ })?;
+
+ let gpuvm_unique = GpuVm::new::<Error, _>(
+ c"Tyr::GpuVm",
+ ddev,
+ &*dummy_obj,
+ range.clone(),
+ reserve_range,
+ GpuVmData::<'drm> {
+ _phantom: PhantomData::<&()>,
+ },
+ )
+ .inspect_err(|e| {
+ dev_err!(dev, "Failed to create GpuVm: {:?}", e);
+ })?;
+ let gpuvm = ARef::from(&*gpuvm_unique);
+
+ let as_data = Arc::pin_init(VmAsData::new(&mmu, dev, va_bits, pa_bits), GFP_KERNEL)?;
+
+ let vm = Arc::pin_init(
+ pin_init!(Self{
+ as_data,
+ dev,
+ mmu: mmu.into(),
+ gpuvm,
+ gpuvm_unique <- new_mutex!(gpuvm_unique),
+ va_range: range,
+ }),
+ GFP_KERNEL,
+ )?;
+
+ Ok(vm)
+ }
+
+ /// Returns the parent device used by this VM for DMA mapping and page-table operations.
+ pub(crate) fn dev(&self) -> &'drm Device<Bound> {
+ self.dev
+ }
+
+ /// Activate the VM in a hardware address space slot.
+ pub(crate) fn activate(&self) -> Result {
+ self.mmu
+ .activate_vm(self.as_data.as_arc_borrow())
+ .inspect_err(|e| {
+ dev_err!(self.dev, "Failed to activate VM: {:?}", e);
+ })
+ }
+
+ /// Deactivate the VM by evicting it from its address space slot.
+ fn deactivate(&self) -> Result {
+ self.mmu.deactivate_vm(&self.as_data).inspect_err(|e| {
+ dev_err!(self.dev, "Failed to deactivate VM: {:?}", e);
+ })
+ }
+
+ /// Kills the VM by deactivating it and unmapping all regions.
+ pub(crate) fn kill(&self) {
+ // TODO: Turn the VM into a state where it can't be used.
+ let _ = self.deactivate();
+ let _ = self
+ .unmap_range(self.va_range.start, self.va_range.end - self.va_range.start)
+ .inspect_err(|e| {
+ dev_err!(self.dev, "Failed to unmap range during deactivate: {:?}", e);
+ });
+ }
+
+ /// Executes a virtual memory operation.
+ ///
+ /// This handles both map and unmap operations by coordinating between the
+ /// GPUVM framework and the hardware page table.
+ fn exec_op<'a>(
+ &self,
+ gpuvm_unique: &mut UniqueRefGpuVm<GpuVmData<'drm>>,
+ req: VmOpRequest<'drm>,
+ resources: &'a mut VmOpResources<'drm>,
+ ) -> Result {
+ let pt = &self.as_data.page_table;
+
+ match req.op_type {
+ VmOpType::Map(args) => {
+ let mut pt_upd = PtUpdateContext::new(
+ self.dev,
+ pt,
+ &self.mmu,
+ &self.as_data,
+ req.region,
+ PtOpType::Map(PtMapArgs {
+ prot: args.flags.to_prot(),
+ }),
+ resources,
+ )?;
+
+ gpuvm_unique.sm_map(OpMapRequest {
+ addr: pt_upd.region.start,
+ range: pt_upd.region.end - pt_upd.region.start,
+ gem_offset: args.bo_offset,
+ vm_bo: &args.vm_bo,
+ context: &mut pt_upd,
+ })
+ //PtUpdateContext drops here flushing the page table
+ }
+ VmOpType::Unmap => {
+ let mut pt_upd = PtUpdateContext::new(
+ self.dev,
+ pt,
+ &self.mmu,
+ &self.as_data,
+ req.region,
+ PtOpType::Unmap,
+ resources,
+ )?;
+
+ gpuvm_unique.sm_unmap(
+ pt_upd.region.start,
+ pt_upd.region.end - pt_upd.region.start,
+ &mut pt_upd,
+ )
+ //PtUpdateContext drops here flushing the page table
+ }
+ }
+ }
+
+ /// Maps a GEM buffer object range into the VM at the specified virtual address.
+ ///
+ /// This creates a mapping from GPU virtual address `va` to the physical pages
+ /// backing the GEM object, starting at `bo_offset` bytes into the object and
+ /// spanning `map_size` bytes. The mapping respects the access permissions and
+ /// caching behavior specified in `flags`.
+ pub(crate) fn map_bo_range(
+ &self,
+ bo: &Bo,
+ bo_offset: u64,
+ map_size: u64,
+ va: u64,
+ flags: VmMapFlags,
+ ) -> Result {
+ if map_size == 0
+ || va % SZ_4K as u64 != 0
+ || bo_offset % SZ_4K as u64 != 0
+ || map_size % SZ_4K as u64 != 0
+ {
+ return Err(EINVAL);
+ }
+
+ let bo_size = u64::try_from(bo.size()).map_err(|_| EOVERFLOW)?;
+ let bo_end = bo_offset.checked_add(map_size).ok_or(EINVAL)?;
+
+ if bo_end > bo_size {
+ dev_err!(
+ self.dev,
+ "BO mapping range {:#x}..{:#x} exceeds BO size {:#x}",
+ bo_offset,
+ bo_end,
+ bo_size
+ );
+ return Err(EINVAL);
+ }
+
+ let va_end: u64 = va.checked_add(map_size).ok_or(EINVAL)?;
+
+ let req = VmOpRequest {
+ op_type: VmOpType::Map(VmMapArgs {
+ vm_bo: self.gpuvm.obtain(bo, ())?,
+ flags,
+ bo_offset,
+ }),
+ region: va..va_end,
+ };
+ let mut resources = VmOpResources {
+ preallocated_gpuvas: [
+ Some(GpuVaAlloc::<GpuVmData<'drm>>::new(GFP_KERNEL)?),
+ Some(GpuVaAlloc::<GpuVmData<'drm>>::new(GFP_KERNEL)?),
+ Some(GpuVaAlloc::<GpuVmData<'drm>>::new(GFP_KERNEL)?),
+ ],
+ };
+ let result = {
+ let mut gpuvm_unique = self.gpuvm_unique.lock();
+ self.exec_op(gpuvm_unique.as_mut().get_mut(), req, &mut resources)
+ };
+ // We flush the defer cleanup list now. Things will be different in
+ // the asynchronous VM_BIND path, where we want the cleanup to
+ // happen outside the DMA signalling path.
+ self.gpuvm.deferred_cleanup();
+ result
+ }
+
+ /// Unmaps a virtual address range from the VM.
+ ///
+ /// This removes any existing mappings in the specified range, freeing the
+ /// virtual address space for reuse.
+ pub(crate) fn unmap_range(&self, va: u64, size: u64) -> Result {
+ if size == 0 || va % SZ_4K as u64 != 0 || size % SZ_4K as u64 != 0 {
+ return Err(EINVAL);
+ }
+
+ let end = va.checked_add(size).ok_or(EINVAL)?;
+
+ if va < self.va_range.start || end > self.va_range.end {
+ dev_err!(
+ self.dev,
+ "Unmap range {:#x}..{:#x} exceeds VM range {:#x}..{:#x}",
+ va,
+ end,
+ self.va_range.start,
+ self.va_range.end
+ );
+ return Err(EINVAL);
+ }
+
+ let req = VmOpRequest {
+ op_type: VmOpType::Unmap,
+ region: va..end,
+ };
+
+ let full_vm = va == self.va_range.start && end == self.va_range.end;
+
+ let mut resources = VmOpResources {
+ preallocated_gpuvas: if full_vm {
+ // Unmapping the entire VM cannot split an existing mapping,
+ // so no GPUVA objects are needed for remap operations.
+ [None, None, None]
+ } else {
+ [
+ Some(GpuVaAlloc::<GpuVmData<'drm>>::new(GFP_KERNEL)?),
+ Some(GpuVaAlloc::<GpuVmData<'drm>>::new(GFP_KERNEL)?),
+ Some(GpuVaAlloc::<GpuVmData<'drm>>::new(GFP_KERNEL)?),
+ ]
+ },
+ };
+ let result = {
+ let mut gpuvm_unique = self.gpuvm_unique.lock();
+ self.exec_op(gpuvm_unique.as_mut().get_mut(), req, &mut resources)
+ };
+ // We flush the defer cleanup list now. Things will be different in
+ // the asynchronous VM_BIND path, where we want the cleanup to
+ // happen outside the DMA signalling path.
+ self.gpuvm.deferred_cleanup();
+ result
+ }
+}
+
+impl<'drm> DriverGpuVm for GpuVmData<'drm> {
+ type Driver = TyrDrmDriver;
+ type Object = Bo;
+ type VmBoData = ();
+ type VaData = ();
+ type SmContext<'ctx>
+ = PtUpdateContext<'ctx, 'drm>
+ where
+ Self: 'ctx;
+
+ /// Create a new mapping.
+ fn sm_step_map<'op>(
+ &mut self,
+ op: OpMap<'op, Self>,
+ context: &mut Self::SmContext<'_>,
+ ) -> Result<OpMapped<'op, Self>, Error> {
+ let start_iova = op.addr();
+ let mut iova = start_iova;
+ let mut bytes_left_to_map = op.length();
+ let mut gem_offset = op.gem_offset();
+
+ // Make sure that the end of the requested GEM range doesn't run past the
+ // end of the GEM buffer itself.
+ let gem_range_end = op.gem_offset().checked_add(op.length()).ok_or(EINVAL)?;
+
+ if gem_range_end > op.obj().size() as u64 {
+ dev_err!(
+ context.dev,
+ "Requested GEM range ends at {} which is beyond the GEM buffer size {}",
+ gem_range_end,
+ op.obj().size()
+ );
+ return Err(EINVAL);
+ }
+
+ let sgt = op.obj().sg_table(context.dev).inspect_err(|e| {
+ dev_err!(context.dev, "Failed to get sg_table: {:?}", e);
+ })?;
+ let prot = match &context.op_type {
+ PtOpType::Map(args) => args.prot,
+ _ => {
+ return Err(EINVAL);
+ }
+ };
+
+ for sgt_entry in sgt.iter() {
+ // Expressly convert to u64 to work with arm 32-bit builds.
+ #[allow(clippy::useless_conversion)]
+ let mut paddr = u64::from(sgt_entry.dma_address());
+ #[allow(clippy::useless_conversion)]
+ let mut sgt_entry_length = u64::from(sgt_entry.dma_len());
+
+ if bytes_left_to_map == 0 {
+ break;
+ }
+
+ if gem_offset > 0 {
+ // Skip the entire SGT entry if the gem_offset exceeds its length.
+ let skip = u64::min(sgt_entry_length, gem_offset);
+ paddr += skip;
+ sgt_entry_length -= skip;
+ gem_offset -= skip;
+ }
+
+ if sgt_entry_length == 0 {
+ continue;
+ }
+
+ let len = u64::min(sgt_entry_length, bytes_left_to_map);
+
+ let segment_mapped = match pt_map(context.dev, context.pt, iova, paddr, len, prot) {
+ Ok(segment_mapped) => segment_mapped,
+ Err(e) => {
+ // clean up any successful mappings from previous SGT entries.
+ let total_mapped = iova - start_iova;
+ if total_mapped > 0 {
+ let _ = pt_unmap(
+ context.dev,
+ context.pt,
+ start_iova..(start_iova + total_mapped),
+ );
+ }
+ return Err(e);
+ }
+ };
+
+ bytes_left_to_map -= segment_mapped;
+ iova += segment_mapped;
+ }
+
+ if bytes_left_to_map != 0 {
+ let total_mapped = iova - start_iova;
+
+ if total_mapped > 0 {
+ let _ = pt_unmap(context.dev, context.pt, start_iova..iova);
+ }
+
+ dev_err!(
+ context.dev,
+ "SG table is too small for requested mapping: {} bytes remain",
+ bytes_left_to_map
+ );
+
+ return Err(EINVAL);
+ }
+
+ let gpuva = context.preallocated_gpuva()?;
+ let op = op.insert(gpuva, pin_init::init_zeroed());
+
+ Ok(op)
+ }
+
+ /// Indicates that an existing mapping should be removed.
+ fn sm_step_unmap<'op>(
+ &mut self,
+ op: OpUnmap<'op, Self>,
+ context: &mut Self::SmContext<'_>,
+ ) -> Result<OpUnmapped<'op, Self>, Error> {
+ let start_iova = op.va().addr();
+ let length = op.va().length();
+
+ let region = start_iova..(start_iova + length);
+ pt_unmap(context.dev, context.pt, region.clone()).inspect_err(|e| {
+ dev_err!(
+ context.dev,
+ "Failed to unmap region {:#x}..{:#x}: {:?}",
+ region.start,
+ region.end,
+ e
+ );
+ })?;
+
+ let (op_unmapped, _va_removed) = op.remove();
+
+ Ok(op_unmapped)
+ }
+
+ /// Split up an existing mapping.
+ fn sm_step_remap<'op>(
+ &mut self,
+ op: OpRemap<'op, Self>,
+ context: &mut Self::SmContext<'_>,
+ ) -> Result<OpRemapped<'op, Self>, Error> {
+ let unmap_start = if let Some(prev) = op.prev() {
+ prev.addr() + prev.length()
+ } else {
+ op.va_to_unmap().addr()
+ };
+
+ let unmap_end = if let Some(next) = op.next() {
+ next.addr()
+ } else {
+ op.va_to_unmap().addr() + op.va_to_unmap().length()
+ };
+
+ let unmap_length = unmap_end - unmap_start;
+
+ if unmap_length > 0 {
+ let region = unmap_start..(unmap_start + unmap_length);
+ pt_unmap(context.dev, context.pt, region.clone()).inspect_err(|e| {
+ dev_err!(
+ context.dev,
+ "Failed to unmap remap region {:#x}..{:#x}: {:?}",
+ region.start,
+ region.end,
+ e
+ );
+ })?;
+ }
+
+ let prev_va = context.preallocated_gpuva()?;
+ let next_va = context.preallocated_gpuva()?;
+
+ let (op_remapped, remap_ret) = op.remap(
+ [prev_va, next_va],
+ pin_init::init_zeroed(),
+ pin_init::init_zeroed(),
+ );
+
+ if let Some(unused_va) = remap_ret.unused_va {
+ context.return_preallocated_gpuva(unused_va);
+ }
+
+ Ok(op_remapped)
+ }
+}
+
+/// This function selects the largest supported block size (currently 4KB or 2MB)
+/// that can be used for a mapping at the given address and size, respecting alignment constraints.
+///
+/// We can map multiple pages at once but we can't exceed the size of the
+/// table entry itself. So, if mapping 4KB pages, figure out how many pages
+/// can be mapped before we hit the 2MB boundary. Or, if mapping 2MB pages,
+/// figure out how many pages can be mapped before hitting the 1GB boundary
+/// Returns the page size (4KB or 2MB) and the number of pages that can be mapped at that size.
+fn get_pgsize(addr: u64, size: u64) -> (u64, u64) {
+ // Get the distance to the next boundary of 2MB block
+ let blk_offset_2m = addr.wrapping_neg() % (SZ_2M as u64);
+
+ // Use 4K blocks if the address is not 2MB aligned, or we have less than 2MB to map
+ if blk_offset_2m != 0 || size < SZ_2M as u64 {
+ let pgcount = if blk_offset_2m == 0 {
+ size / SZ_4K as u64
+ } else {
+ u64::min(blk_offset_2m, size) / SZ_4K as u64
+ };
+ return (SZ_4K as u64, pgcount);
+ }
+
+ let blk_offset_1g = addr.wrapping_neg() % (SZ_1G as u64);
+ let blk_offset = if blk_offset_1g == 0 {
+ SZ_1G as u64
+ } else {
+ blk_offset_1g
+ };
+ let pgcount = u64::min(blk_offset, size) / SZ_2M as u64;
+
+ (SZ_2M as u64, pgcount)
+}
+
+/// Maps a physical address range into the page table at the specified virtual address.
+///
+/// This function maps `len` bytes of physical memory starting at `paddr` to the
+/// virtual address `iova`, using the protection flags specified in `prot`. It
+/// automatically selects optimal page sizes to minimize page table overhead.
+///
+/// If the mapping fails partway through, all successfully mapped pages are
+/// unmapped before returning an error.
+///
+/// Returns the number of bytes successfully mapped.
+fn pt_map(
+ dev: &Device,
+ pt: &IoPageTable<'_, ARM64LPAES1>,
+ iova: u64,
+ paddr: u64,
+ len: u64,
+ prot: u32,
+) -> Result<u64> {
+ let mut segment_mapped = 0u64;
+ while segment_mapped < len {
+ let remaining = len - segment_mapped;
+ let curr_iova = iova + segment_mapped;
+ let curr_paddr = paddr + segment_mapped;
+
+ let (pgsize, pgcount) = get_pgsize(curr_iova | curr_paddr, remaining);
+
+ // On 32-bit systems, usize is only 32 bits, so check that
+ // the iova can be converted without truncation.
+ let curr_iova = match usize::try_from(curr_iova) {
+ Ok(curr_iova) => curr_iova,
+ Err(_) => {
+ dev_err!(
+ dev,
+ "curr_iova {:#x} cannot be represented as usize (max {:#x})",
+ curr_iova,
+ usize::MAX
+ );
+
+ if segment_mapped > 0 {
+ let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped));
+ }
+
+ return Err(EOVERFLOW);
+ }
+ };
+
+ // SAFETY:
+ // No other io-pgtable operation can currently access this range because Tyr holds
+ // the gpuvm_unique mutex for the entire sm_map() operation.
+ // The addresses being mapped won't overlap any existing mappings in this
+ // page table because drm_gpuvm_sm_map() checks each requested mapping and either unmaps
+ // or remaps any overlap before creating the new mapping.
+ let (mapped, result) = unsafe {
+ pt.map_pages(
+ curr_iova,
+ curr_paddr as PhysAddr,
+ pgsize as usize,
+ pgcount as usize,
+ prot,
+ GFP_KERNEL,
+ )
+ };
+
+ if let Err(e) = result {
+ // If map_pages fails, mapped will be zero because the ARM LPAE backend
+ // only updates the mapped value after the entire request succeeds.
+ dev_err!(dev, "pt.map_pages failed at iova {:#x}: {:?}", curr_iova, e);
+ if segment_mapped > 0 {
+ let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped));
+ }
+ return Err(e);
+ }
+
+ if mapped == 0 {
+ dev_err!(dev, "Failed to map any pages at iova {:#x}", curr_iova);
+ if segment_mapped > 0 {
+ let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped));
+ }
+ return Err(ENOMEM);
+ }
+
+ segment_mapped += mapped as u64;
+ }
+
+ Ok(segment_mapped)
+}
+
+/// Unmaps a virtual address range from the page table.
+///
+/// This function removes all page table entries in the specified range,
+/// automatically handling different page sizes that may be present.
+fn pt_unmap(dev: &Device, pt: &IoPageTable<'_, ARM64LPAES1>, range: Range<u64>) -> Result {
+ let mut iova = range.start;
+ let mut bytes_left_to_unmap = range.end - range.start;
+
+ while bytes_left_to_unmap > 0 {
+ // It is fine to use just the iova to determine the page size
+ // because if the actual mapping was represented with smaller page sizes,
+ // (e.g. because the physical address was not 2MiB aligned)
+ // the ARM LPAE backend will notice and handle the lower-level table correctly.
+ let (pgsize, pgcount) = get_pgsize(iova, bytes_left_to_unmap);
+
+ // On 32-bit systems, usize is only 32 bits, so check that
+ // the iova can be converted without truncation.
+ let iova_usize = usize::try_from(iova).map_err(|_| {
+ dev_err!(
+ dev,
+ "IOVA {:#x} cannot be represented as usize (max {:#x})",
+ iova,
+ usize::MAX
+ );
+ EOVERFLOW
+ })?;
+
+ // SAFETY:
+ // No other io-pgtable operation can currently access this range because Tyr holds
+ // the gpuvm_unique mutex for the entire sm_unmap() operation.
+ // We know that this page table has one or more consecutive mappings
+ // starting at `iova` with the total size of `pgcount * pgsize` because
+ // gpuvm callbacks provide exactly the range that was previously mapped.
+ let unmapped = unsafe { pt.unmap_pages(iova_usize, pgsize as usize, pgcount as usize) };
+
+ if unmapped == 0 {
+ dev_err!(dev, "Failed to unmap any bytes at iova {:#x}", iova_usize);
+ return Err(EINVAL);
+ }
+
+ bytes_left_to_unmap -= unmapped as u64;
+ iova += unmapped as u64;
+ }
+
+ Ok(())
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-29 20:55:06 +0000