From 034dd340b08be1f2f0477ad16131d609f9dbd53c Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 30 Aug 2026 09:22:00 -0700 Subject: Merge tag 'trace-v7.3-2' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace Pull tracing fixes from Steven Rostedt: - Fix error output of boot instance creation failure Currently if a boot instance creation fails, instead of printing out the name of the instance that failed, it prints "(null)". That is because it prints "cur_str" that had already been processed by strsep(). Print the saved name instead. While at it, print the error code of the failure. - Fix use-after-free for same named historgrams Histograms can be named so that they can be used in multiple events. But if the named histogram has a variable attached, the second event that uses the named histogram which duplicates it and needs to free the original after duplication leaves the old variable in place and still visible. If another histogram uses than variable, it will use the stale one which will try to reference the freed duplicate histogram and crash the kernel. Free the duplicate variables along with the duplicated histogram data. - Check return value of kthread_run() in event self test The events self tests uses a kthread for testing but does not check if it succeeded in creating a kthread. If the kthread creation were to fail, the code will still try to call kthread_stop() on the error returned. - Fix race between reading trace_pipe and updating subbuffer size If a user is reading the trace_pipe file at the same time they update the ring buffer sub-buffer size, can cause the trace_pipe read to read stale data. Add trace_access_lock() around updating the ring buffer sub-buffer size. - Fix eventfs_inode on failure path in creation of the events directory In the creation of the "events" directory, if after allocating the eventfs_inode a failure is detected, it calls cleanup_ei() which calls free_ei(). The free_ei() will test if eventfs_inode being freed has no children. It is a bug if it does. But on the failure case of the creation of the "events" directory, the children lists have not yet been initialized and the free will trigger a warning because list_empty() on an uninitialized list returns false. Move the initialization into init_ei() where it makes more sense and makes sure that a created eventfs_inode has its lists initialized upon creation. - Check return value of kthread_run() in ftrace direct sample code The sample code that shows how to use the ftrace direct calls does not test the return of kthread_run() to see if it succeeds. Return a failure if the kthread_run() doesn't succeed. - Clear user events state on fork in case of alloc failure On fork, the child gets a pointer to the parent's user events state. It makes a copy of it then updates the child's pointer to it. But if the allocation fails, the duplication function leaves the child with a pointer to its parent's descriptor. When the child cleans up its data, it will free the parent's descriptor while the parent is still using it. In the duplication function, set the child's user_event_mm to NULL before testing if the allocation succeeded, and when it exits it will not free the parent's descriptor. - Fix retry exhaustion in simple ring buffer reader swap simple_ring_buffer_swap_reader_page() starts with retry set to 8 and post-decrements it only after a failed link replacement. On the final attempt, a successful replacement leaves retry at zero, while a failed replacement leaves it at -1. But the check for success expects the retry value to be non-zero and exits with an error on zero. This is the opposite result. Fix it. - Fail nicely when the remote swap_reader_page() returns an error Currently, if the swap_reader_page() of a remote buffer fails, it triggers a WARN_ON_ONCE() and continues normally. Instead, have it exit with an error and a pr_warn() print instead of a full WARNING. * tag 'trace-v7.3-2' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace: ring-buffer: Stop remote reader update when page swap fails tracing: Fix retry exhaustion in simple ring buffer reader swap tracing/user_events: Clear copied tracing state before fork duplication samples/ftrace: Fix kthread_stop() on ERR_PTR in ftrace-direct-multi-modify samples/ftrace: Fix kthread_stop() on ERR_PTR in ftrace-direct-modify eventfs: Initialize ei->children and ei->list in init_ei() tracing: Fix use-after-free in trace_pipe read on sub-buffer order change tracing: Fix crash passing ERR_PTR to kthread_stop() tracing: Fix use-after-free with same-name named triggers tracing: Fix logged instance name on creation failure --- drivers/gpu/drm/xe/.gitignore | 5 + drivers/gpu/drm/xe/.kunitconfig | 18 + drivers/gpu/drm/xe/.kunitconfig-display | 11 + drivers/gpu/drm/xe/Kconfig | 142 + drivers/gpu/drm/xe/Kconfig.debug | 130 + drivers/gpu/drm/xe/Kconfig.profile | 76 + drivers/gpu/drm/xe/Makefile | 400 ++ drivers/gpu/drm/xe/abi/gsc_command_header_abi.h | 46 + drivers/gpu/drm/xe/abi/gsc_mkhi_commands_abi.h | 39 + drivers/gpu/drm/xe/abi/gsc_proxy_commands_abi.h | 44 + drivers/gpu/drm/xe/abi/gsc_pxp_commands_abi.h | 100 + drivers/gpu/drm/xe/abi/guc_actions_abi.h | 288 ++ drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h | 279 ++ drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h | 706 +++ drivers/gpu/drm/xe/abi/guc_capture_abi.h | 187 + drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h | 129 + .../gpu/drm/xe/abi/guc_communication_mmio_abi.h | 49 + drivers/gpu/drm/xe/abi/guc_errors_abi.h | 101 + drivers/gpu/drm/xe/abi/guc_klvs_abi.h | 546 +++ drivers/gpu/drm/xe/abi/guc_lfd_abi.h | 170 + drivers/gpu/drm/xe/abi/guc_lic_abi.h | 77 + drivers/gpu/drm/xe/abi/guc_log_abi.h | 109 + drivers/gpu/drm/xe/abi/guc_messages_abi.h | 251 + drivers/gpu/drm/xe/abi/guc_relay_actions_abi.h | 247 + .../gpu/drm/xe/abi/guc_relay_communication_abi.h | 118 + drivers/gpu/drm/xe/abi/guc_scheduler_abi.h | 58 + drivers/gpu/drm/xe/abi/uc_fw_abi.h | 405 ++ drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h | 27 + drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h | 65 + .../gpu/drm/xe/compat-i915-headers/i915_config.h | 16 + .../xe/compat-i915-headers/i915_gtt_view_types.h | 7 + .../gpu/drm/xe/compat-i915-headers/i915_reg_defs.h | 6 + .../xe/compat-i915-headers/intel_clock_gating.h | 14 + .../gpu/drm/xe/compat-i915-headers/intel_uncore.h | 139 + .../xe/compat-i915-headers/intel_uncore_trace.h | 6 + drivers/gpu/drm/xe/display/tests/Makefile | 13 + drivers/gpu/drm/xe/display/xe_display.c | 509 ++ drivers/gpu/drm/xe/display/xe_display.h | 88 + drivers/gpu/drm/xe/display/xe_display_bo.c | 212 + drivers/gpu/drm/xe/display/xe_display_bo.h | 15 + drivers/gpu/drm/xe/display/xe_display_pcode.c | 38 + drivers/gpu/drm/xe/display/xe_display_pcode.h | 9 + drivers/gpu/drm/xe/display/xe_display_rpm.c | 77 + drivers/gpu/drm/xe/display/xe_display_rpm.h | 11 + drivers/gpu/drm/xe/display/xe_display_vma.h | 18 + drivers/gpu/drm/xe/display/xe_display_wa.c | 19 + drivers/gpu/drm/xe/display/xe_dsb_buffer.c | 102 + drivers/gpu/drm/xe/display/xe_dsb_buffer.h | 9 + drivers/gpu/drm/xe/display/xe_fb_pin.c | 491 ++ drivers/gpu/drm/xe/display/xe_fb_pin.h | 21 + drivers/gpu/drm/xe/display/xe_frontbuffer.c | 71 + drivers/gpu/drm/xe/display/xe_frontbuffer.h | 9 + drivers/gpu/drm/xe/display/xe_hdcp_gsc.c | 217 + drivers/gpu/drm/xe/display/xe_hdcp_gsc.h | 9 + drivers/gpu/drm/xe/display/xe_initial_plane.c | 196 + drivers/gpu/drm/xe/display/xe_initial_plane.h | 9 + drivers/gpu/drm/xe/display/xe_panic.c | 110 + drivers/gpu/drm/xe/display/xe_panic.h | 9 + drivers/gpu/drm/xe/display/xe_stolen.c | 113 + drivers/gpu/drm/xe/display/xe_stolen.h | 9 + drivers/gpu/drm/xe/display/xe_tdf.c | 15 + drivers/gpu/drm/xe/instructions/xe_alu_commands.h | 79 + .../drm/xe/instructions/xe_gfx_state_commands.h | 18 + .../gpu/drm/xe/instructions/xe_gfxpipe_commands.h | 174 + drivers/gpu/drm/xe/instructions/xe_gpu_commands.h | 76 + drivers/gpu/drm/xe/instructions/xe_gsc_commands.h | 36 + drivers/gpu/drm/xe/instructions/xe_instr_defs.h | 35 + drivers/gpu/drm/xe/instructions/xe_mfx_commands.h | 28 + drivers/gpu/drm/xe/instructions/xe_mi_commands.h | 103 + drivers/gpu/drm/xe/regs/xe_bars.h | 12 + drivers/gpu/drm/xe/regs/xe_engine_regs.h | 237 + drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h | 29 + drivers/gpu/drm/xe/regs/xe_gsc_regs.h | 58 + drivers/gpu/drm/xe/regs/xe_gt_regs.h | 666 +++ drivers/gpu/drm/xe/regs/xe_gtt_defs.h | 42 + drivers/gpu/drm/xe/regs/xe_guc_regs.h | 165 + drivers/gpu/drm/xe/regs/xe_hw_error_regs.h | 88 + drivers/gpu/drm/xe/regs/xe_i2c_regs.h | 25 + drivers/gpu/drm/xe/regs/xe_irq_regs.h | 98 + drivers/gpu/drm/xe/regs/xe_lrc_layout.h | 46 + drivers/gpu/drm/xe/regs/xe_mchbar_regs.h | 48 + drivers/gpu/drm/xe/regs/xe_mert_regs.h | 23 + drivers/gpu/drm/xe/regs/xe_oa_regs.h | 123 + drivers/gpu/drm/xe/regs/xe_pcode_regs.h | 33 + drivers/gpu/drm/xe/regs/xe_pmt.h | 34 + drivers/gpu/drm/xe/regs/xe_pxp_regs.h | 23 + drivers/gpu/drm/xe/regs/xe_reg_defs.h | 138 + drivers/gpu/drm/xe/regs/xe_regs.h | 67 + drivers/gpu/drm/xe/regs/xe_soc_remapper_regs.h | 14 + drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h | 36 + drivers/gpu/drm/xe/tests/Makefile | 14 + drivers/gpu/drm/xe/tests/xe_args_test.c | 275 ++ drivers/gpu/drm/xe/tests/xe_bo.c | 874 ++++ drivers/gpu/drm/xe/tests/xe_dma_buf.c | 299 ++ .../gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c | 319 ++ drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c | 329 ++ drivers/gpu/drm/xe/tests/xe_guc_db_mgr_test.c | 201 + drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c | 833 ++++ drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c | 136 + .../gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c | 429 ++ drivers/gpu/drm/xe/tests/xe_guc_relay_test.c | 522 ++ drivers/gpu/drm/xe/tests/xe_kunit_helpers.c | 129 + drivers/gpu/drm/xe/tests/xe_kunit_helpers.h | 19 + drivers/gpu/drm/xe/tests/xe_live_test_mod.c | 31 + drivers/gpu/drm/xe/tests/xe_lmtt_test.c | 73 + drivers/gpu/drm/xe/tests/xe_migrate.c | 785 +++ drivers/gpu/drm/xe/tests/xe_mocs.c | 193 + drivers/gpu/drm/xe/tests/xe_pci.c | 420 ++ drivers/gpu/drm/xe/tests/xe_pci_test.c | 84 + drivers/gpu/drm/xe/tests/xe_pci_test.h | 36 + drivers/gpu/drm/xe/tests/xe_rtp.c | 38 + drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c | 112 + drivers/gpu/drm/xe/tests/xe_rtp_test.c | 730 +++ drivers/gpu/drm/xe/tests/xe_rtp_test.h | 23 + drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c | 89 + .../gpu/drm/xe/tests/xe_sriov_pf_service_kunit.c | 227 + drivers/gpu/drm/xe/tests/xe_test.h | 63 + drivers/gpu/drm/xe/tests/xe_test_mod.c | 10 + drivers/gpu/drm/xe/tests/xe_wa_test.c | 80 + drivers/gpu/drm/xe/xe_amc.c | 197 + drivers/gpu/drm/xe/xe_amc.h | 25 + drivers/gpu/drm/xe/xe_args.h | 170 + drivers/gpu/drm/xe/xe_assert.h | 176 + drivers/gpu/drm/xe/xe_bb.c | 174 + drivers/gpu/drm/xe/xe_bb.h | 28 + drivers/gpu/drm/xe/xe_bb_types.h | 20 + drivers/gpu/drm/xe/xe_bo.c | 4092 ++++++++++++++++ drivers/gpu/drm/xe/xe_bo.h | 617 +++ drivers/gpu/drm/xe/xe_bo_doc.h | 179 + drivers/gpu/drm/xe/xe_bo_evict.c | 352 ++ drivers/gpu/drm/xe/xe_bo_evict.h | 21 + drivers/gpu/drm/xe/xe_bo_types.h | 144 + drivers/gpu/drm/xe/xe_configfs.c | 1418 ++++++ drivers/gpu/drm/xe/xe_configfs.h | 65 + drivers/gpu/drm/xe/xe_debugfs.c | 777 +++ drivers/gpu/drm/xe/xe_debugfs.h | 23 + drivers/gpu/drm/xe/xe_defaults.h | 26 + drivers/gpu/drm/xe/xe_dep_job_types.h | 29 + drivers/gpu/drm/xe/xe_dep_scheduler.c | 143 + drivers/gpu/drm/xe/xe_dep_scheduler.h | 26 + drivers/gpu/drm/xe/xe_devcoredump.c | 517 ++ drivers/gpu/drm/xe/xe_devcoredump.h | 35 + drivers/gpu/drm/xe/xe_devcoredump_types.h | 93 + drivers/gpu/drm/xe/xe_device.c | 1563 ++++++ drivers/gpu/drm/xe/xe_device.h | 300 ++ drivers/gpu/drm/xe/xe_device_sysfs.c | 289 ++ drivers/gpu/drm/xe/xe_device_sysfs.h | 13 + drivers/gpu/drm/xe/xe_device_types.h | 678 +++ drivers/gpu/drm/xe/xe_device_wa_oob.rules | 9 + drivers/gpu/drm/xe/xe_dma_buf.c | 398 ++ drivers/gpu/drm/xe/xe_dma_buf.h | 15 + drivers/gpu/drm/xe/xe_drm_client.c | 410 ++ drivers/gpu/drm/xe/xe_drm_client.h | 70 + drivers/gpu/drm/xe/xe_drm_ras.c | 218 + drivers/gpu/drm/xe/xe_drm_ras.h | 15 + drivers/gpu/drm/xe/xe_drm_ras_types.h | 48 + drivers/gpu/drm/xe/xe_drv.h | 22 + drivers/gpu/drm/xe/xe_eu_stall.c | 1043 ++++ drivers/gpu/drm/xe/xe_eu_stall.h | 25 + drivers/gpu/drm/xe/xe_exec.c | 382 ++ drivers/gpu/drm/xe/xe_exec.h | 14 + drivers/gpu/drm/xe/xe_exec_queue.c | 1873 ++++++++ drivers/gpu/drm/xe/xe_exec_queue.h | 165 + drivers/gpu/drm/xe/xe_exec_queue_types.h | 350 ++ drivers/gpu/drm/xe/xe_execlist.c | 486 ++ drivers/gpu/drm/xe/xe_execlist.h | 21 + drivers/gpu/drm/xe/xe_execlist_types.h | 51 + drivers/gpu/drm/xe/xe_force_wake.c | 302 ++ drivers/gpu/drm/xe/xe_force_wake.h | 115 + drivers/gpu/drm/xe/xe_force_wake_types.h | 110 + drivers/gpu/drm/xe/xe_gen_wa_oob.c | 213 + drivers/gpu/drm/xe/xe_ggtt.c | 1167 +++++ drivers/gpu/drm/xe/xe_ggtt.h | 63 + drivers/gpu/drm/xe/xe_ggtt_types.h | 21 + drivers/gpu/drm/xe/xe_gpu_scheduler.c | 139 + drivers/gpu/drm/xe/xe_gpu_scheduler.h | 101 + drivers/gpu/drm/xe/xe_gpu_scheduler_types.h | 59 + drivers/gpu/drm/xe/xe_gsc.c | 629 +++ drivers/gpu/drm/xe/xe_gsc.h | 27 + drivers/gpu/drm/xe/xe_gsc_debugfs.c | 69 + drivers/gpu/drm/xe/xe_gsc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_gsc_proxy.c | 559 +++ drivers/gpu/drm/xe/xe_gsc_proxy.h | 21 + drivers/gpu/drm/xe/xe_gsc_submit.c | 218 + drivers/gpu/drm/xe/xe_gsc_submit.h | 32 + drivers/gpu/drm/xe/xe_gsc_types.h | 76 + drivers/gpu/drm/xe/xe_gt.c | 1193 +++++ drivers/gpu/drm/xe/xe_gt.h | 163 + drivers/gpu/drm/xe/xe_gt_ccs_mode.c | 218 + drivers/gpu/drm/xe/xe_gt_ccs_mode.h | 24 + drivers/gpu/drm/xe/xe_gt_clock.c | 84 + drivers/gpu/drm/xe/xe_gt_clock.h | 16 + drivers/gpu/drm/xe/xe_gt_debugfs.c | 394 ++ drivers/gpu/drm/xe/xe_gt_debugfs.h | 16 + drivers/gpu/drm/xe/xe_gt_freq.c | 297 ++ drivers/gpu/drm/xe/xe_gt_freq.h | 13 + drivers/gpu/drm/xe/xe_gt_idle.c | 434 ++ drivers/gpu/drm/xe/xe_gt_idle.h | 22 + drivers/gpu/drm/xe/xe_gt_idle_types.h | 43 + drivers/gpu/drm/xe/xe_gt_mcr.c | 911 ++++ drivers/gpu/drm/xe/xe_gt_mcr.h | 74 + drivers/gpu/drm/xe/xe_gt_printk.h | 128 + drivers/gpu/drm/xe/xe_gt_sriov_pf.c | 304 ++ drivers/gpu/drm/xe/xe_gt_sriov_pf.h | 51 + drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 3521 ++++++++++++++ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h | 108 + drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h | 60 + drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c | 2203 +++++++++ drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h | 44 + drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h | 139 + drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c | 916 ++++ drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.h | 19 + drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h | 35 + drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c | 1069 +++++ drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h | 54 + .../gpu/drm/xe/xe_gt_sriov_pf_migration_types.h | 28 + drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c | 151 + drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h | 27 + drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor_types.h | 22 + drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c | 796 ++++ drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h | 37 + drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h | 70 + drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c | 426 ++ drivers/gpu/drm/xe/xe_gt_sriov_pf_service.h | 34 + drivers/gpu/drm/xe/xe_gt_sriov_pf_service_types.h | 52 + drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h | 64 + drivers/gpu/drm/xe/xe_gt_sriov_printk.h | 37 + drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 1625 +++++++ drivers/gpu/drm/xe/xe_gt_sriov_vf.h | 46 + drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c | 84 + drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.h | 14 + drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h | 102 + drivers/gpu/drm/xe/xe_gt_stats.c | 203 + drivers/gpu/drm/xe/xe_gt_stats.h | 64 + drivers/gpu/drm/xe/xe_gt_stats_types.h | 205 + drivers/gpu/drm/xe/xe_gt_sysfs.c | 55 + drivers/gpu/drm/xe/xe_gt_sysfs.h | 19 + drivers/gpu/drm/xe/xe_gt_sysfs_types.h | 26 + drivers/gpu/drm/xe/xe_gt_throttle.c | 267 ++ drivers/gpu/drm/xe/xe_gt_throttle.h | 17 + drivers/gpu/drm/xe/xe_gt_topology.c | 351 ++ drivers/gpu/drm/xe/xe_gt_topology.h | 56 + drivers/gpu/drm/xe/xe_gt_types.h | 395 ++ drivers/gpu/drm/xe/xe_guard.h | 119 + drivers/gpu/drm/xe/xe_guc.c | 1921 ++++++++ drivers/gpu/drm/xe/xe_guc.h | 104 + drivers/gpu/drm/xe/xe_guc_ads.c | 1128 +++++ drivers/gpu/drm/xe/xe_guc_ads.h | 21 + drivers/gpu/drm/xe/xe_guc_ads_types.h | 34 + drivers/gpu/drm/xe/xe_guc_buf.c | 206 + drivers/gpu/drm/xe/xe_guc_buf.h | 49 + drivers/gpu/drm/xe/xe_guc_buf_types.h | 28 + drivers/gpu/drm/xe/xe_guc_capture.c | 2057 ++++++++ drivers/gpu/drm/xe/xe_guc_capture.h | 63 + drivers/gpu/drm/xe/xe_guc_capture_types.h | 70 + drivers/gpu/drm/xe/xe_guc_ct.c | 2326 +++++++++ drivers/gpu/drm/xe/xe_guc_ct.h | 90 + drivers/gpu/drm/xe/xe_guc_ct_types.h | 173 + drivers/gpu/drm/xe/xe_guc_db_mgr.c | 267 ++ drivers/gpu/drm/xe/xe_guc_db_mgr.h | 22 + drivers/gpu/drm/xe/xe_guc_debugfs.c | 148 + drivers/gpu/drm/xe/xe_guc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_guc_engine_activity.c | 523 ++ drivers/gpu/drm/xe/xe_guc_engine_activity.h | 22 + drivers/gpu/drm/xe/xe_guc_engine_activity_types.h | 102 + drivers/gpu/drm/xe/xe_guc_exec_queue_types.h | 81 + drivers/gpu/drm/xe/xe_guc_fwif.h | 321 ++ drivers/gpu/drm/xe/xe_guc_hwconfig.c | 202 + drivers/gpu/drm/xe/xe_guc_hwconfig.h | 20 + drivers/gpu/drm/xe/xe_guc_hxg_helpers.h | 108 + drivers/gpu/drm/xe/xe_guc_id_mgr.c | 280 ++ drivers/gpu/drm/xe/xe_guc_id_mgr.h | 22 + drivers/gpu/drm/xe/xe_guc_klv_helpers.c | 377 ++ drivers/gpu/drm/xe/xe_guc_klv_helpers.h | 74 + drivers/gpu/drm/xe/xe_guc_klv_thresholds_set.h | 71 + .../gpu/drm/xe/xe_guc_klv_thresholds_set_types.h | 74 + drivers/gpu/drm/xe/xe_guc_log.c | 680 +++ drivers/gpu/drm/xe/xe_guc_log.h | 75 + drivers/gpu/drm/xe/xe_guc_log_types.h | 57 + drivers/gpu/drm/xe/xe_guc_pagefault.c | 103 + drivers/gpu/drm/xe/xe_guc_pagefault.h | 15 + drivers/gpu/drm/xe/xe_guc_pc.c | 1448 ++++++ drivers/gpu/drm/xe/xe_guc_pc.h | 44 + drivers/gpu/drm/xe/xe_guc_pc_types.h | 40 + drivers/gpu/drm/xe/xe_guc_rc.c | 131 + drivers/gpu/drm/xe/xe_guc_rc.h | 16 + drivers/gpu/drm/xe/xe_guc_relay.c | 971 ++++ drivers/gpu/drm/xe/xe_guc_relay.h | 37 + drivers/gpu/drm/xe/xe_guc_relay_types.h | 40 + drivers/gpu/drm/xe/xe_guc_submit.c | 3736 +++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 60 + drivers/gpu/drm/xe/xe_guc_submit_types.h | 146 + drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 407 ++ drivers/gpu/drm/xe/xe_guc_tlb_inval.h | 19 + drivers/gpu/drm/xe/xe_guc_types.h | 127 + drivers/gpu/drm/xe/xe_heci_gsc.c | 248 + drivers/gpu/drm/xe/xe_heci_gsc.h | 40 + drivers/gpu/drm/xe/xe_huc.c | 314 ++ drivers/gpu/drm/xe/xe_huc.h | 28 + drivers/gpu/drm/xe/xe_huc_debugfs.c | 69 + drivers/gpu/drm/xe/xe_huc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_huc_types.h | 24 + drivers/gpu/drm/xe/xe_hw_engine.c | 1177 +++++ drivers/gpu/drm/xe/xe_hw_engine.h | 81 + drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c | 681 +++ drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h | 43 + drivers/gpu/drm/xe/xe_hw_engine_group.c | 481 ++ drivers/gpu/drm/xe/xe_hw_engine_group.h | 31 + drivers/gpu/drm/xe/xe_hw_engine_group_types.h | 51 + drivers/gpu/drm/xe/xe_hw_engine_types.h | 200 + drivers/gpu/drm/xe/xe_hw_error.c | 555 +++ drivers/gpu/drm/xe/xe_hw_error.h | 15 + drivers/gpu/drm/xe/xe_hw_fence.c | 249 + drivers/gpu/drm/xe/xe_hw_fence.h | 31 + drivers/gpu/drm/xe/xe_hw_fence_types.h | 75 + drivers/gpu/drm/xe/xe_hwmon.c | 1585 +++++++ drivers/gpu/drm/xe/xe_hwmon.h | 19 + drivers/gpu/drm/xe/xe_i2c.c | 393 ++ drivers/gpu/drm/xe/xe_i2c.h | 69 + drivers/gpu/drm/xe/xe_irq.c | 1053 +++++ drivers/gpu/drm/xe/xe_irq.h | 26 + drivers/gpu/drm/xe/xe_late_bind_fw.c | 464 ++ drivers/gpu/drm/xe/xe_late_bind_fw.h | 17 + drivers/gpu/drm/xe/xe_late_bind_fw_types.h | 78 + drivers/gpu/drm/xe/xe_lmtt.c | 609 +++ drivers/gpu/drm/xe/xe_lmtt.h | 29 + drivers/gpu/drm/xe/xe_lmtt_2l.c | 150 + drivers/gpu/drm/xe/xe_lmtt_ml.c | 161 + drivers/gpu/drm/xe/xe_lmtt_types.h | 63 + drivers/gpu/drm/xe/xe_lrc.c | 2818 +++++++++++ drivers/gpu/drm/xe/xe_lrc.h | 190 + drivers/gpu/drm/xe/xe_lrc_types.h | 81 + drivers/gpu/drm/xe/xe_macros.h | 22 + drivers/gpu/drm/xe/xe_map.h | 93 + drivers/gpu/drm/xe/xe_mem_pool.c | 403 ++ drivers/gpu/drm/xe/xe_mem_pool.h | 35 + drivers/gpu/drm/xe/xe_mem_pool_types.h | 21 + drivers/gpu/drm/xe/xe_memirq.c | 591 +++ drivers/gpu/drm/xe/xe_memirq.h | 30 + drivers/gpu/drm/xe/xe_memirq_types.h | 28 + drivers/gpu/drm/xe/xe_mert.c | 118 + drivers/gpu/drm/xe/xe_mert.h | 35 + drivers/gpu/drm/xe/xe_migrate.c | 2637 +++++++++++ drivers/gpu/drm/xe/xe_migrate.h | 185 + drivers/gpu/drm/xe/xe_migrate_doc.h | 88 + drivers/gpu/drm/xe/xe_mmio.c | 447 ++ drivers/gpu/drm/xe/xe_mmio.h | 45 + drivers/gpu/drm/xe/xe_mmio_gem.c | 226 + drivers/gpu/drm/xe/xe_mmio_gem.h | 20 + drivers/gpu/drm/xe/xe_mmio_types.h | 64 + drivers/gpu/drm/xe/xe_mocs.c | 831 ++++ drivers/gpu/drm/xe/xe_mocs.h | 16 + drivers/gpu/drm/xe/xe_module.c | 219 + drivers/gpu/drm/xe/xe_module.h | 35 + drivers/gpu/drm/xe/xe_nvm.c | 183 + drivers/gpu/drm/xe/xe_nvm.h | 13 + drivers/gpu/drm/xe/xe_oa.c | 2884 +++++++++++ drivers/gpu/drm/xe/xe_oa.h | 25 + drivers/gpu/drm/xe/xe_oa_types.h | 274 ++ drivers/gpu/drm/xe/xe_observation.c | 128 + drivers/gpu/drm/xe/xe_observation.h | 19 + drivers/gpu/drm/xe/xe_page_reclaim.c | 158 + drivers/gpu/drm/xe/xe_page_reclaim.h | 128 + drivers/gpu/drm/xe/xe_pagefault.c | 497 ++ drivers/gpu/drm/xe/xe_pagefault.h | 19 + drivers/gpu/drm/xe/xe_pagefault_types.h | 138 + drivers/gpu/drm/xe/xe_pat.c | 797 ++++ drivers/gpu/drm/xe/xe_pat.h | 93 + drivers/gpu/drm/xe/xe_pci.c | 1478 ++++++ drivers/gpu/drm/xe/xe_pci.h | 15 + drivers/gpu/drm/xe/xe_pci_error.c | 139 + drivers/gpu/drm/xe/xe_pci_error.h | 13 + drivers/gpu/drm/xe/xe_pci_rebar.c | 106 + drivers/gpu/drm/xe/xe_pci_rebar.h | 13 + drivers/gpu/drm/xe/xe_pci_sriov.c | 264 ++ drivers/gpu/drm/xe/xe_pci_sriov.h | 21 + drivers/gpu/drm/xe/xe_pci_types.h | 98 + drivers/gpu/drm/xe/xe_pcode.c | 384 ++ drivers/gpu/drm/xe/xe_pcode.h | 44 + drivers/gpu/drm/xe/xe_pcode_api.h | 109 + drivers/gpu/drm/xe/xe_platform_types.h | 43 + drivers/gpu/drm/xe/xe_pm.c | 1037 ++++ drivers/gpu/drm/xe/xe_pm.h | 58 + drivers/gpu/drm/xe/xe_pmu.c | 600 +++ drivers/gpu/drm/xe/xe_pmu.h | 18 + drivers/gpu/drm/xe/xe_pmu_types.h | 39 + drivers/gpu/drm/xe/xe_preempt_fence.c | 191 + drivers/gpu/drm/xe/xe_preempt_fence.h | 61 + drivers/gpu/drm/xe/xe_preempt_fence_types.h | 34 + drivers/gpu/drm/xe/xe_printk.h | 129 + drivers/gpu/drm/xe/xe_psmi.c | 294 ++ drivers/gpu/drm/xe/xe_psmi.h | 14 + drivers/gpu/drm/xe/xe_pt.c | 2934 ++++++++++++ drivers/gpu/drm/xe/xe_pt.h | 52 + drivers/gpu/drm/xe/xe_pt_types.h | 130 + drivers/gpu/drm/xe/xe_pt_walk.c | 161 + drivers/gpu/drm/xe/xe_pt_walk.h | 152 + drivers/gpu/drm/xe/xe_pxp.c | 1037 ++++ drivers/gpu/drm/xe/xe_pxp.h | 35 + drivers/gpu/drm/xe/xe_pxp_debugfs.c | 129 + drivers/gpu/drm/xe/xe_pxp_debugfs.h | 13 + drivers/gpu/drm/xe/xe_pxp_submit.c | 602 +++ drivers/gpu/drm/xe/xe_pxp_submit.h | 22 + drivers/gpu/drm/xe/xe_pxp_types.h | 146 + drivers/gpu/drm/xe/xe_query.c | 824 ++++ drivers/gpu/drm/xe/xe_query.h | 14 + drivers/gpu/drm/xe/xe_range_fence.c | 163 + drivers/gpu/drm/xe/xe_range_fence.h | 75 + drivers/gpu/drm/xe/xe_ras.c | 695 +++ drivers/gpu/drm/xe/xe_ras.h | 22 + drivers/gpu/drm/xe/xe_ras_types.h | 289 ++ drivers/gpu/drm/xe/xe_reg_sr.c | 312 ++ drivers/gpu/drm/xe/xe_reg_sr.h | 35 + drivers/gpu/drm/xe/xe_reg_sr_types.h | 31 + drivers/gpu/drm/xe/xe_reg_whitelist.c | 358 ++ drivers/gpu/drm/xe/xe_reg_whitelist.h | 31 + drivers/gpu/drm/xe/xe_res_cursor.h | 368 ++ drivers/gpu/drm/xe/xe_ring_ops.c | 652 +++ drivers/gpu/drm/xe/xe_ring_ops.h | 17 + drivers/gpu/drm/xe/xe_ring_ops_types.h | 28 + drivers/gpu/drm/xe/xe_rtp.c | 478 ++ drivers/gpu/drm/xe/xe_rtp.h | 609 +++ drivers/gpu/drm/xe/xe_rtp_helpers.h | 85 + drivers/gpu/drm/xe/xe_rtp_types.h | 163 + drivers/gpu/drm/xe/xe_sa.c | 252 + drivers/gpu/drm/xe/xe_sa.h | 90 + drivers/gpu/drm/xe/xe_sa_types.h | 22 + drivers/gpu/drm/xe/xe_sched_job.c | 360 ++ drivers/gpu/drm/xe/xe_sched_job.h | 95 + drivers/gpu/drm/xe/xe_sched_job_types.h | 83 + drivers/gpu/drm/xe/xe_shrinker.c | 306 ++ drivers/gpu/drm/xe/xe_shrinker.h | 16 + drivers/gpu/drm/xe/xe_sleep.h | 57 + drivers/gpu/drm/xe/xe_soc_remapper.c | 53 + drivers/gpu/drm/xe/xe_soc_remapper.h | 13 + drivers/gpu/drm/xe/xe_sriov.c | 176 + drivers/gpu/drm/xe/xe_sriov.h | 45 + drivers/gpu/drm/xe/xe_sriov_packet.c | 528 +++ drivers/gpu/drm/xe/xe_sriov_packet.h | 30 + drivers/gpu/drm/xe/xe_sriov_packet_types.h | 75 + drivers/gpu/drm/xe/xe_sriov_pf.c | 283 ++ drivers/gpu/drm/xe/xe_sriov_pf.h | 31 + drivers/gpu/drm/xe/xe_sriov_pf_control.c | 303 ++ drivers/gpu/drm/xe/xe_sriov_pf_control.h | 23 + drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c | 392 ++ drivers/gpu/drm/xe/xe_sriov_pf_debugfs.h | 18 + drivers/gpu/drm/xe/xe_sriov_pf_helpers.h | 75 + drivers/gpu/drm/xe/xe_sriov_pf_migration.c | 366 ++ drivers/gpu/drm/xe/xe_sriov_pf_migration.h | 30 + drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h | 37 + drivers/gpu/drm/xe/xe_sriov_pf_provision.c | 572 +++ drivers/gpu/drm/xe/xe_sriov_pf_provision.h | 50 + drivers/gpu/drm/xe/xe_sriov_pf_provision_types.h | 36 + drivers/gpu/drm/xe/xe_sriov_pf_service.c | 216 + drivers/gpu/drm/xe/xe_sriov_pf_service.h | 23 + drivers/gpu/drm/xe/xe_sriov_pf_service_types.h | 36 + drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c | 664 +++ drivers/gpu/drm/xe/xe_sriov_pf_sysfs.h | 16 + drivers/gpu/drm/xe/xe_sriov_pf_types.h | 70 + drivers/gpu/drm/xe/xe_sriov_printk.h | 46 + drivers/gpu/drm/xe/xe_sriov_types.h | 40 + drivers/gpu/drm/xe/xe_sriov_vf.c | 286 ++ drivers/gpu/drm/xe/xe_sriov_vf.h | 20 + drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 521 ++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.h | 37 + drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h | 48 + drivers/gpu/drm/xe/xe_sriov_vf_types.h | 47 + drivers/gpu/drm/xe/xe_sriov_vfio.c | 81 + drivers/gpu/drm/xe/xe_step.c | 298 ++ drivers/gpu/drm/xe/xe_step.h | 27 + drivers/gpu/drm/xe/xe_step_types.h | 20 + drivers/gpu/drm/xe/xe_survivability_mode.c | 462 ++ drivers/gpu/drm/xe/xe_survivability_mode.h | 18 + drivers/gpu/drm/xe/xe_survivability_mode_types.h | 55 + drivers/gpu/drm/xe/xe_svm.c | 2085 ++++++++ drivers/gpu/drm/xe/xe_svm.h | 443 ++ drivers/gpu/drm/xe/xe_sync.c | 446 ++ drivers/gpu/drm/xe/xe_sync.h | 49 + drivers/gpu/drm/xe/xe_sync_types.h | 31 + drivers/gpu/drm/xe/xe_sysctrl.c | 127 + drivers/gpu/drm/xe/xe_sysctrl.h | 24 + drivers/gpu/drm/xe/xe_sysctrl_event.c | 88 + drivers/gpu/drm/xe/xe_sysctrl_event_types.h | 54 + drivers/gpu/drm/xe/xe_sysctrl_mailbox.c | 385 ++ drivers/gpu/drm/xe/xe_sysctrl_mailbox.h | 33 + drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h | 71 + drivers/gpu/drm/xe/xe_sysctrl_types.h | 36 + drivers/gpu/drm/xe/xe_tile.c | 249 + drivers/gpu/drm/xe/xe_tile.h | 47 + drivers/gpu/drm/xe/xe_tile_debugfs.c | 147 + drivers/gpu/drm/xe/xe_tile_debugfs.h | 16 + drivers/gpu/drm/xe/xe_tile_printk.h | 127 + drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c | 252 + drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.h | 15 + drivers/gpu/drm/xe/xe_tile_sriov_printk.h | 33 + drivers/gpu/drm/xe/xe_tile_sriov_vf.c | 160 + drivers/gpu/drm/xe/xe_tile_sriov_vf.h | 20 + drivers/gpu/drm/xe/xe_tile_sriov_vf_types.h | 23 + drivers/gpu/drm/xe/xe_tile_sysfs.c | 61 + drivers/gpu/drm/xe/xe_tile_sysfs.h | 19 + drivers/gpu/drm/xe/xe_tile_sysfs_types.h | 27 + drivers/gpu/drm/xe/xe_tile_types.h | 139 + drivers/gpu/drm/xe/xe_tlb_inval.c | 572 +++ drivers/gpu/drm/xe/xe_tlb_inval.h | 54 + drivers/gpu/drm/xe/xe_tlb_inval_job.c | 319 ++ drivers/gpu/drm/xe/xe_tlb_inval_job.h | 38 + drivers/gpu/drm/xe/xe_tlb_inval_types.h | 149 + drivers/gpu/drm/xe/xe_trace.c | 9 + drivers/gpu/drm/xe/xe_trace.h | 520 ++ drivers/gpu/drm/xe/xe_trace_bo.c | 9 + drivers/gpu/drm/xe/xe_trace_bo.h | 263 ++ drivers/gpu/drm/xe/xe_trace_guc.c | 9 + drivers/gpu/drm/xe/xe_trace_guc.h | 159 + drivers/gpu/drm/xe/xe_trace_lrc.c | 9 + drivers/gpu/drm/xe/xe_trace_lrc.h | 79 + drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c | 330 ++ drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h | 33 + drivers/gpu/drm/xe/xe_ttm_sys_mgr.c | 119 + drivers/gpu/drm/xe/xe_ttm_sys_mgr.h | 13 + drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 523 ++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 46 + drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h | 48 + drivers/gpu/drm/xe/xe_tuning.c | 274 ++ drivers/gpu/drm/xe/xe_tuning.h | 25 + drivers/gpu/drm/xe/xe_uc.c | 340 ++ drivers/gpu/drm/xe/xe_uc.h | 26 + drivers/gpu/drm/xe/xe_uc_debugfs.c | 30 + drivers/gpu/drm/xe/xe_uc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_uc_fw.c | 973 ++++ drivers/gpu/drm/xe/xe_uc_fw.h | 197 + drivers/gpu/drm/xe/xe_uc_fw_types.h | 155 + drivers/gpu/drm/xe/xe_uc_types.h | 28 + drivers/gpu/drm/xe/xe_userptr.c | 434 ++ drivers/gpu/drm/xe/xe_userptr.h | 136 + drivers/gpu/drm/xe/xe_validation.c | 277 ++ drivers/gpu/drm/xe/xe_validation.h | 192 + drivers/gpu/drm/xe/xe_vm.c | 4992 ++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 439 ++ drivers/gpu/drm/xe/xe_vm_doc.h | 555 +++ drivers/gpu/drm/xe/xe_vm_madvise.c | 749 +++ drivers/gpu/drm/xe/xe_vm_madvise.h | 16 + drivers/gpu/drm/xe/xe_vm_types.h | 556 +++ drivers/gpu/drm/xe/xe_vram.c | 402 ++ drivers/gpu/drm/xe/xe_vram.h | 26 + drivers/gpu/drm/xe/xe_vram_freq.c | 123 + drivers/gpu/drm/xe/xe_vram_freq.h | 13 + drivers/gpu/drm/xe/xe_vram_types.h | 74 + drivers/gpu/drm/xe/xe_vsec.c | 225 + drivers/gpu/drm/xe/xe_vsec.h | 15 + drivers/gpu/drm/xe/xe_wa.c | 1032 ++++ drivers/gpu/drm/xe/xe_wa.h | 60 + drivers/gpu/drm/xe/xe_wa_oob.rules | 74 + drivers/gpu/drm/xe/xe_wait_user_fence.c | 182 + drivers/gpu/drm/xe/xe_wait_user_fence.h | 15 + drivers/gpu/drm/xe/xe_wopcm.c | 287 ++ drivers/gpu/drm/xe/xe_wopcm.h | 16 + drivers/gpu/drm/xe/xe_wopcm_types.h | 26 + 557 files changed, 144727 insertions(+) create mode 100644 drivers/gpu/drm/xe/.gitignore create mode 100644 drivers/gpu/drm/xe/.kunitconfig create mode 100644 drivers/gpu/drm/xe/.kunitconfig-display create mode 100644 drivers/gpu/drm/xe/Kconfig create mode 100644 drivers/gpu/drm/xe/Kconfig.debug create mode 100644 drivers/gpu/drm/xe/Kconfig.profile create mode 100644 drivers/gpu/drm/xe/Makefile create mode 100644 drivers/gpu/drm/xe/abi/gsc_command_header_abi.h create mode 100644 drivers/gpu/drm/xe/abi/gsc_mkhi_commands_abi.h create mode 100644 drivers/gpu/drm/xe/abi/gsc_proxy_commands_abi.h create mode 100644 drivers/gpu/drm/xe/abi/gsc_pxp_commands_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_actions_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_capture_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_errors_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_klvs_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_lfd_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_lic_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_log_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_messages_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_relay_actions_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_relay_communication_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_scheduler_abi.h create mode 100644 drivers/gpu/drm/xe/abi/uc_fw_abi.h create mode 100644 drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h create mode 100644 drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h create mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_config.h create mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_gtt_view_types.h create mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_reg_defs.h create mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h create mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h create mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_uncore_trace.h create mode 100644 drivers/gpu/drm/xe/display/tests/Makefile create mode 100644 drivers/gpu/drm/xe/display/xe_display.c create mode 100644 drivers/gpu/drm/xe/display/xe_display.h create mode 100644 drivers/gpu/drm/xe/display/xe_display_bo.c create mode 100644 drivers/gpu/drm/xe/display/xe_display_bo.h create mode 100644 drivers/gpu/drm/xe/display/xe_display_pcode.c create mode 100644 drivers/gpu/drm/xe/display/xe_display_pcode.h create mode 100644 drivers/gpu/drm/xe/display/xe_display_rpm.c create mode 100644 drivers/gpu/drm/xe/display/xe_display_rpm.h create mode 100644 drivers/gpu/drm/xe/display/xe_display_vma.h create mode 100644 drivers/gpu/drm/xe/display/xe_display_wa.c create mode 100644 drivers/gpu/drm/xe/display/xe_dsb_buffer.c create mode 100644 drivers/gpu/drm/xe/display/xe_dsb_buffer.h create mode 100644 drivers/gpu/drm/xe/display/xe_fb_pin.c create mode 100644 drivers/gpu/drm/xe/display/xe_fb_pin.h create mode 100644 drivers/gpu/drm/xe/display/xe_frontbuffer.c create mode 100644 drivers/gpu/drm/xe/display/xe_frontbuffer.h create mode 100644 drivers/gpu/drm/xe/display/xe_hdcp_gsc.c create mode 100644 drivers/gpu/drm/xe/display/xe_hdcp_gsc.h create mode 100644 drivers/gpu/drm/xe/display/xe_initial_plane.c create mode 100644 drivers/gpu/drm/xe/display/xe_initial_plane.h create mode 100644 drivers/gpu/drm/xe/display/xe_panic.c create mode 100644 drivers/gpu/drm/xe/display/xe_panic.h create mode 100644 drivers/gpu/drm/xe/display/xe_stolen.c create mode 100644 drivers/gpu/drm/xe/display/xe_stolen.h create mode 100644 drivers/gpu/drm/xe/display/xe_tdf.c create mode 100644 drivers/gpu/drm/xe/instructions/xe_alu_commands.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_gpu_commands.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_gsc_commands.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_instr_defs.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_mfx_commands.h create mode 100644 drivers/gpu/drm/xe/instructions/xe_mi_commands.h create mode 100644 drivers/gpu/drm/xe/regs/xe_bars.h create mode 100644 drivers/gpu/drm/xe/regs/xe_engine_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_gsc_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_gt_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_gtt_defs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_guc_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_hw_error_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_i2c_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_irq_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_lrc_layout.h create mode 100644 drivers/gpu/drm/xe/regs/xe_mchbar_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_mert_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_oa_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_pcode_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_pmt.h create mode 100644 drivers/gpu/drm/xe/regs/xe_pxp_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_reg_defs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_soc_remapper_regs.h create mode 100644 drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h create mode 100644 drivers/gpu/drm/xe/tests/Makefile create mode 100644 drivers/gpu/drm/xe/tests/xe_args_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_bo.c create mode 100644 drivers/gpu/drm/xe/tests/xe_dma_buf.c create mode 100644 drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_db_mgr_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_relay_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_kunit_helpers.c create mode 100644 drivers/gpu/drm/xe/tests/xe_kunit_helpers.h create mode 100644 drivers/gpu/drm/xe/tests/xe_live_test_mod.c create mode 100644 drivers/gpu/drm/xe/tests/xe_lmtt_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_migrate.c create mode 100644 drivers/gpu/drm/xe/tests/xe_mocs.c create mode 100644 drivers/gpu/drm/xe/tests/xe_pci.c create mode 100644 drivers/gpu/drm/xe/tests/xe_pci_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_pci_test.h create mode 100644 drivers/gpu/drm/xe/tests/xe_rtp.c create mode 100644 drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_rtp_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_rtp_test.h create mode 100644 drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c create mode 100644 drivers/gpu/drm/xe/tests/xe_sriov_pf_service_kunit.c create mode 100644 drivers/gpu/drm/xe/tests/xe_test.h create mode 100644 drivers/gpu/drm/xe/tests/xe_test_mod.c create mode 100644 drivers/gpu/drm/xe/tests/xe_wa_test.c create mode 100644 drivers/gpu/drm/xe/xe_amc.c create mode 100644 drivers/gpu/drm/xe/xe_amc.h create mode 100644 drivers/gpu/drm/xe/xe_args.h create mode 100644 drivers/gpu/drm/xe/xe_assert.h create mode 100644 drivers/gpu/drm/xe/xe_bb.c create mode 100644 drivers/gpu/drm/xe/xe_bb.h create mode 100644 drivers/gpu/drm/xe/xe_bb_types.h create mode 100644 drivers/gpu/drm/xe/xe_bo.c create mode 100644 drivers/gpu/drm/xe/xe_bo.h create mode 100644 drivers/gpu/drm/xe/xe_bo_doc.h create mode 100644 drivers/gpu/drm/xe/xe_bo_evict.c create mode 100644 drivers/gpu/drm/xe/xe_bo_evict.h create mode 100644 drivers/gpu/drm/xe/xe_bo_types.h create mode 100644 drivers/gpu/drm/xe/xe_configfs.c create mode 100644 drivers/gpu/drm/xe/xe_configfs.h create mode 100644 drivers/gpu/drm/xe/xe_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_defaults.h create mode 100644 drivers/gpu/drm/xe/xe_dep_job_types.h create mode 100644 drivers/gpu/drm/xe/xe_dep_scheduler.c create mode 100644 drivers/gpu/drm/xe/xe_dep_scheduler.h create mode 100644 drivers/gpu/drm/xe/xe_devcoredump.c create mode 100644 drivers/gpu/drm/xe/xe_devcoredump.h create mode 100644 drivers/gpu/drm/xe/xe_devcoredump_types.h create mode 100644 drivers/gpu/drm/xe/xe_device.c create mode 100644 drivers/gpu/drm/xe/xe_device.h create mode 100644 drivers/gpu/drm/xe/xe_device_sysfs.c create mode 100644 drivers/gpu/drm/xe/xe_device_sysfs.h create mode 100644 drivers/gpu/drm/xe/xe_device_types.h create mode 100644 drivers/gpu/drm/xe/xe_device_wa_oob.rules create mode 100644 drivers/gpu/drm/xe/xe_dma_buf.c create mode 100644 drivers/gpu/drm/xe/xe_dma_buf.h create mode 100644 drivers/gpu/drm/xe/xe_drm_client.c create mode 100644 drivers/gpu/drm/xe/xe_drm_client.h create mode 100644 drivers/gpu/drm/xe/xe_drm_ras.c create mode 100644 drivers/gpu/drm/xe/xe_drm_ras.h create mode 100644 drivers/gpu/drm/xe/xe_drm_ras_types.h create mode 100644 drivers/gpu/drm/xe/xe_drv.h create mode 100644 drivers/gpu/drm/xe/xe_eu_stall.c create mode 100644 drivers/gpu/drm/xe/xe_eu_stall.h create mode 100644 drivers/gpu/drm/xe/xe_exec.c create mode 100644 drivers/gpu/drm/xe/xe_exec.h create mode 100644 drivers/gpu/drm/xe/xe_exec_queue.c create mode 100644 drivers/gpu/drm/xe/xe_exec_queue.h create mode 100644 drivers/gpu/drm/xe/xe_exec_queue_types.h create mode 100644 drivers/gpu/drm/xe/xe_execlist.c create mode 100644 drivers/gpu/drm/xe/xe_execlist.h create mode 100644 drivers/gpu/drm/xe/xe_execlist_types.h create mode 100644 drivers/gpu/drm/xe/xe_force_wake.c create mode 100644 drivers/gpu/drm/xe/xe_force_wake.h create mode 100644 drivers/gpu/drm/xe/xe_force_wake_types.h create mode 100644 drivers/gpu/drm/xe/xe_gen_wa_oob.c create mode 100644 drivers/gpu/drm/xe/xe_ggtt.c create mode 100644 drivers/gpu/drm/xe/xe_ggtt.h create mode 100644 drivers/gpu/drm/xe/xe_ggtt_types.h create mode 100644 drivers/gpu/drm/xe/xe_gpu_scheduler.c create mode 100644 drivers/gpu/drm/xe/xe_gpu_scheduler.h create mode 100644 drivers/gpu/drm/xe/xe_gpu_scheduler_types.h create mode 100644 drivers/gpu/drm/xe/xe_gsc.c create mode 100644 drivers/gpu/drm/xe/xe_gsc.h create mode 100644 drivers/gpu/drm/xe/xe_gsc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_gsc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_gsc_proxy.c create mode 100644 drivers/gpu/drm/xe/xe_gsc_proxy.h create mode 100644 drivers/gpu/drm/xe/xe_gsc_submit.c create mode 100644 drivers/gpu/drm/xe/xe_gsc_submit.h create mode 100644 drivers/gpu/drm/xe/xe_gsc_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt.c create mode 100644 drivers/gpu/drm/xe/xe_gt.h create mode 100644 drivers/gpu/drm/xe/xe_gt_ccs_mode.c create mode 100644 drivers/gpu/drm/xe/xe_gt_ccs_mode.h create mode 100644 drivers/gpu/drm/xe/xe_gt_clock.c create mode 100644 drivers/gpu/drm/xe/xe_gt_clock.h create mode 100644 drivers/gpu/drm/xe/xe_gt_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_gt_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_gt_freq.c create mode 100644 drivers/gpu/drm/xe/xe_gt_freq.h create mode 100644 drivers/gpu/drm/xe/xe_gt_idle.c create mode 100644 drivers/gpu/drm/xe/xe_gt_idle.h create mode 100644 drivers/gpu/drm/xe/xe_gt_idle_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_mcr.c create mode 100644 drivers/gpu/drm/xe/xe_gt_mcr.h create mode 100644 drivers/gpu/drm/xe/xe_gt_printk.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_migration_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_service.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_service_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_printk.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_vf.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_vf.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_stats.c create mode 100644 drivers/gpu/drm/xe/xe_gt_stats.h create mode 100644 drivers/gpu/drm/xe/xe_gt_stats_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sysfs.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sysfs.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sysfs_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_throttle.c create mode 100644 drivers/gpu/drm/xe/xe_gt_throttle.h create mode 100644 drivers/gpu/drm/xe/xe_gt_topology.c create mode 100644 drivers/gpu/drm/xe/xe_gt_topology.h create mode 100644 drivers/gpu/drm/xe/xe_gt_types.h create mode 100644 drivers/gpu/drm/xe/xe_guard.h create mode 100644 drivers/gpu/drm/xe/xe_guc.c create mode 100644 drivers/gpu/drm/xe/xe_guc.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ads.c create mode 100644 drivers/gpu/drm/xe/xe_guc_ads.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ads_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_buf.c create mode 100644 drivers/gpu/drm/xe/xe_guc_buf.h create mode 100644 drivers/gpu/drm/xe/xe_guc_buf_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_capture.c create mode 100644 drivers/gpu/drm/xe/xe_guc_capture.h create mode 100644 drivers/gpu/drm/xe/xe_guc_capture_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ct.c create mode 100644 drivers/gpu/drm/xe/xe_guc_ct.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ct_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_db_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_guc_db_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_guc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_guc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_guc_engine_activity.c create mode 100644 drivers/gpu/drm/xe/xe_guc_engine_activity.h create mode 100644 drivers/gpu/drm/xe/xe_guc_engine_activity_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_exec_queue_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_fwif.h create mode 100644 drivers/gpu/drm/xe/xe_guc_hwconfig.c create mode 100644 drivers/gpu/drm/xe/xe_guc_hwconfig.h create mode 100644 drivers/gpu/drm/xe/xe_guc_hxg_helpers.h create mode 100644 drivers/gpu/drm/xe/xe_guc_id_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_guc_id_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_guc_klv_helpers.c create mode 100644 drivers/gpu/drm/xe/xe_guc_klv_helpers.h create mode 100644 drivers/gpu/drm/xe/xe_guc_klv_thresholds_set.h create mode 100644 drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_log.c create mode 100644 drivers/gpu/drm/xe/xe_guc_log.h create mode 100644 drivers/gpu/drm/xe/xe_guc_log_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_pagefault.c create mode 100644 drivers/gpu/drm/xe/xe_guc_pagefault.h create mode 100644 drivers/gpu/drm/xe/xe_guc_pc.c create mode 100644 drivers/gpu/drm/xe/xe_guc_pc.h create mode 100644 drivers/gpu/drm/xe/xe_guc_pc_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_rc.c create mode 100644 drivers/gpu/drm/xe/xe_guc_rc.h create mode 100644 drivers/gpu/drm/xe/xe_guc_relay.c create mode 100644 drivers/gpu/drm/xe/xe_guc_relay.h create mode 100644 drivers/gpu/drm/xe/xe_guc_relay_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_submit.c create mode 100644 drivers/gpu/drm/xe/xe_guc_submit.h create mode 100644 drivers/gpu/drm/xe/xe_guc_submit_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_tlb_inval.c create mode 100644 drivers/gpu/drm/xe/xe_guc_tlb_inval.h create mode 100644 drivers/gpu/drm/xe/xe_guc_types.h create mode 100644 drivers/gpu/drm/xe/xe_heci_gsc.c create mode 100644 drivers/gpu/drm/xe/xe_heci_gsc.h create mode 100644 drivers/gpu/drm/xe/xe_huc.c create mode 100644 drivers/gpu/drm/xe/xe_huc.h create mode 100644 drivers/gpu/drm/xe/xe_huc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_huc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_huc_types.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine.c create mode 100644 drivers/gpu/drm/xe/xe_hw_engine.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_group.c create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_group.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_group_types.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_types.h create mode 100644 drivers/gpu/drm/xe/xe_hw_error.c create mode 100644 drivers/gpu/drm/xe/xe_hw_error.h create mode 100644 drivers/gpu/drm/xe/xe_hw_fence.c create mode 100644 drivers/gpu/drm/xe/xe_hw_fence.h create mode 100644 drivers/gpu/drm/xe/xe_hw_fence_types.h create mode 100644 drivers/gpu/drm/xe/xe_hwmon.c create mode 100644 drivers/gpu/drm/xe/xe_hwmon.h create mode 100644 drivers/gpu/drm/xe/xe_i2c.c create mode 100644 drivers/gpu/drm/xe/xe_i2c.h create mode 100644 drivers/gpu/drm/xe/xe_irq.c create mode 100644 drivers/gpu/drm/xe/xe_irq.h create mode 100644 drivers/gpu/drm/xe/xe_late_bind_fw.c create mode 100644 drivers/gpu/drm/xe/xe_late_bind_fw.h create mode 100644 drivers/gpu/drm/xe/xe_late_bind_fw_types.h create mode 100644 drivers/gpu/drm/xe/xe_lmtt.c create mode 100644 drivers/gpu/drm/xe/xe_lmtt.h create mode 100644 drivers/gpu/drm/xe/xe_lmtt_2l.c create mode 100644 drivers/gpu/drm/xe/xe_lmtt_ml.c create mode 100644 drivers/gpu/drm/xe/xe_lmtt_types.h create mode 100644 drivers/gpu/drm/xe/xe_lrc.c create mode 100644 drivers/gpu/drm/xe/xe_lrc.h create mode 100644 drivers/gpu/drm/xe/xe_lrc_types.h create mode 100644 drivers/gpu/drm/xe/xe_macros.h create mode 100644 drivers/gpu/drm/xe/xe_map.h create mode 100644 drivers/gpu/drm/xe/xe_mem_pool.c create mode 100644 drivers/gpu/drm/xe/xe_mem_pool.h create mode 100644 drivers/gpu/drm/xe/xe_mem_pool_types.h create mode 100644 drivers/gpu/drm/xe/xe_memirq.c create mode 100644 drivers/gpu/drm/xe/xe_memirq.h create mode 100644 drivers/gpu/drm/xe/xe_memirq_types.h create mode 100644 drivers/gpu/drm/xe/xe_mert.c create mode 100644 drivers/gpu/drm/xe/xe_mert.h create mode 100644 drivers/gpu/drm/xe/xe_migrate.c create mode 100644 drivers/gpu/drm/xe/xe_migrate.h create mode 100644 drivers/gpu/drm/xe/xe_migrate_doc.h create mode 100644 drivers/gpu/drm/xe/xe_mmio.c create mode 100644 drivers/gpu/drm/xe/xe_mmio.h create mode 100644 drivers/gpu/drm/xe/xe_mmio_gem.c create mode 100644 drivers/gpu/drm/xe/xe_mmio_gem.h create mode 100644 drivers/gpu/drm/xe/xe_mmio_types.h create mode 100644 drivers/gpu/drm/xe/xe_mocs.c create mode 100644 drivers/gpu/drm/xe/xe_mocs.h create mode 100644 drivers/gpu/drm/xe/xe_module.c create mode 100644 drivers/gpu/drm/xe/xe_module.h create mode 100644 drivers/gpu/drm/xe/xe_nvm.c create mode 100644 drivers/gpu/drm/xe/xe_nvm.h create mode 100644 drivers/gpu/drm/xe/xe_oa.c create mode 100644 drivers/gpu/drm/xe/xe_oa.h create mode 100644 drivers/gpu/drm/xe/xe_oa_types.h create mode 100644 drivers/gpu/drm/xe/xe_observation.c create mode 100644 drivers/gpu/drm/xe/xe_observation.h create mode 100644 drivers/gpu/drm/xe/xe_page_reclaim.c create mode 100644 drivers/gpu/drm/xe/xe_page_reclaim.h create mode 100644 drivers/gpu/drm/xe/xe_pagefault.c create mode 100644 drivers/gpu/drm/xe/xe_pagefault.h create mode 100644 drivers/gpu/drm/xe/xe_pagefault_types.h create mode 100644 drivers/gpu/drm/xe/xe_pat.c create mode 100644 drivers/gpu/drm/xe/xe_pat.h create mode 100644 drivers/gpu/drm/xe/xe_pci.c create mode 100644 drivers/gpu/drm/xe/xe_pci.h create mode 100644 drivers/gpu/drm/xe/xe_pci_error.c create mode 100644 drivers/gpu/drm/xe/xe_pci_error.h create mode 100644 drivers/gpu/drm/xe/xe_pci_rebar.c create mode 100644 drivers/gpu/drm/xe/xe_pci_rebar.h create mode 100644 drivers/gpu/drm/xe/xe_pci_sriov.c create mode 100644 drivers/gpu/drm/xe/xe_pci_sriov.h create mode 100644 drivers/gpu/drm/xe/xe_pci_types.h create mode 100644 drivers/gpu/drm/xe/xe_pcode.c create mode 100644 drivers/gpu/drm/xe/xe_pcode.h create mode 100644 drivers/gpu/drm/xe/xe_pcode_api.h create mode 100644 drivers/gpu/drm/xe/xe_platform_types.h create mode 100644 drivers/gpu/drm/xe/xe_pm.c create mode 100644 drivers/gpu/drm/xe/xe_pm.h create mode 100644 drivers/gpu/drm/xe/xe_pmu.c create mode 100644 drivers/gpu/drm/xe/xe_pmu.h create mode 100644 drivers/gpu/drm/xe/xe_pmu_types.h create mode 100644 drivers/gpu/drm/xe/xe_preempt_fence.c create mode 100644 drivers/gpu/drm/xe/xe_preempt_fence.h create mode 100644 drivers/gpu/drm/xe/xe_preempt_fence_types.h create mode 100644 drivers/gpu/drm/xe/xe_printk.h create mode 100644 drivers/gpu/drm/xe/xe_psmi.c create mode 100644 drivers/gpu/drm/xe/xe_psmi.h create mode 100644 drivers/gpu/drm/xe/xe_pt.c create mode 100644 drivers/gpu/drm/xe/xe_pt.h create mode 100644 drivers/gpu/drm/xe/xe_pt_types.h create mode 100644 drivers/gpu/drm/xe/xe_pt_walk.c create mode 100644 drivers/gpu/drm/xe/xe_pt_walk.h create mode 100644 drivers/gpu/drm/xe/xe_pxp.c create mode 100644 drivers/gpu/drm/xe/xe_pxp.h create mode 100644 drivers/gpu/drm/xe/xe_pxp_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_pxp_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_pxp_submit.c create mode 100644 drivers/gpu/drm/xe/xe_pxp_submit.h create mode 100644 drivers/gpu/drm/xe/xe_pxp_types.h create mode 100644 drivers/gpu/drm/xe/xe_query.c create mode 100644 drivers/gpu/drm/xe/xe_query.h create mode 100644 drivers/gpu/drm/xe/xe_range_fence.c create mode 100644 drivers/gpu/drm/xe/xe_range_fence.h create mode 100644 drivers/gpu/drm/xe/xe_ras.c create mode 100644 drivers/gpu/drm/xe/xe_ras.h create mode 100644 drivers/gpu/drm/xe/xe_ras_types.h create mode 100644 drivers/gpu/drm/xe/xe_reg_sr.c create mode 100644 drivers/gpu/drm/xe/xe_reg_sr.h create mode 100644 drivers/gpu/drm/xe/xe_reg_sr_types.h create mode 100644 drivers/gpu/drm/xe/xe_reg_whitelist.c create mode 100644 drivers/gpu/drm/xe/xe_reg_whitelist.h create mode 100644 drivers/gpu/drm/xe/xe_res_cursor.h create mode 100644 drivers/gpu/drm/xe/xe_ring_ops.c create mode 100644 drivers/gpu/drm/xe/xe_ring_ops.h create mode 100644 drivers/gpu/drm/xe/xe_ring_ops_types.h create mode 100644 drivers/gpu/drm/xe/xe_rtp.c create mode 100644 drivers/gpu/drm/xe/xe_rtp.h create mode 100644 drivers/gpu/drm/xe/xe_rtp_helpers.h create mode 100644 drivers/gpu/drm/xe/xe_rtp_types.h create mode 100644 drivers/gpu/drm/xe/xe_sa.c create mode 100644 drivers/gpu/drm/xe/xe_sa.h create mode 100644 drivers/gpu/drm/xe/xe_sa_types.h create mode 100644 drivers/gpu/drm/xe/xe_sched_job.c create mode 100644 drivers/gpu/drm/xe/xe_sched_job.h create mode 100644 drivers/gpu/drm/xe/xe_sched_job_types.h create mode 100644 drivers/gpu/drm/xe/xe_shrinker.c create mode 100644 drivers/gpu/drm/xe/xe_shrinker.h create mode 100644 drivers/gpu/drm/xe/xe_sleep.h create mode 100644 drivers/gpu/drm/xe/xe_soc_remapper.c create mode 100644 drivers/gpu/drm/xe/xe_soc_remapper.h create mode 100644 drivers/gpu/drm/xe/xe_sriov.c create mode 100644 drivers/gpu/drm/xe/xe_sriov.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_packet.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_packet.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_packet_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_control.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_control.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_helpers.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_migration.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_migration.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_provision.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_provision.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_provision_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_service.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_service.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_service_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_sysfs.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_pf_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_printk.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_ccs.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_ccs.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_types.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_vfio.c create mode 100644 drivers/gpu/drm/xe/xe_step.c create mode 100644 drivers/gpu/drm/xe/xe_step.h create mode 100644 drivers/gpu/drm/xe/xe_step_types.h create mode 100644 drivers/gpu/drm/xe/xe_survivability_mode.c create mode 100644 drivers/gpu/drm/xe/xe_survivability_mode.h create mode 100644 drivers/gpu/drm/xe/xe_survivability_mode_types.h create mode 100644 drivers/gpu/drm/xe/xe_svm.c create mode 100644 drivers/gpu/drm/xe/xe_svm.h create mode 100644 drivers/gpu/drm/xe/xe_sync.c create mode 100644 drivers/gpu/drm/xe/xe_sync.h create mode 100644 drivers/gpu/drm/xe/xe_sync_types.h create mode 100644 drivers/gpu/drm/xe/xe_sysctrl.c create mode 100644 drivers/gpu/drm/xe/xe_sysctrl.h create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_event.c create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_event_types.h create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_mailbox.c create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_mailbox.h create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_types.h create mode 100644 drivers/gpu/drm/xe/xe_tile.c create mode 100644 drivers/gpu/drm/xe/xe_tile.h create mode 100644 drivers/gpu/drm/xe/xe_tile_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_tile_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_tile_printk.h create mode 100644 drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_tile_sriov_printk.h create mode 100644 drivers/gpu/drm/xe/xe_tile_sriov_vf.c create mode 100644 drivers/gpu/drm/xe/xe_tile_sriov_vf.h create mode 100644 drivers/gpu/drm/xe/xe_tile_sriov_vf_types.h create mode 100644 drivers/gpu/drm/xe/xe_tile_sysfs.c create mode 100644 drivers/gpu/drm/xe/xe_tile_sysfs.h create mode 100644 drivers/gpu/drm/xe/xe_tile_sysfs_types.h create mode 100644 drivers/gpu/drm/xe/xe_tile_types.h create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval.c create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval.h create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval_job.c create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval_job.h create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval_types.h create mode 100644 drivers/gpu/drm/xe/xe_trace.c create mode 100644 drivers/gpu/drm/xe/xe_trace.h create mode 100644 drivers/gpu/drm/xe/xe_trace_bo.c create mode 100644 drivers/gpu/drm/xe/xe_trace_bo.h create mode 100644 drivers/gpu/drm/xe/xe_trace_guc.c create mode 100644 drivers/gpu/drm/xe/xe_trace_guc.h create mode 100644 drivers/gpu/drm/xe/xe_trace_lrc.c create mode 100644 drivers/gpu/drm/xe/xe_trace_lrc.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_sys_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_ttm_sys_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_vram_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_ttm_vram_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h create mode 100644 drivers/gpu/drm/xe/xe_tuning.c create mode 100644 drivers/gpu/drm/xe/xe_tuning.h create mode 100644 drivers/gpu/drm/xe/xe_uc.c create mode 100644 drivers/gpu/drm/xe/xe_uc.h create mode 100644 drivers/gpu/drm/xe/xe_uc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_uc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_uc_fw.c create mode 100644 drivers/gpu/drm/xe/xe_uc_fw.h create mode 100644 drivers/gpu/drm/xe/xe_uc_fw_types.h create mode 100644 drivers/gpu/drm/xe/xe_uc_types.h create mode 100644 drivers/gpu/drm/xe/xe_userptr.c create mode 100644 drivers/gpu/drm/xe/xe_userptr.h create mode 100644 drivers/gpu/drm/xe/xe_validation.c create mode 100644 drivers/gpu/drm/xe/xe_validation.h create mode 100644 drivers/gpu/drm/xe/xe_vm.c create mode 100644 drivers/gpu/drm/xe/xe_vm.h create mode 100644 drivers/gpu/drm/xe/xe_vm_doc.h create mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.c create mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.h create mode 100644 drivers/gpu/drm/xe/xe_vm_types.h create mode 100644 drivers/gpu/drm/xe/xe_vram.c create mode 100644 drivers/gpu/drm/xe/xe_vram.h create mode 100644 drivers/gpu/drm/xe/xe_vram_freq.c create mode 100644 drivers/gpu/drm/xe/xe_vram_freq.h create mode 100644 drivers/gpu/drm/xe/xe_vram_types.h create mode 100644 drivers/gpu/drm/xe/xe_vsec.c create mode 100644 drivers/gpu/drm/xe/xe_vsec.h create mode 100644 drivers/gpu/drm/xe/xe_wa.c create mode 100644 drivers/gpu/drm/xe/xe_wa.h create mode 100644 drivers/gpu/drm/xe/xe_wa_oob.rules create mode 100644 drivers/gpu/drm/xe/xe_wait_user_fence.c create mode 100644 drivers/gpu/drm/xe/xe_wait_user_fence.h create mode 100644 drivers/gpu/drm/xe/xe_wopcm.c create mode 100644 drivers/gpu/drm/xe/xe_wopcm.h create mode 100644 drivers/gpu/drm/xe/xe_wopcm_types.h (limited to 'drivers/gpu/drm/xe') diff --git a/drivers/gpu/drm/xe/.gitignore b/drivers/gpu/drm/xe/.gitignore new file mode 100644 index 000000000..6dad8a5a2 --- /dev/null +++ b/drivers/gpu/drm/xe/.gitignore @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +*.hdrtest +/generated +/xe_gen_wa_oob +!.kunitconfig-display diff --git a/drivers/gpu/drm/xe/.kunitconfig b/drivers/gpu/drm/xe/.kunitconfig new file mode 100644 index 000000000..ad4b9b4a9 --- /dev/null +++ b/drivers/gpu/drm/xe/.kunitconfig @@ -0,0 +1,18 @@ +# xe dependencies +CONFIG_KUNIT=y +CONFIG_PCI=y +CONFIG_PCI_IOV=y +CONFIG_DEBUG_FS=y +CONFIG_DRM=y +CONFIG_DRM_FBDEV_EMULATION=y +CONFIG_DRM_KMS_HELPER=y +CONFIG_DRM_XE=y +CONFIG_DRM_XE_DISPLAY=n +CONFIG_EXPERT=y +CONFIG_FB=y +CONFIG_DRM_XE_KUNIT_TEST=y +CONFIG_LOCK_DEBUGGING_SUPPORT=y +CONFIG_PROVE_LOCKING=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_LOCKDEP=y +CONFIG_DEBUG_LOCKDEP=y diff --git a/drivers/gpu/drm/xe/.kunitconfig-display b/drivers/gpu/drm/xe/.kunitconfig-display new file mode 100644 index 000000000..17020aa4d --- /dev/null +++ b/drivers/gpu/drm/xe/.kunitconfig-display @@ -0,0 +1,11 @@ +CONFIG_EXPERT=y +CONFIG_MODULES=y +CONFIG_KUNIT=y +CONFIG_PCI=y +CONFIG_DEBUG_FS=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DRM=m +CONFIG_DRM_XE=m +CONFIG_DRM_XE_DISPLAY=y +CONFIG_DRM_XE_KUNIT_TEST=m diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig new file mode 100644 index 000000000..4d7dcaff2 --- /dev/null +++ b/drivers/gpu/drm/xe/Kconfig @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DRM_XE + tristate "Intel Xe2 Graphics" + depends on DRM && PCI + depends on KUNIT || !KUNIT + depends on INTEL_VSEC || !INTEL_VSEC + depends on X86_PLATFORM_DEVICES || !(X86 && ACPI) + depends on PAGE_SIZE_4KB || COMPILE_TEST || BROKEN + select INTERVAL_TREE + # we need shmfs for the swappable backing store, and in particular + # the shmem_readpage() which depends upon tmpfs + select SHMEM + select TMPFS + select DRM_BUDDY + select DRM_CLIENT_SELECTION + select DRM_KMS_HELPER + select DRM_KUNIT_TEST_HELPERS if DRM_XE_KUNIT_TEST != n + select DRM_PANEL + select DRM_SUBALLOC_HELPER + select DRM_DISPLAY_DP_HELPER + select DRM_DISPLAY_DSC_HELPER + select DRM_DISPLAY_HDCP_HELPER + select DRM_DISPLAY_HDMI_HELPER + select DRM_DISPLAY_HELPER + select DRM_MIPI_DSI + select RELAY + select IRQ_WORK + # xe depends on ACPI_VIDEO when ACPI is enabled + # but for select to work, need to select ACPI_VIDEO's dependencies, ick + select BACKLIGHT_CLASS_DEVICE if ACPI + select INPUT if ACPI + select ACPI_VIDEO if X86 && ACPI + select ACPI_WMI if X86 && ACPI + select SYNC_FILE + select CRC32 + select SND_HDA_I915 if SND_HDA_CORE + select CEC_CORE if CEC_NOTIFIER + select VMAP_PFN + select DRM_TTM + select DRM_TTM_HELPER + select DRM_EXEC + select DRM_GPUSVM if !UML + select DRM_GPUVM + select DRM_SCHED + select MMU_NOTIFIER + select WANT_DEV_COREDUMP + select AUXILIARY_BUS + select REGMAP if I2C + help + Driver for Intel Xe2 series GPUs and later. Experimental support + for Xe series is also available. + + If "M" is selected, the module will be called xe. + +config DRM_XE_DISPLAY + bool "Enable display support" + depends on DRM_XE && DRM_XE=m && HAS_IOPORT + select FB_IOMEM_HELPERS if DRM_FBDEV_EMULATION + select I2C + select I2C_ALGOBIT + default y + help + Disable this option only if you want to compile out display support. + +config DRM_XE_DP_TUNNEL + bool "Enable DP tunnel support" + depends on DRM_XE_DISPLAY + depends on USB4 + select DRM_DISPLAY_DP_TUNNEL + default y + help + Choose this option to detect DP tunnels and enable the Bandwidth + Allocation mode for such tunnels. This allows using the maximum + resolution allowed by the link BW on all displays sharing the + link BW, for instance on a Thunderbolt link. + + If in doubt say "Y". + +config DRM_XE_GPUSVM + bool "Enable CPU to GPU address mirroring" + depends on DRM_XE + depends on !UML + depends on ZONE_DEVICE + default y + select DEVICE_PRIVATE + select DRM_GPUSVM + help + Enable this option if you want support for CPU to GPU address + mirroring. + + If in doubt say "Y". + +config DRM_XE_PAGEMAP + bool "Enable device memory pool for SVM" + depends on DRM_XE_GPUSVM + select GET_FREE_REGION + default y + help + Disable this option only if you don't want to expose local device + memory for SVM. Will reduce KMD memory footprint when disabled. + + If in doubt say "Y". + +config DRM_XE_FORCE_PROBE + string "Force probe xe for selected Intel hardware IDs" + depends on DRM_XE + help + This is the default value for the xe.force_probe module + parameter. Using the module parameter overrides this option. + + Force probe the xe for Intel graphics devices that are + recognized but not properly supported by this kernel version. It is + recommended to upgrade to a kernel version with proper support as soon + as it is available. + + It can also be used to block the probe of recognized and fully + supported devices. + + Use "" to disable force probe. If in doubt, use this. + + Use "[,,...]" to force probe the xe for listed + devices. For example, "4500" or "4500,4571". + + Use "*" to force probe the driver for all known devices. + + Use "!" right before the ID to block the probe of the device. For + example, "4500,!4571" forces the probe of 4500 and blocks the probe of + 4571. + + Use "!*" to block the probe of the driver for all known devices. + +menu "drm/Xe Debugging" +depends on DRM_XE +depends on EXPERT +source "drivers/gpu/drm/xe/Kconfig.debug" +endmenu + +menu "drm/xe Profile Guided Optimisation" + visible if EXPERT + depends on DRM_XE + source "drivers/gpu/drm/xe/Kconfig.profile" +endmenu diff --git a/drivers/gpu/drm/xe/Kconfig.debug b/drivers/gpu/drm/xe/Kconfig.debug new file mode 100644 index 000000000..79118d9ef --- /dev/null +++ b/drivers/gpu/drm/xe/Kconfig.debug @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DRM_XE_WERROR + bool "Force GCC to throw an error instead of a warning when compiling" + # As this may inadvertently break the build, only allow the user + # to shoot oneself in the foot iff they aim really hard + depends on EXPERT + # We use the dependency on !COMPILE_TEST to not be enabled in + # allmodconfig or allyesconfig configurations + depends on !COMPILE_TEST + default n + help + Add -Werror to the build flags for (and only for) xe.ko. + Do not enable this unless you are writing code for the xe.ko module. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG + bool "Enable additional driver debugging" + depends on DRM_XE + depends on EXPERT + depends on !COMPILE_TEST + default n + help + Choose this option to turn on extra driver debugging that may affect + performance but will catch some internal issues. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_VM + bool "Enable extra VM debugging info" + default n + help + Enable extra VM debugging info + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_SRIOV + bool "Enable extra SR-IOV debugging" + default n + imply DRM_XE_DEBUG_MEMIRQ + help + Enable extra SR-IOV debugging info. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_MEMIRQ + bool "Enable extra memirq debugging" + default n + help + Choose this option to enable additional debugging info for + memory based interrupts. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_MEM + bool "Enable passing SYS/VRAM addresses to user space" + default n + help + Pass object location through uapi. Intended for extended + testing and development only. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_KUNIT_TEST + tristate "KUnit tests for the drm xe driver" if !KUNIT_ALL_TESTS + depends on DRM_XE && KUNIT && DEBUG_FS + default KUNIT_ALL_TESTS + select DRM_EXPORT_FOR_TESTS if m + help + Choose this option to allow the driver to perform selftests under + the kunit framework + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_PAGE_SIZE + bool "Enable debug control for user BO page-size allocation" + depends on DRM_XE_DEBUG && DEBUG_FS + help + Expose a debugfs knob to override user BO page-size allocation + handling for validation and debug. Supported modes include forced + 2M, forced 1G, and a mixed mode that exercises 4K, 64K, 2M, and + 1G page-size paths on platforms that support them. + + This is an unstable debugfs interface intended for development and + validation only. Its layout, contents, and existence may change or + be removed at any time with no regression warranty. + + Recommended for driver developers only. + If in doubt, say "N". + +config DRM_XE_DEBUG_GUC + bool "Enable extra GuC related debug options" + depends on DRM_XE_DEBUG + default n + select STACKDEPOT + help + Choose this option when debugging guc issues. + The GuC log buffer is increased to the maximum allowed, which should + be large enough for complex issues. The tracking of FAST_REQ messages + is extended to include a record of the calling stack, which is then + dumped on a FAST_REQ error notification. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_USERPTR_INVAL_INJECT + bool "Inject userptr invalidation -EINVAL errors" + depends on DRM_GPUSVM + default n + help + Choose this option when debugging error paths that + are hit during checks for userptr invalidations. + + Recommended for driver developers only. + If in doubt, say "N". diff --git a/drivers/gpu/drm/xe/Kconfig.profile b/drivers/gpu/drm/xe/Kconfig.profile new file mode 100644 index 000000000..e07517d12 --- /dev/null +++ b/drivers/gpu/drm/xe/Kconfig.profile @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DRM_XE_JOB_TIMEOUT_MAX + int "Hard upper limit for job timeout (ms)" + default 10000 # milliseconds + help + Absolute upper bound (in milliseconds) for the per-engine-class job + timeout. This is the maximum value that can be written to the sysfs + job_timeout_ms knob, regardless of privileges. To raise this ceiling, + increase this value and rebuild the kernel. +config DRM_XE_JOB_TIMEOUT_MIN + int "Hard lower limit for job timeout (ms)" + default 1 # milliseconds + help + Absolute lower bound (in milliseconds) for the per-engine-class job + timeout. This is the minimum value that can be written to the sysfs + job_timeout_ms knob, regardless of privileges. + + Note: the job timeout default (5000 ms) is hardcoded in the driver + and is not configurable here. Use the sysfs job_timeout_ms knob at + runtime to change the engine-class default. +config DRM_XE_TIMESLICE_MAX + int "Hard upper limit for timeslice duration (us)" + default 10000000 # microseconds + help + Absolute upper bound (in microseconds) for the timeslice duration. + This caps both the sysfs timeslice_duration_us knob and the value + accepted via the DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE UAPI for + processes with CAP_SYS_NICE when DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT + is enabled. +config DRM_XE_TIMESLICE_MIN + int "Hard lower limit for timeslice duration (us)" + default 1 # microseconds + help + Absolute lower bound (in microseconds) for the timeslice duration. + This caps both the sysfs timeslice_duration_us knob and the value + accepted via the DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE UAPI for + processes with CAP_SYS_NICE when DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT + is enabled. +config DRM_XE_PREEMPT_TIMEOUT + int "Default preempt timeout (us, jiffy granularity)" + default 640000 # microseconds + help + Initial per-engine-class preemption timeout (in microseconds). This + is the value the driver programs at boot; it can be changed at + runtime via the sysfs preempt_timeout_us knob. + + This is how long the driver waits for the current context to reach + an arbitration point and yield the GPU voluntarily when a + higher-priority context becomes runnable. If the context does not + yield before the timer expires, the HW is reset to allow the + higher-priority context to execute. + + The range userspace may write via sysfs is bounded by + DRM_XE_PREEMPT_TIMEOUT_MIN and DRM_XE_PREEMPT_TIMEOUT_MAX. +config DRM_XE_PREEMPT_TIMEOUT_MAX + int "Hard upper limit for preempt timeout (us)" + default 10000000 # microseconds + help + Absolute upper bound (in microseconds) for the per-engine-class + preemption timeout. This is the maximum value that can be written to + the sysfs preempt_timeout_us knob, regardless of privileges. +config DRM_XE_PREEMPT_TIMEOUT_MIN + int "Hard lower limit for preempt timeout (us)" + default 1 # microseconds + help + Absolute lower bound (in microseconds) for the per-engine-class + preemption timeout. This is the minimum value that can be written to + the sysfs preempt_timeout_us knob, regardless of privileges. +config DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT + bool "Default configuration of limitation on scheduler timeout" + default y + help + Configures the enablement of limitation on scheduler timeout + to apply to applicable user. For elevated user, all above MIN + and MAX values will apply when this configuration is enable to + apply limitation. By default limitation is applied. diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile new file mode 100644 index 000000000..c92468cb9 --- /dev/null +++ b/drivers/gpu/drm/xe/Makefile @@ -0,0 +1,400 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the drm device driver. This driver provides support for the +# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. + +# Enable W=1 warnings not enabled in drm subsystem Makefile +subdir-ccflags-y += $(call cc-option, -Wformat-truncation) + +# Enable -Werror in CI and development +subdir-ccflags-$(CONFIG_DRM_XE_WERROR) += -Werror + +subdir-ccflags-y += -I$(obj) -I$(src) + +# generated sources + +hostprogs := xe_gen_wa_oob +generated_oob := $(obj)/generated/xe_wa_oob.c $(obj)/generated/xe_wa_oob.h +quiet_cmd_wa_oob = GEN $(notdir $(generated_oob)) + cmd_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_wa_oob.rules $(generated_oob) +$(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \ + $(src)/xe_wa_oob.rules + $(call cmd,wa_oob) + +generated_device_oob := $(obj)/generated/xe_device_wa_oob.c $(obj)/generated/xe_device_wa_oob.h +quiet_cmd_device_wa_oob = GEN $(notdir $(generated_device_oob)) + cmd_device_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_device_wa_oob.rules $(generated_device_oob) +$(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe_gen_wa_oob \ + $(src)/xe_device_wa_oob.rules + $(call cmd,device_wa_oob) + +# Please keep these build lists sorted! + +# core driver code + +xe-y += xe_bb.o \ + xe_bo.o \ + xe_bo_evict.o \ + xe_dep_scheduler.o \ + xe_devcoredump.o \ + xe_device.o \ + xe_device_sysfs.o \ + xe_dma_buf.o \ + xe_drm_client.o \ + xe_drm_ras.o \ + xe_eu_stall.o \ + xe_exec.o \ + xe_exec_queue.o \ + xe_execlist.o \ + xe_force_wake.o \ + xe_ggtt.o \ + xe_gpu_scheduler.o \ + xe_gsc.o \ + xe_gsc_debugfs.o \ + xe_gsc_proxy.o \ + xe_gsc_submit.o \ + xe_gt.o \ + xe_gt_ccs_mode.o \ + xe_gt_clock.o \ + xe_gt_freq.o \ + xe_gt_idle.o \ + xe_gt_mcr.o \ + xe_gt_sysfs.o \ + xe_gt_throttle.o \ + xe_gt_topology.o \ + xe_guc.o \ + xe_guc_ads.o \ + xe_guc_buf.o \ + xe_guc_capture.o \ + xe_guc_ct.o \ + xe_guc_db_mgr.o \ + xe_guc_engine_activity.o \ + xe_guc_hwconfig.o \ + xe_guc_id_mgr.o \ + xe_guc_klv_helpers.o \ + xe_guc_log.o \ + xe_guc_pagefault.o \ + xe_guc_pc.o \ + xe_guc_rc.o \ + xe_guc_submit.o \ + xe_guc_tlb_inval.o \ + xe_heci_gsc.o \ + xe_huc.o \ + xe_hw_engine.o \ + xe_hw_engine_class_sysfs.o \ + xe_hw_engine_group.o \ + xe_hw_error.o \ + xe_hw_fence.o \ + xe_irq.o \ + xe_late_bind_fw.o \ + xe_lrc.o \ + xe_mem_pool.o \ + xe_migrate.o \ + xe_mmio.o \ + xe_mmio_gem.o \ + xe_mocs.o \ + xe_module.o \ + xe_nvm.o \ + xe_oa.o \ + xe_observation.o \ + xe_pagefault.o \ + xe_page_reclaim.o \ + xe_pat.o \ + xe_pci.o \ + xe_pci_error.o \ + xe_pci_rebar.o \ + xe_pcode.o \ + xe_pm.o \ + xe_preempt_fence.o \ + xe_psmi.o \ + xe_pt.o \ + xe_pt_walk.o \ + xe_pxp.o \ + xe_pxp_debugfs.o \ + xe_pxp_submit.o \ + xe_query.o \ + xe_range_fence.o \ + xe_ras.o \ + xe_reg_sr.o \ + xe_reg_whitelist.o \ + xe_ring_ops.o \ + xe_rtp.o \ + xe_sa.o \ + xe_sched_job.o \ + xe_shrinker.o \ + xe_soc_remapper.o \ + xe_step.o \ + xe_survivability_mode.o \ + xe_sync.o \ + xe_sysctrl.o \ + xe_sysctrl_event.o \ + xe_sysctrl_mailbox.o \ + xe_tile.o \ + xe_tile_sysfs.o \ + xe_tlb_inval.o \ + xe_tlb_inval_job.o \ + xe_trace.o \ + xe_trace_bo.o \ + xe_trace_guc.o \ + xe_trace_lrc.o \ + xe_ttm_stolen_mgr.o \ + xe_ttm_sys_mgr.o \ + xe_ttm_vram_mgr.o \ + xe_tuning.o \ + xe_uc.o \ + xe_uc_fw.o \ + xe_validation.o \ + xe_vm.o \ + xe_vm_madvise.o \ + xe_vram.o \ + xe_vram_freq.o \ + xe_vsec.o \ + xe_wa.o \ + xe_wait_user_fence.o \ + xe_wopcm.o + +xe-$(CONFIG_I2C) += xe_i2c.o \ + xe_amc.o + +xe-$(CONFIG_DRM_XE_GPUSVM) += xe_svm.o +xe-$(CONFIG_DRM_GPUSVM) += xe_userptr.o + +# graphics hardware monitoring (HWMON) support +xe-$(CONFIG_HWMON) += xe_hwmon.o + +xe-$(CONFIG_PERF_EVENTS) += xe_pmu.o +xe-$(CONFIG_CONFIGFS_FS) += xe_configfs.o + +# graphics virtualization (SR-IOV) support +xe-y += \ + xe_gt_sriov_vf.o \ + xe_guc_relay.o \ + xe_memirq.o \ + xe_sriov.o \ + xe_sriov_vf.o \ + xe_sriov_vf_ccs.o \ + xe_tile_sriov_vf.o + +xe-$(CONFIG_PCI_IOV) += \ + xe_gt_sriov_pf.o \ + xe_gt_sriov_pf_config.o \ + xe_gt_sriov_pf_control.o \ + xe_gt_sriov_pf_migration.o \ + xe_gt_sriov_pf_monitor.o \ + xe_gt_sriov_pf_policy.o \ + xe_gt_sriov_pf_service.o \ + xe_lmtt.o \ + xe_lmtt_2l.o \ + xe_lmtt_ml.o \ + xe_mert.o \ + xe_pci_sriov.o \ + xe_sriov_packet.o \ + xe_sriov_pf.o \ + xe_sriov_pf_control.o \ + xe_sriov_pf_debugfs.o \ + xe_sriov_pf_migration.o \ + xe_sriov_pf_provision.o \ + xe_sriov_pf_service.o \ + xe_sriov_pf_sysfs.o \ + xe_tile_sriov_pf_debugfs.o + +ifdef CONFIG_XE_VFIO_PCI + xe-$(CONFIG_PCI_IOV) += xe_sriov_vfio.o +endif + +# include helpers for tests even when XE is built-in +ifdef CONFIG_DRM_XE_KUNIT_TEST +xe-y += tests/xe_kunit_helpers.o +endif + +# i915 Display compat #defines and #includes +subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \ + -I$(src)/compat-i915-headers \ + -I$(srctree)/drivers/gpu/drm/i915/display/ + +# Rule to build display code shared with i915 +$(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE + $(call cmd,force_checksrc) + $(call if_changed_rule,cc_o_c) + +# Display code specific to xe +xe-$(CONFIG_DRM_XE_DISPLAY) += \ + display/xe_display.o \ + display/xe_display_bo.o \ + display/xe_display_pcode.o \ + display/xe_display_rpm.o \ + display/xe_display_wa.o \ + display/xe_dsb_buffer.o \ + display/xe_fb_pin.o \ + display/xe_frontbuffer.o \ + display/xe_hdcp_gsc.o \ + display/xe_initial_plane.o \ + display/xe_panic.o \ + display/xe_stolen.o \ + display/xe_tdf.o + +# Display code shared with i915 +xe-$(CONFIG_DRM_XE_DISPLAY) += \ + i915-display/icl_dsi.o \ + i915-display/intel_alpm.o \ + i915-display/intel_atomic.o \ + i915-display/intel_audio.o \ + i915-display/intel_backlight.o \ + i915-display/intel_bios.o \ + i915-display/intel_bo.o \ + i915-display/intel_bw.o \ + i915-display/intel_casf.o \ + i915-display/intel_cdclk.o \ + i915-display/intel_cmtg.o \ + i915-display/intel_color.o \ + i915-display/intel_colorop.o \ + i915-display/intel_color_pipeline.o \ + i915-display/intel_combo_phy.o \ + i915-display/intel_connector.o \ + i915-display/intel_crtc.o \ + i915-display/intel_crtc_state_dump.o \ + i915-display/intel_cursor.o \ + i915-display/intel_cx0_phy.o \ + i915-display/intel_dbuf_bw.o \ + i915-display/intel_ddi.o \ + i915-display/intel_ddi_buf_trans.o \ + i915-display/intel_de.o \ + i915-display/intel_display.o \ + i915-display/intel_display_conversion.o \ + i915-display/intel_display_device.o \ + i915-display/intel_display_driver.o \ + i915-display/intel_display_irq.o \ + i915-display/intel_display_params.o \ + i915-display/intel_display_power.o \ + i915-display/intel_display_power_map.o \ + i915-display/intel_display_power_well.o \ + i915-display/intel_display_reset.o \ + i915-display/intel_display_rpm.o \ + i915-display/intel_display_rps.o \ + i915-display/intel_display_trace.o \ + i915-display/intel_display_utils.o \ + i915-display/intel_display_wa.o \ + i915-display/intel_dkl_phy.o \ + i915-display/intel_dmc.o \ + i915-display/intel_dmc_wl.o \ + i915-display/intel_dp.o \ + i915-display/intel_dp_aux.o \ + i915-display/intel_dp_aux_backlight.o \ + i915-display/intel_dp_hdcp.o \ + i915-display/intel_dp_link_caps.o \ + i915-display/intel_dp_link_training.o \ + i915-display/intel_dp_mst.o \ + i915-display/intel_dp_test.o \ + i915-display/intel_dpll.o \ + i915-display/intel_dpll_mgr.o \ + i915-display/intel_dpt.o \ + i915-display/intel_dram.o \ + i915-display/intel_drrs.o \ + i915-display/intel_dsb.o \ + i915-display/intel_dsi.o \ + i915-display/intel_dsi_dcs_backlight.o \ + i915-display/intel_dsi_vbt.o \ + i915-display/intel_encoder.o \ + i915-display/intel_fb.o \ + i915-display/intel_fbc.o \ + i915-display/intel_fdi.o \ + i915-display/intel_fifo_underrun.o \ + i915-display/intel_flipq.o \ + i915-display/intel_frontbuffer.o \ + i915-display/intel_global_state.o \ + i915-display/intel_gmbus.o \ + i915-display/intel_hdcp.o \ + i915-display/intel_hdcp_gsc_message.o \ + i915-display/intel_hdmi.o \ + i915-display/intel_hotplug.o \ + i915-display/intel_hotplug_irq.o \ + i915-display/intel_hti.o \ + i915-display/intel_initial_plane.o \ + i915-display/intel_link_bw.o \ + i915-display/intel_lspcon.o \ + i915-display/intel_lt_phy.o \ + i915-display/intel_mchbar.o \ + i915-display/intel_modeset_lock.o \ + i915-display/intel_modeset_setup.o \ + i915-display/intel_modeset_verify.o \ + i915-display/intel_overlay.o \ + i915-display/intel_panel.o \ + i915-display/intel_parent.o \ + i915-display/intel_pch.o \ + i915-display/intel_pfit.o \ + i915-display/intel_plane.o \ + i915-display/intel_pmdemand.o \ + i915-display/intel_pps.o \ + i915-display/intel_psr.o \ + i915-display/intel_qp_tables.o \ + i915-display/intel_quirks.o \ + i915-display/intel_rom.o \ + i915-display/intel_snps_hdmi_pll.o \ + i915-display/intel_snps_phy.o \ + i915-display/intel_tc.o \ + i915-display/intel_vblank.o \ + i915-display/intel_vdsc.o \ + i915-display/intel_vga.o \ + i915-display/intel_vrr.o \ + i915-display/intel_wm.o \ + i915-display/skl_prefill.o \ + i915-display/skl_scaler.o \ + i915-display/skl_universal_plane.o \ + i915-display/skl_watermark.o \ + i915-display/vlv_sideband.o + +ifeq ($(CONFIG_ACPI),y) + xe-$(CONFIG_DRM_XE_DISPLAY) += \ + i915-display/intel_acpi.o \ + i915-display/intel_opregion.o +endif + +ifeq ($(CONFIG_DRM_FBDEV_EMULATION),y) + xe-$(CONFIG_DRM_XE_DISPLAY) += i915-display/intel_fbdev.o +endif + +ifeq ($(CONFIG_DEBUG_FS),y) + xe-y += xe_debugfs.o \ + xe_gt_debugfs.o \ + xe_gt_sriov_vf_debugfs.o \ + xe_gt_stats.o \ + xe_guc_debugfs.o \ + xe_huc_debugfs.o \ + xe_tile_debugfs.o \ + xe_uc_debugfs.o + + xe-$(CONFIG_PCI_IOV) += xe_gt_sriov_pf_debugfs.o + + xe-$(CONFIG_DRM_XE_DISPLAY) += \ + i915-display/intel_display_debugfs.o \ + i915-display/intel_display_debugfs_params.o \ + i915-display/intel_pipe_crc.o +endif + +xe-$(CONFIG_DRM_XE_DP_TUNNEL) += \ + i915-display/intel_dp_tunnel.o + +obj-$(CONFIG_DRM_XE) += xe.o +obj-$(CONFIG_DRM_XE_KUNIT_TEST) += tests/ +ifeq ($(CONFIG_DRM_XE_DISPLAY),y) + obj-$(CONFIG_DRM_XE_KUNIT_TEST) += display/tests/ +endif + +# header test +hdrtest_find_args := -not -path xe_rtp_helpers.h +ifneq ($(CONFIG_DRM_XE_DISPLAY),y) + hdrtest_find_args += -not -path display/\* -not -path compat-i915-headers/\* -not -path xe_display.h +endif + +always-$(CONFIG_DRM_XE_WERROR) += \ + $(patsubst %.h,%.hdrtest, $(shell cd $(src) && find * -name '*.h' $(hdrtest_find_args))) + +quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $< -include $<; \ + $(srctree)/scripts/kernel-doc -none -Werror $<; touch $@ + +$(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) + +uses_generated_oob := $(addprefix $(obj)/, $(xe-y)) +$(uses_generated_oob): $(obj)/generated/xe_wa_oob.h $(obj)/generated/xe_device_wa_oob.h diff --git a/drivers/gpu/drm/xe/abi/gsc_command_header_abi.h b/drivers/gpu/drm/xe/abi/gsc_command_header_abi.h new file mode 100644 index 000000000..a4c264680 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/gsc_command_header_abi.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _ABI_GSC_COMMAND_HEADER_ABI_H +#define _ABI_GSC_COMMAND_HEADER_ABI_H + +#include + +struct intel_gsc_mtl_header { + u32 validity_marker; +#define GSC_HECI_VALIDITY_MARKER 0xA578875A + + u8 heci_client_id; + + u8 reserved1; + + u16 header_version; +#define MTL_GSC_HEADER_VERSION 1 + + /* FW allows host to decide host_session handle as it sees fit. */ + u64 host_session_handle; + + /* handle generated by FW for messages that need to be re-submitted */ + u64 gsc_message_handle; + + u32 message_size; /* lower 20 bits only, upper 12 are reserved */ + + /* + * Flags mask: + * Bit 0: Pending + * Bit 1: Session Cleanup; + * Bits 2-15: Flags + * Bits 16-31: Extension Size + * According to internal spec flags are either input or output + * we distinguish the flags using OUTFLAG or INFLAG + */ + u32 flags; +#define GSC_OUTFLAG_MSG_PENDING BIT(0) +#define GSC_INFLAG_MSG_CLEANUP BIT(1) + + u32 status; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/gsc_mkhi_commands_abi.h b/drivers/gpu/drm/xe/abi/gsc_mkhi_commands_abi.h new file mode 100644 index 000000000..ad4d04187 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/gsc_mkhi_commands_abi.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _ABI_GSC_MKHI_COMMANDS_ABI_H +#define _ABI_GSC_MKHI_COMMANDS_ABI_H + +#include + +/* Heci client ID for MKHI commands */ +#define HECI_MEADDRESS_MKHI 7 + +/* Generic MKHI header */ +struct gsc_mkhi_header { + u8 group_id; + u8 command; + u8 reserved; + u8 result; +} __packed; + +/* GFX_SRV commands */ +#define MKHI_GROUP_ID_GFX_SRV 0x30 + +#define MKHI_GFX_SRV_GET_HOST_COMPATIBILITY_VERSION (0x42) + +struct gsc_get_compatibility_version_in { + struct gsc_mkhi_header header; +} __packed; + +struct gsc_get_compatibility_version_out { + struct gsc_mkhi_header header; + u16 proj_major; + u16 compat_major; + u16 compat_minor; + u16 reserved[5]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/gsc_proxy_commands_abi.h b/drivers/gpu/drm/xe/abi/gsc_proxy_commands_abi.h new file mode 100644 index 000000000..80bbf06a3 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/gsc_proxy_commands_abi.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _ABI_GSC_PROXY_COMMANDS_ABI_H +#define _ABI_GSC_PROXY_COMMANDS_ABI_H + +#include + +/* Heci client ID for proxy commands */ +#define HECI_MEADDRESS_PROXY 10 + +/* FW-defined proxy header */ +struct xe_gsc_proxy_header { + /* + * hdr: + * Bits 0-7: type of the proxy message (see enum xe_gsc_proxy_type) + * Bits 8-15: rsvd + * Bits 16-31: length in bytes of the payload following the proxy header + */ + u32 hdr; +#define GSC_PROXY_TYPE GENMASK(7, 0) +#define GSC_PROXY_PAYLOAD_LENGTH GENMASK(31, 16) + + u32 source; /* Source of the Proxy message */ + u32 destination; /* Destination of the Proxy message */ +#define GSC_PROXY_ADDRESSING_KMD 0x10000 +#define GSC_PROXY_ADDRESSING_GSC 0x20000 +#define GSC_PROXY_ADDRESSING_CSME 0x30000 + + u32 status; /* Command status */ +} __packed; + +/* FW-defined proxy types */ +enum xe_gsc_proxy_type { + GSC_PROXY_MSG_TYPE_PROXY_INVALID = 0, + GSC_PROXY_MSG_TYPE_PROXY_QUERY = 1, + GSC_PROXY_MSG_TYPE_PROXY_PAYLOAD = 2, + GSC_PROXY_MSG_TYPE_PROXY_END = 3, + GSC_PROXY_MSG_TYPE_PROXY_NOTIFICATION = 4, +}; + +#endif diff --git a/drivers/gpu/drm/xe/abi/gsc_pxp_commands_abi.h b/drivers/gpu/drm/xe/abi/gsc_pxp_commands_abi.h new file mode 100644 index 000000000..290e431cf --- /dev/null +++ b/drivers/gpu/drm/xe/abi/gsc_pxp_commands_abi.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _ABI_GSC_PXP_COMMANDS_ABI_H +#define _ABI_GSC_PXP_COMMANDS_ABI_H + +#include +#include + +/* Heci client ID for PXP commands */ +#define HECI_MEADDRESS_PXP 17 + +#define PXP_APIVER(x, y) (((x) & 0xFFFF) << 16 | ((y) & 0xFFFF)) + +/* + * A PXP sub-section in an HECI packet can be up to 64K big in each direction. + * This does not include the top-level GSC header. + */ +#define PXP_MAX_PACKET_SIZE SZ_64K + +/* + * there are a lot of status codes for PXP, but we only define the cross-API + * common ones that we actually can handle in the kernel driver. Other failure + * codes should be printed to error msg for debug. + */ +enum pxp_status { + PXP_STATUS_SUCCESS = 0x0, + PXP_STATUS_ERROR_API_VERSION = 0x1002, + PXP_STATUS_NOT_READY = 0x100e, + PXP_STATUS_PLATFCONFIG_KF1_NOVERIF = 0x101a, + PXP_STATUS_PLATFCONFIG_KF1_BAD = 0x101f, + PXP_STATUS_PLATFCONFIG_FIXED_KF1_NOT_SUPPORTED = 0x1037, + PXP_STATUS_OP_NOT_PERMITTED = 0x4013 +}; + +/* Common PXP FW message header */ +struct pxp_cmd_header { + u32 api_version; + u32 command_id; + union { + u32 status; /* out */ + u32 stream_id; /* in */ +#define PXP_CMDHDR_EXTDATA_SESSION_VALID GENMASK(0, 0) +#define PXP_CMDHDR_EXTDATA_APP_TYPE GENMASK(1, 1) +#define PXP_CMDHDR_EXTDATA_SESSION_ID GENMASK(17, 2) + }; + /* Length of the message (excluding the header) */ + u32 buffer_len; +} __packed; + +#define PXP43_CMDID_INVALIDATE_STREAM_KEY 0x00000007 +#define PXP43_CMDID_INIT_SESSION 0x00000036 +#define PXP43_CMDID_NEW_HUC_AUTH 0x0000003F /* MTL+ */ + +/* PXP-Input-Packet: HUC Auth-only */ +struct pxp43_new_huc_auth_in { + struct pxp_cmd_header header; + u64 huc_base_address; + u32 huc_size; +} __packed; + +/* PXP-Output-Packet: HUC Load and Authentication or Auth-only */ +struct pxp43_huc_auth_out { + struct pxp_cmd_header header; +} __packed; + +/* PXP-Input-Packet: Init PXP session */ +struct pxp43_create_arb_in { + struct pxp_cmd_header header; + /* header.stream_id fields for vesion 4.3 of Init PXP session: */ + #define PXP43_INIT_SESSION_VALID BIT(0) + #define PXP43_INIT_SESSION_APPTYPE BIT(1) + #define PXP43_INIT_SESSION_APPID GENMASK(17, 2) + u32 protection_mode; + #define PXP43_INIT_SESSION_PROTECTION_ARB 0x2 + u32 sub_session_id; + u32 init_flags; + u32 rsvd[12]; +} __packed; + +/* PXP-Input-Packet: Init PXP session */ +struct pxp43_create_arb_out { + struct pxp_cmd_header header; + u32 rsvd[8]; +} __packed; + +/* PXP-Input-Packet: Invalidate Stream Key */ +struct pxp43_inv_stream_key_in { + struct pxp_cmd_header header; + u32 rsvd[3]; +} __packed; + +/* PXP-Output-Packet: Invalidate Stream Key */ +struct pxp43_inv_stream_key_out { + struct pxp_cmd_header header; + u32 rsvd; +} __packed; +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h new file mode 100644 index 000000000..f5c9b3703 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h @@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_ACTIONS_ABI_H +#define _ABI_GUC_ACTIONS_ABI_H + +/** + * DOC: HOST2GUC_SELF_CFG + * + * This message is used by Host KMD to setup of the `GuC Self Config KLVs`_. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_SELF_CFG` = 0x0508 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:16 | **KLV_KEY** - KLV key, see `GuC Self Config KLVs`_ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **KLV_LEN** - KLV length | + * | | | | + * | | | - 32 bit KLV = 1 | + * | | | - 64 bit KLV = 2 | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **VALUE32** - Bits 31-0 of the KLV value | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **VALUE64** - Bits 63-32 of the KLV value (**KLV_LEN** = 2) | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = **NUM** - 1 if KLV was parsed, 0 if not recognized | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_HOST2GUC_SELF_CFG 0x0508 + +#define HOST2GUC_SELF_CFG_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 3u) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY (0xffffu << 16) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN (0xffffu << 0) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32 GUC_HXG_REQUEST_MSG_n_DATAn +#define HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64 GUC_HXG_REQUEST_MSG_n_DATAn + +#define HOST2GUC_SELF_CFG_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_SELF_CFG_RESPONSE_MSG_0_NUM GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: HOST2GUC_CONTROL_CTB + * + * This H2G action allows Vf Host to enable or disable H2G and G2H `CT Buffer`_. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_CONTROL_CTB` = 0x4509 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **CONTROL** - control `CTB based communication`_ | + * | | | | + * | | | - _`GUC_CTB_CONTROL_DISABLE` = 0 | + * | | | - _`GUC_CTB_CONTROL_ENABLE` = 1 | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_HOST2GUC_CONTROL_CTB 0x4509 + +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL GUC_HXG_REQUEST_MSG_n_DATAn +#define GUC_CTB_CONTROL_DISABLE 0u +#define GUC_CTB_CONTROL_ENABLE 1u + +#define HOST2GUC_CONTROL_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_CONTROL_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +/* legacy definitions */ + +enum xe_guc_action { + XE_GUC_ACTION_DEFAULT = 0x0, + XE_GUC_ACTION_REQUEST_PREEMPTION = 0x2, + XE_GUC_ACTION_REQUEST_ENGINE_RESET = 0x3, + XE_GUC_ACTION_ALLOCATE_DOORBELL = 0x10, + XE_GUC_ACTION_DEALLOCATE_DOORBELL = 0x20, + XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE = 0x30, + XE_GUC_ACTION_UK_LOG_ENABLE_LOGGING = 0x40, + XE_GUC_ACTION_FORCE_LOG_BUFFER_FLUSH = 0x302, + XE_GUC_ACTION_ENTER_S_STATE = 0x501, + XE_GUC_ACTION_EXIT_S_STATE = 0x502, + XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE = 0x506, + XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV = 0x509, + XE_GUC_ACTION_SCHED_CONTEXT = 0x1000, + XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET = 0x1001, + XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE = 0x1002, + XE_GUC_ACTION_SCHED_ENGINE_MODE_SET = 0x1003, + XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE = 0x1004, + XE_GUC_ACTION_SET_CONTEXT_PRIORITY = 0x1005, + XE_GUC_ACTION_SET_CONTEXT_EXECUTION_QUANTUM = 0x1006, + XE_GUC_ACTION_SET_CONTEXT_PREEMPTION_TIMEOUT = 0x1007, + XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION = 0x1008, + XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION = 0x1009, + XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES = 0x100B, + XE_GUC_ACTION_AUTHENTICATE_HUC = 0x4000, + XE_GUC_ACTION_GET_HWCONFIG = 0x4100, + XE_GUC_ACTION_REGISTER_CONTEXT = 0x4502, + XE_GUC_ACTION_DEREGISTER_CONTEXT = 0x4503, + XE_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER = 0x4505, + XE_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER = 0x4506, + XE_GUC_ACTION_REGISTER_G2G = 0x4507, + XE_GUC_ACTION_DEREGISTER_G2G = 0x4508, + XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600, + XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601, + XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE = 0x4602, + XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC = 0x4603, + XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE = 0x4604, + XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR = 0x4605, + XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507, + XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A, + XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER = 0x550C, + XE_GUC_ACTION_SET_FUNCTION_ENGINE_ACTIVITY_BUFFER = 0x550D, + XE_GUC_ACTION_OPT_IN_FEATURE_KLV = 0x550E, + XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR = 0x6000, + XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC = 0x6002, + XE_GUC_ACTION_PAGE_FAULT_RES_DESC = 0x6003, + XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY = 0x6004, + XE_GUC_ACTION_NOTIFY_UNCORRECTABLE_LOCAL_ERROR = 0x6005, + XE_GUC_ACTION_TLB_INVALIDATION = 0x7000, + XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001, + XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002, + XE_GUC_ACTION_PAGE_RECLAMATION = 0x7003, + XE_GUC_ACTION_PAGE_RECLAMATION_DONE = 0x7004, + XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002, + XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003, + XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004, + XE_GUC_ACTION_NOTIFY_EXCEPTION = 0x8005, + XE_GUC_ACTION_TEST_G2G_SEND = 0xF001, + XE_GUC_ACTION_TEST_G2G_RECV = 0xF002, + XE_GUC_ACTION_LIMIT +}; + +enum xe_guc_preempt_options { + XE_GUC_PREEMPT_OPTION_DROP_WORK_Q = 0x4, + XE_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q = 0x8, +}; + +enum xe_guc_register_context_param_offsets { + XE_GUC_REGISTER_CONTEXT_DATA_0_MBZ = 0, + XE_GUC_REGISTER_CONTEXT_DATA_1_FLAGS, + XE_GUC_REGISTER_CONTEXT_DATA_2_CONTEXT_INDEX, + XE_GUC_REGISTER_CONTEXT_DATA_3_ENGINE_CLASS, + XE_GUC_REGISTER_CONTEXT_DATA_4_ENGINE_SUBMIT_MASK, + XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER, + XE_GUC_REGISTER_CONTEXT_DATA_6_WQ_DESC_ADDR_UPPER, + XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER, + XE_GUC_REGISTER_CONTEXT_DATA_8_WQ_BUF_BASE_UPPER, + XE_GUC_REGISTER_CONTEXT_DATA_9_WQ_BUF_SIZE, + XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR, + XE_GUC_REGISTER_CONTEXT_MSG_LEN, +}; + +enum xe_guc_register_context_multi_lrc_param_offsets { + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_0_MBZ = 0, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_1_FLAGS, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_2_PARENT_CONTEXT, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_3_ENGINE_CLASS, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_4_ENGINE_SUBMIT_MASK, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_6_WQ_DESC_ADDR_UPPER, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_8_WQ_BUF_BASE_UPPER, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_9_WQ_BUF_SIZE, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR, + XE_GUC_REGISTER_CONTEXT_MULTI_LRC_MSG_MIN_LEN = 11, +}; + +enum xe_guc_report_status { + XE_GUC_REPORT_STATUS_UNKNOWN = 0x0, + XE_GUC_REPORT_STATUS_ACKED = 0x1, + XE_GUC_REPORT_STATUS_ERROR = 0x2, + XE_GUC_REPORT_STATUS_COMPLETE = 0x4, +}; + +enum xe_guc_sleep_state_status { + XE_GUC_SLEEP_STATE_SUCCESS = 0x1, + XE_GUC_SLEEP_STATE_PREEMPT_TO_IDLE_FAILED = 0x2, + XE_GUC_SLEEP_STATE_ENGINE_RESET_FAILED = 0x3 +#define XE_GUC_SLEEP_STATE_INVALID_MASK 0x80000000 +}; + +#define GUC_LOG_CONTROL_LOGGING_ENABLED (1 << 0) +#define GUC_LOG_CONTROL_VERBOSITY_SHIFT 4 +#define GUC_LOG_CONTROL_VERBOSITY_MASK (0xF << GUC_LOG_CONTROL_VERBOSITY_SHIFT) +#define GUC_LOG_CONTROL_DEFAULT_LOGGING (1 << 8) + +enum xe_guc_state_capture_event_status { + XE_GUC_STATE_CAPTURE_EVENT_STATUS_SUCCESS = 0x0, + XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE = 0x1, +}; + +#define XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK 0x000000FF +#define XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN 1 + +#define XE_GUC_TLB_INVAL_TYPE_SHIFT 0 +#define XE_GUC_TLB_INVAL_MODE_SHIFT 8 +/* Flush PPC or SMRO caches along with TLB invalidation request */ +#define XE_GUC_TLB_INVAL_FLUSH_CACHE (1 << 31) + +enum xe_guc_tlb_invalidation_type { + XE_GUC_TLB_INVAL_FULL = 0x0, + XE_GUC_TLB_INVAL_PAGE_SELECTIVE = 0x1, + XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX = 0x2, + XE_GUC_TLB_INVAL_GUC = 0x3, +}; + +/* + * 0: Heavy mode of Invalidation: + * The pipeline of the engine(s) for which the invalidation is targeted to is + * blocked, and all the in-flight transactions are guaranteed to be Globally + * Observed before completing the TLB invalidation + * 1: Lite mode of Invalidation: + * TLBs of the targeted engine(s) are immediately invalidated. + * In-flight transactions are NOT guaranteed to be Globally Observed before + * completing TLB invalidation. + * Light Invalidation Mode is to be used only when + * it can be guaranteed (by SW) that the address translations remain invariant + * for the in-flight transactions across the TLB invalidation. In other words, + * this mode can be used when the TLB invalidation is intended to clear out the + * stale cached translations that are no longer in use. Light Invalidation Mode + * is much faster than the Heavy Invalidation Mode, as it does not wait for the + * in-flight transactions to be GOd. + */ +enum xe_guc_tlb_inval_mode { + XE_GUC_TLB_INVAL_MODE_HEAVY = 0x0, + XE_GUC_TLB_INVAL_MODE_LITE = 0x1, +}; + +/* + * GuC to GuC communication (de-)registration fields: + */ +enum xe_guc_g2g_type { + XE_G2G_TYPE_IN = 0x0, + XE_G2G_TYPE_OUT, + XE_G2G_TYPE_LIMIT, +}; + +#define XE_G2G_REGISTER_DEVICE REG_GENMASK(16, 16) +#define XE_G2G_REGISTER_TILE REG_GENMASK(15, 12) +#define XE_G2G_REGISTER_TYPE REG_GENMASK(11, 8) +#define XE_G2G_REGISTER_SIZE REG_GENMASK(7, 0) + +#define XE_G2G_DEREGISTER_DEVICE REG_GENMASK(16, 16) +#define XE_G2G_DEREGISTER_TILE REG_GENMASK(15, 12) +#define XE_G2G_DEREGISTER_TYPE REG_GENMASK(11, 8) + +/* invalid type for XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR */ +#define XE_GUC_CAT_ERR_TYPE_INVALID 0xdeadbeef + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h new file mode 100644 index 000000000..ce5c59517 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _GUC_ACTIONS_SLPC_ABI_H_ +#define _GUC_ACTIONS_SLPC_ABI_H_ + +#include + +/** + * DOC: SLPC SHARED DATA STRUCTURE + * + * +----+------+--------------------------------------------------------------+ + * | CL | Bytes| Description | + * +====+======+==============================================================+ + * | 1 | 0-3 | SHARED DATA SIZE | + * | +------+--------------------------------------------------------------+ + * | | 4-7 | GLOBAL STATE | + * | +------+--------------------------------------------------------------+ + * | | 8-11 | DISPLAY DATA ADDRESS | + * | +------+--------------------------------------------------------------+ + * | | 12:63| PADDING | + * +----+------+--------------------------------------------------------------+ + * | | 0:63 | PADDING(PLATFORM INFO) | + * +----+------+--------------------------------------------------------------+ + * | 3 | 0-3 | TASK STATE DATA | + * + +------+--------------------------------------------------------------+ + * | | 4:63 | PADDING | + * +----+------+--------------------------------------------------------------+ + * |4-21|0:1087| OVERRIDE PARAMS AND BIT FIELDS | + * +----+------+--------------------------------------------------------------+ + * | | | PADDING + EXTRA RESERVED PAGE | + * +----+------+--------------------------------------------------------------+ + */ + +/* + * SLPC exposes certain parameters for global configuration by the host. + * These are referred to as override parameters, because in most cases + * the host will not need to modify the default values used by SLPC. + * SLPC remembers the default values which allows the host to easily restore + * them by simply unsetting the override. The host can set or unset override + * parameters during SLPC (re-)initialization using the SLPC Reset event. + * The host can also set or unset override parameters on the fly using the + * Parameter Set and Parameter Unset events + */ + +#define SLPC_MAX_OVERRIDE_PARAMETERS 256 +#define SLPC_OVERRIDE_BITFIELD_SIZE \ + (SLPC_MAX_OVERRIDE_PARAMETERS / 32) + +#define SLPC_PAGE_SIZE_BYTES 4096 +#define SLPC_CACHELINE_SIZE_BYTES 64 +#define SLPC_SHARED_DATA_SIZE_BYTE_HEADER SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_MODE_DEFN_TABLE_SIZE SLPC_PAGE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_MAX (2 * SLPC_PAGE_SIZE_BYTES) + +/* + * Cacheline size aligned (Total size needed for + * SLPM_KMD_MAX_OVERRIDE_PARAMETERS=256 is 1088 bytes) + */ +#define SLPC_OVERRIDE_PARAMS_TOTAL_BYTES (((((SLPC_MAX_OVERRIDE_PARAMETERS * 4) \ + + ((SLPC_MAX_OVERRIDE_PARAMETERS / 32) * 4)) \ + + (SLPC_CACHELINE_SIZE_BYTES - 1)) / SLPC_CACHELINE_SIZE_BYTES) * \ + SLPC_CACHELINE_SIZE_BYTES) + +#define SLPC_SHARED_DATA_SIZE_BYTE_OTHER (SLPC_SHARED_DATA_SIZE_BYTE_MAX - \ + (SLPC_SHARED_DATA_SIZE_BYTE_HEADER \ + + SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO \ + + SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE \ + + SLPC_OVERRIDE_PARAMS_TOTAL_BYTES \ + + SLPC_SHARED_DATA_MODE_DEFN_TABLE_SIZE)) + +enum slpc_task_enable { + SLPC_PARAM_TASK_DEFAULT = 0, + SLPC_PARAM_TASK_ENABLED, + SLPC_PARAM_TASK_DISABLED, + SLPC_PARAM_TASK_UNKNOWN +}; + +enum slpc_global_state { + SLPC_GLOBAL_STATE_NOT_RUNNING = 0, + SLPC_GLOBAL_STATE_INITIALIZING = 1, + SLPC_GLOBAL_STATE_RESETTING = 2, + SLPC_GLOBAL_STATE_RUNNING = 3, + SLPC_GLOBAL_STATE_SHUTTING_DOWN = 4, + SLPC_GLOBAL_STATE_ERROR = 5 +}; + +enum slpc_param_id { + SLPC_PARAM_TASK_ENABLE_GTPERF = 0, + SLPC_PARAM_TASK_DISABLE_GTPERF = 1, + SLPC_PARAM_TASK_ENABLE_BALANCER = 2, + SLPC_PARAM_TASK_DISABLE_BALANCER = 3, + SLPC_PARAM_TASK_ENABLE_DCC = 4, + SLPC_PARAM_TASK_DISABLE_DCC = 5, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ = 6, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ = 7, + SLPC_PARAM_GLOBAL_MIN_GT_SLICE_FREQ_MHZ = 8, + SLPC_PARAM_GLOBAL_MAX_GT_SLICE_FREQ_MHZ = 9, + SLPC_PARAM_GTPERF_THRESHOLD_MAX_FPS = 10, + SLPC_PARAM_GLOBAL_DISABLE_GT_FREQ_MANAGEMENT = 11, + SLPC_PARAM_GTPERF_ENABLE_FRAMERATE_STALLING = 12, + SLPC_PARAM_GLOBAL_DISABLE_RC6_MODE_CHANGE = 13, + SLPC_PARAM_GLOBAL_OC_UNSLICE_FREQ_MHZ = 14, + SLPC_PARAM_GLOBAL_OC_SLICE_FREQ_MHZ = 15, + SLPC_PARAM_GLOBAL_ENABLE_IA_GT_BALANCING = 16, + SLPC_PARAM_GLOBAL_ENABLE_ADAPTIVE_BURST_TURBO = 17, + SLPC_PARAM_GLOBAL_ENABLE_EVAL_MODE = 18, + SLPC_PARAM_GLOBAL_ENABLE_BALANCER_IN_NON_GAMING_MODE = 19, + SLPC_PARAM_GLOBAL_RT_MODE_TURBO_FREQ_DELTA_MHZ = 20, + SLPC_PARAM_PWRGATE_RC_MODE = 21, + SLPC_PARAM_EDR_MODE_COMPUTE_TIMEOUT_MS = 22, + SLPC_PARAM_EDR_QOS_FREQ_MHZ = 23, + SLPC_PARAM_MEDIA_FF_RATIO_MODE = 24, + SLPC_PARAM_ENABLE_IA_FREQ_LIMITING = 25, + SLPC_PARAM_STRATEGIES = 26, + SLPC_PARAM_POWER_PROFILE = 27, + SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY = 28, + SLPC_MAX_PARAM = 32, +}; + +enum slpc_media_ratio_mode { + SLPC_MEDIA_RATIO_MODE_DYNAMIC_CONTROL = 0, + SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_ONE = 1, + SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_TWO = 2, +}; + +enum slpc_gucrc_mode { + SLPC_GUCRC_MODE_HW = 0, + SLPC_GUCRC_MODE_GUCRC_NO_RC6 = 1, + SLPC_GUCRC_MODE_GUCRC_STATIC_TIMEOUT = 2, + SLPC_GUCRC_MODE_GUCRC_DYNAMIC_HYSTERESIS = 3, + + SLPC_GUCRC_MODE_MAX, +}; + +enum slpc_event_id { + SLPC_EVENT_RESET = 0, + SLPC_EVENT_SHUTDOWN = 1, + SLPC_EVENT_PLATFORM_INFO_CHANGE = 2, + SLPC_EVENT_DISPLAY_MODE_CHANGE = 3, + SLPC_EVENT_FLIP_COMPLETE = 4, + SLPC_EVENT_QUERY_TASK_STATE = 5, + SLPC_EVENT_PARAMETER_SET = 6, + SLPC_EVENT_PARAMETER_UNSET = 7, +}; + +struct slpc_task_state_data { + union { + u32 task_status_padding; + struct { + u32 status; +#define SLPC_GTPERF_TASK_ENABLED REG_BIT(0) +#define SLPC_DCC_TASK_ENABLED REG_BIT(11) +#define SLPC_IN_DCC REG_BIT(12) +#define SLPC_BALANCER_ENABLED REG_BIT(15) +#define SLPC_IBC_TASK_ENABLED REG_BIT(16) +#define SLPC_BALANCER_IA_LMT_ENABLED REG_BIT(17) +#define SLPC_BALANCER_IA_LMT_ACTIVE REG_BIT(18) + }; + }; + union { + u32 freq_padding; + struct { +#define SLPC_MAX_UNSLICE_FREQ_MASK REG_GENMASK(7, 0) +#define SLPC_MIN_UNSLICE_FREQ_MASK REG_GENMASK(15, 8) +#define SLPC_MAX_SLICE_FREQ_MASK REG_GENMASK(23, 16) +#define SLPC_MIN_SLICE_FREQ_MASK REG_GENMASK(31, 24) + u32 freq; + }; + }; +} __packed; + +#define SLPC_CTX_FREQ_REQ_IS_COMPUTE REG_BIT(28) +#define SLPC_OPTIMIZED_STRATEGY_COMPUTE REG_BIT(0) + +struct slpc_shared_data_header { + /* Total size in bytes of this shared buffer. */ + u32 size; + u32 global_state; + u32 display_data_addr; +} __packed; + +struct slpc_override_params { + u32 bits[SLPC_OVERRIDE_BITFIELD_SIZE]; + u32 values[SLPC_MAX_OVERRIDE_PARAMETERS]; +} __packed; + +struct slpc_shared_data { + struct slpc_shared_data_header header; + u8 shared_data_header_pad[SLPC_SHARED_DATA_SIZE_BYTE_HEADER - + sizeof(struct slpc_shared_data_header)]; + + u8 platform_info_pad[SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO]; + + struct slpc_task_state_data task_state_data; + u8 task_state_data_pad[SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE - + sizeof(struct slpc_task_state_data)]; + + struct slpc_override_params override_params; + u8 override_params_pad[SLPC_OVERRIDE_PARAMS_TOTAL_BYTES - + sizeof(struct slpc_override_params)]; + + u8 shared_data_pad[SLPC_SHARED_DATA_SIZE_BYTE_OTHER]; + + /* PAGE 2 (4096 bytes), mode based parameter will be removed soon */ + u8 reserved_mode_definition[4096]; +} __packed; + +enum slpc_power_profile { + SLPC_POWER_PROFILE_BASE = 0x0, + SLPC_POWER_PROFILE_POWER_SAVING = 0x1 +}; + +/** + * DOC: SLPC H2G MESSAGE FORMAT + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_PC_SLPM_REQUEST` = 0x3003 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:8 | **EVENT_ID** | + * + +-------+--------------------------------------------------------------+ + * | | 7:0 | **EVENT_ARGC** - number of data arguments | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **EVENT_DATA1** | + * +---+-------+--------------------------------------------------------------+ + * |...| 31:0 | ... | + * +---+-------+--------------------------------------------------------------+ + * |2+n| 31:0 | **EVENT_DATAn** | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST 0x3003 + +#define HOST2GUC_PC_SLPC_REQUEST_MSG_MIN_LEN \ + (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_PC_SLPC_EVENT_MAX_INPUT_ARGS 9 +#define HOST2GUC_PC_SLPC_REQUEST_MSG_MAX_LEN \ + (HOST2GUC_PC_SLPC_REQUEST_REQUEST_MSG_MIN_LEN + \ + HOST2GUC_PC_SLPC_EVENT_MAX_INPUT_ARGS) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID (0xffu << 8) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC (0xffu << 0) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_N_EVENT_DATA_N GUC_HXG_REQUEST_MSG_n_DATAn + +/** + * DOC: SETUP_PC_GUCRC + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_FAST_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC` = 0x3004 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **MODE** = GUCRC_HOST_CONTROL(0), GUCRC_FIRMWARE_CONTROL(1) | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC 0x3004u +#define GUCRC_HOST_CONTROL 0u +#define GUCRC_FIRMWARE_CONTROL 1u + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h new file mode 100644 index 000000000..d9f21202e --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h @@ -0,0 +1,706 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _ABI_GUC_ACTIONS_SRIOV_ABI_H +#define _ABI_GUC_ACTIONS_SRIOV_ABI_H + +#include "guc_communication_ctb_abi.h" + +/** + * DOC: GUC2PF_RELAY_FROM_VF + * + * This message is used by the GuC firmware to forward a VF2PF `Relay Message`_ + * received from the Virtual Function (VF) driver to this Physical Function (PF) + * driver. + * + * This message is always sent as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF` = 0x5100 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VFID** - source VF identifier | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **RELAY_ID** - VF/PF message ID | + * +---+-------+-----------------+--------------------------------------------+ + * | 3 | 31:0 | **RELAY_DATA1** | | + * +---+-------+-----------------+ | + * |...| | | [Embedded `Relay Message`_] | + * +---+-------+-----------------+ | + * | n | 31:0 | **RELAY_DATAx** | | + * +---+-------+-----------------+--------------------------------------------+ + */ +#define XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF 0x5100 + +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN (GUC_HXG_EVENT_MSG_MIN_LEN + 2u) +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN \ + (GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN + GUC_RELAY_MSG_MAX_LEN) +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_0_MBZ GUC_HXG_EVENT_MSG_0_DATA0 +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_3_RELAY_DATA1 GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_n_RELAY_DATAx GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2PF_RELAY_FROM_VF_EVENT_MSG_NUM_RELAY_DATA GUC_RELAY_MSG_MAX_LEN + +/** + * DOC: PF2GUC_RELAY_TO_VF + * + * This H2G message is used by the Physical Function (PF) driver to send embedded + * VF2PF `Relay Message`_ to the VF. + * + * This action message must be sent over CTB as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = `GUC_HXG_TYPE_FAST_REQUEST`_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`XE_GUC_ACTION_PF2GUC_RELAY_TO_VF` = 0x5101 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VFID** - target VF identifier | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **RELAY_ID** - VF/PF message ID | + * +---+-------+-----------------+--------------------------------------------+ + * | 3 | 31:0 | **RELAY_DATA1** | | + * +---+-------+-----------------+ | + * |...| | | [Embedded `Relay Message`_] | + * +---+-------+-----------------+ | + * | n | 31:0 | **RELAY_DATAx** | | + * +---+-------+-----------------+--------------------------------------------+ + */ +#define XE_GUC_ACTION_PF2GUC_RELAY_TO_VF 0x5101 + +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 2u) +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_MAX_LEN \ + (PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN + GUC_RELAY_MSG_MAX_LEN) +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_3_RELAY_DATA1 GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_n_RELAY_DATAx GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_RELAY_TO_VF_REQUEST_MSG_NUM_RELAY_DATA GUC_RELAY_MSG_MAX_LEN + +/** + * DOC: GUC2VF_RELAY_FROM_PF + * + * This message is used by the GuC firmware to deliver `Relay Message`_ from the + * Physical Function (PF) driver to this Virtual Function (VF) driver. + * See `GuC Relay Communication`_ for details. + * + * This message is always sent over CTB. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF` = 0x5102 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **RELAY_ID** - VF/PF message ID | + * +---+-------+-----------------+--------------------------------------------+ + * | 2 | 31:0 | **RELAY_DATA1** | | + * +---+-------+-----------------+ | + * |...| | | [Embedded `Relay Message`_] | + * +---+-------+-----------------+ | + * | n | 31:0 | **RELAY_DATAx** | | + * +---+-------+-----------------+--------------------------------------------+ + */ +#define XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF 0x5102 + +#define GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN (GUC_HXG_EVENT_MSG_MIN_LEN + 1u) +#define GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN \ + (GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN + GUC_RELAY_MSG_MAX_LEN) +#define GUC2VF_RELAY_FROM_PF_EVENT_MSG_0_MBZ GUC_HXG_EVENT_MSG_0_DATA0 +#define GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2VF_RELAY_FROM_PF_EVENT_MSG_n_RELAY_DATAx GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2VF_RELAY_FROM_PF_EVENT_MSG_NUM_RELAY_DATA GUC_RELAY_MSG_MAX_LEN + +/** + * DOC: VF2GUC_RELAY_TO_PF + * + * This message is used by the Virtual Function (VF) drivers to communicate with + * the Physical Function (PF) driver and send `Relay Message`_ to the PF driver. + * See `GuC Relay Communication`_ for details. + * + * This message must be sent over CTB. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ or GUC_HXG_TYPE_FAST_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`XE_GUC_ACTION_VF2GUC_RELAY_TO_PF` = 0x5103 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **RELAY_ID** - VF/PF message ID | + * +---+-------+-----------------+--------------------------------------------+ + * | 2 | 31:0 | **RELAY_DATA1** | | + * +---+-------+-----------------+ | + * |...| | | [Embedded `Relay Message`_] | + * +---+-------+-----------------+ | + * | n | 31:0 | **RELAY_DATAx** | | + * +---+-------+-----------------+--------------------------------------------+ + */ +#define XE_GUC_ACTION_VF2GUC_RELAY_TO_PF 0x5103 + +#define VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define VF2GUC_RELAY_TO_PF_REQUEST_MSG_MAX_LEN \ + (VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN + GUC_RELAY_MSG_MAX_LEN) +#define VF2GUC_RELAY_TO_PF_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define VF2GUC_RELAY_TO_PF_REQUEST_MSG_1_RELAY_ID GUC_HXG_REQUEST_MSG_n_DATAn +#define VF2GUC_RELAY_TO_PF_REQUEST_MSG_n_RELAY_DATAx GUC_HXG_REQUEST_MSG_n_DATAn +#define VF2GUC_RELAY_TO_PF_REQUEST_MSG_NUM_RELAY_DATA GUC_RELAY_MSG_MAX_LEN + +/** + * DOC: GUC2PF_ADVERSE_EVENT + * + * This message is used by the GuC to notify PF about adverse events. + * + * This G2H message must be sent as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_GUC2PF_ADVERSE_EVENT` = 0x5104 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | DATA1 = **VFID** - VF identifier | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | DATA2 = **THRESHOLD** - key of the exceeded threshold | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_GUC2PF_ADVERSE_EVENT 0x5104 + +#define GUC2PF_ADVERSE_EVENT_EVENT_MSG_LEN (GUC_HXG_EVENT_MSG_MIN_LEN + 2u) +#define GUC2PF_ADVERSE_EVENT_EVENT_MSG_0_MBZ GUC_HXG_EVENT_MSG_0_DATA0 +#define GUC2PF_ADVERSE_EVENT_EVENT_MSG_1_VFID GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2PF_ADVERSE_EVENT_EVENT_MSG_2_THRESHOLD GUC_HXG_EVENT_MSG_n_DATAn + +/** + * DOC: GUC2PF_VF_STATE_NOTIFY + * + * The GUC2PF_VF_STATE_NOTIFY message is used by the GuC to notify PF about change + * of the VF state. + * + * This G2H message is sent as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_GUC2PF_VF_STATE_NOTIFY` = 0x5106 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | DATA1 = **VFID** - VF identifier | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | DATA2 = **EVENT** - notification event: | + * | | | | + * | | | - _`GUC_PF_NOTIFY_VF_ENABLE` = 1 (only if VFID = 0) | + * | | | - _`GUC_PF_NOTIFY_VF_FLR` = 1 | + * | | | - _`GUC_PF_NOTIFY_VF_FLR_DONE` = 2 | + * | | | - _`GUC_PF_NOTIFY_VF_PAUSE_DONE` = 3 | + * | | | - _`GUC_PF_NOTIFY_VF_FIXUP_DONE` = 4 | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_GUC2PF_VF_STATE_NOTIFY 0x5106u + +#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_LEN (GUC_HXG_EVENT_MSG_MIN_LEN + 2u) +#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_0_MBZ GUC_HXG_EVENT_MSG_0_DATA0 +#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_1_VFID GUC_HXG_EVENT_MSG_n_DATAn +#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_2_EVENT GUC_HXG_EVENT_MSG_n_DATAn +#define GUC_PF_NOTIFY_VF_ENABLE 1u +#define GUC_PF_NOTIFY_VF_FLR 1u +#define GUC_PF_NOTIFY_VF_FLR_DONE 2u +#define GUC_PF_NOTIFY_VF_PAUSE_DONE 3u +#define GUC_PF_NOTIFY_VF_FIXUP_DONE 4u + +/** + * DOC: VF2GUC_MATCH_VERSION + * + * This action is used to match VF interface version used by VF and GuC. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_MATCH_VERSION` = 0x5500 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:24 | **BRANCH** - branch ID of the VF interface | + * | | | (use BRANCH_ANY to request latest version supported by GuC) | + * | +-------+--------------------------------------------------------------+ + * | | 23:16 | **MAJOR** - major version of the VF interface | + * | | | (use MAJOR_ANY to request latest version supported by GuC) | + * | +-------+--------------------------------------------------------------+ + * | | 15:8 | **MINOR** - minor version of the VF interface | + * | | | (use MINOR_ANY to request latest version supported by GuC) | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | **MBZ** | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:24 | **BRANCH** - branch ID of the VF interface | + * | +-------+--------------------------------------------------------------+ + * | | 23:16 | **MAJOR** - major version of the VF interface | + * | +-------+--------------------------------------------------------------+ + * | | 15:8 | **MINOR** - minor version of the VF interface | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | **PATCH** - patch version of the VF interface | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_VF2GUC_MATCH_VERSION 0x5500u + +#define VF2GUC_MATCH_VERSION_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define VF2GUC_MATCH_VERSION_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define VF2GUC_MATCH_VERSION_REQUEST_MSG_1_BRANCH (0xffu << 24) +#define GUC_VERSION_BRANCH_ANY 0 +#define VF2GUC_MATCH_VERSION_REQUEST_MSG_1_MAJOR (0xffu << 16) +#define GUC_VERSION_MAJOR_ANY 0 +#define VF2GUC_MATCH_VERSION_REQUEST_MSG_1_MINOR (0xffu << 8) +#define GUC_VERSION_MINOR_ANY 0 +#define VF2GUC_MATCH_VERSION_REQUEST_MSG_1_MBZ (0xffu << 0) + +#define VF2GUC_MATCH_VERSION_RESPONSE_MSG_LEN (GUC_HXG_RESPONSE_MSG_MIN_LEN + 1u) +#define VF2GUC_MATCH_VERSION_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 +#define VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_BRANCH (0xffu << 24) +#define VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_MAJOR (0xffu << 16) +#define VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_MINOR (0xffu << 8) +#define VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_PATCH (0xffu << 0) + +/** + * DOC: PF2GUC_UPDATE_VGT_POLICY + * + * This message is used by the PF to set `GuC VGT Policy KLVs`_. + * + * This message must be sent as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_PF2GUC_UPDATE_VGT_POLICY` = 0x5502 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **CFG_ADDR_LO** - dword aligned GGTT offset that | + * | | | represents the start of `GuC VGT Policy KLVs`_ list. | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **CFG_ADDR_HI** - upper 32 bits of above offset. | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **CFG_SIZE** - size (in dwords) of the config buffer | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **COUNT** - number of KLVs successfully applied | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_PF2GUC_UPDATE_VGT_POLICY 0x5502u + +#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 3u) +#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_1_CFG_ADDR_LO GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_2_CFG_ADDR_HI GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_3_CFG_SIZE GUC_HXG_REQUEST_MSG_n_DATAn + +#define PF2GUC_UPDATE_VGT_POLICY_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define PF2GUC_UPDATE_VGT_POLICY_RESPONSE_MSG_0_COUNT GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: PF2GUC_UPDATE_VF_CFG + * + * The `PF2GUC_UPDATE_VF_CFG`_ message is used by PF to provision single VF in GuC. + * + * This message must be sent as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_PF2GUC_UPDATE_VF_CFG` = 0x5503 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VFID** - identifier of the VF that the KLV | + * | | | configurations are being applied to | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **CFG_ADDR_LO** - dword aligned GGTT offset that represents | + * | | | the start of a list of virtualization related KLV configs | + * | | | that are to be applied to the VF. | + * | | | If this parameter is zero, the list is not parsed. | + * | | | If full configs address parameter is zero and configs_size is| + * | | | zero associated VF config shall be reset to its default state| + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **CFG_ADDR_HI** - upper 32 bits of configs address. | + * +---+-------+--------------------------------------------------------------+ + * | 4 | 31:0 | **CFG_SIZE** - size (in dwords) of the config buffer | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **COUNT** - number of KLVs successfully applied | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_PF2GUC_UPDATE_VF_CFG 0x5503u + +#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 4u) +#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_1_VFID GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_2_CFG_ADDR_LO GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_3_CFG_ADDR_HI GUC_HXG_REQUEST_MSG_n_DATAn +#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_4_CFG_SIZE GUC_HXG_REQUEST_MSG_n_DATAn + +#define PF2GUC_UPDATE_VF_CFG_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define PF2GUC_UPDATE_VF_CFG_RESPONSE_MSG_0_COUNT GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: PF2GUC_VF_CONTROL + * + * The PF2GUC_VF_CONTROL message is used by the PF to trigger VF state change + * maintained by the GuC. + * + * This H2G message must be sent as `CTB HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_PF2GUC_VF_CONTROL_CMD` = 0x5506 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | DATA1 = **VFID** - VF identifier | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | DATA2 = **COMMAND** - control command: | + * | | | | + * | | | - _`GUC_PF_TRIGGER_VF_PAUSE` = 1 | + * | | | - _`GUC_PF_TRIGGER_VF_RESUME` = 2 | + * | | | - _`GUC_PF_TRIGGER_VF_STOP` = 3 | + * | | | - _`GUC_PF_TRIGGER_VF_FLR_START` = 4 | + * | | | - _`GUC_PF_TRIGGER_VF_FLR_FINISH` = 5 | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_PF2GUC_VF_CONTROL 0x5506u + +#define PF2GUC_VF_CONTROL_REQUEST_MSG_LEN (GUC_HXG_EVENT_MSG_MIN_LEN + 2u) +#define PF2GUC_VF_CONTROL_REQUEST_MSG_0_MBZ GUC_HXG_EVENT_MSG_0_DATA0 +#define PF2GUC_VF_CONTROL_REQUEST_MSG_1_VFID GUC_HXG_EVENT_MSG_n_DATAn +#define PF2GUC_VF_CONTROL_REQUEST_MSG_2_COMMAND GUC_HXG_EVENT_MSG_n_DATAn +#define GUC_PF_TRIGGER_VF_PAUSE 1u +#define GUC_PF_TRIGGER_VF_RESUME 2u +#define GUC_PF_TRIGGER_VF_STOP 3u +#define GUC_PF_TRIGGER_VF_FLR_START 4u +#define GUC_PF_TRIGGER_VF_FLR_FINISH 5u + +/** + * DOC: VF2GUC_VF_RESET + * + * This action is used by VF to reset GuC's VF state. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_VF_RESET` = 0x5507 | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_VF2GUC_VF_RESET 0x5507u + +#define VF2GUC_VF_RESET_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN +#define VF2GUC_VF_RESET_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 + +#define VF2GUC_VF_RESET_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define VF2GUC_VF_RESET_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: VF2GUC_RESFIX_DONE + * + * This action is used by VF to inform the GuC that the VF KMD has completed + * post-migration recovery steps. From GuC VF compatibility 1.27.0 onwards, it + * shall only be sent after posting RESFIX_START and that both @MARKER fields + * must match. + * + * This message must be sent as `MMIO HXG Message`_. + * + * Updated since GuC VF compatibility 1.27.0. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MARKER = MBZ (only prior 1.27.0) | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MARKER - can't be zero (1.27.0+) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_RESFIX_DONE` = 0x5508 | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_VF2GUC_RESFIX_DONE 0x5508u + +#define VF2GUC_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN +#define VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0 + +#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: VF2GUC_QUERY_SINGLE_KLV + * + * This action is used by VF to query value of the single KLV data. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_QUERY_SINGLE_KLV` = 0x5509 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **KEY** - key for which value is requested | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **LENGTH** - length of data in dwords | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VALUE32** - bits 31:0 of value if **LENGTH** >= 1 | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **VALUE64** - bits 63:32 of value if **LENGTH** >= 2 | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **VALUE96** - bits 95:64 of value if **LENGTH** >= 3 | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_VF2GUC_QUERY_SINGLE_KLV 0x5509u + +#define VF2GUC_QUERY_SINGLE_KLV_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define VF2GUC_QUERY_SINGLE_KLV_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define VF2GUC_QUERY_SINGLE_KLV_REQUEST_MSG_1_MBZ (0xffffu << 16) +#define VF2GUC_QUERY_SINGLE_KLV_REQUEST_MSG_1_KEY (0xffffu << 0) + +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_MIN_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_MAX_LEN (GUC_HXG_RESPONSE_MSG_MIN_LEN + 3u) +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_0_MBZ (0xfffu << 16) +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_0_LENGTH (0xffffu << 0) +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_1_VALUE32 GUC_HXG_REQUEST_MSG_n_DATAn +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_2_VALUE64 GUC_HXG_REQUEST_MSG_n_DATAn +#define VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_3_VALUE96 GUC_HXG_REQUEST_MSG_n_DATAn + +/** + * DOC: PF2GUC_SAVE_RESTORE_VF + * + * This message is used by the PF to migrate VF info state maintained by the GuC. + * + * This message must be sent as `CTB HXG Message`_. + * + * Available since GuC version 70.25.0 + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = **OPCODE** - operation to take: | + * | | | | + * | | | - _`GUC_PF_OPCODE_VF_SAVE` = 0 | + * | | | - _`GUC_PF_OPCODE_VF_RESTORE` = 1 | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_PF2GUC_SAVE_RESTORE_VF` = 0x550B | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VFID** - VF identifier | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **ADDR_LO** - lower 32-bits of GGTT offset to the buffer | + * | | | where the VF info will be save to or restored from. | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **ADDR_HI** - upper 32-bits of GGTT offset to the buffer | + * | | | where the VF info will be save to or restored from. | + * +---+-------+--------------------------------------------------------------+ + * | 4 | 27:0 | **SIZE** - size of the buffer (in dwords) | + * | +-------+--------------------------------------------------------------+ + * | | 31:28 | MBZ | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = **USED** - size of used buffer space (in dwords) | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_PF2GUC_SAVE_RESTORE_VF 0x550Bu + +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_LEN (GUC_HXG_EVENT_MSG_MIN_LEN + 4u) +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_0_OPCODE GUC_HXG_EVENT_MSG_0_DATA0 +#define GUC_PF_OPCODE_VF_SAVE 0u +#define GUC_PF_OPCODE_VF_RESTORE 1u +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_1_VFID GUC_HXG_EVENT_MSG_n_DATAn +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_2_ADDR_LO GUC_HXG_EVENT_MSG_n_DATAn +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_3_ADDR_HI GUC_HXG_EVENT_MSG_n_DATAn +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_4_SIZE (0xfffffffu << 0) +#define PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_4_MBZ (0xfu << 28) + +#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_0_USED GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: VF2GUC_RESFIX_START + * + * This action is used by VF to inform the GuC that the VF KMD will be starting + * post-migration recovery fixups. The @MARKER sent with this action must match + * with the MARKER posted in the VF2GUC_RESFIX_DONE message. + * + * This message must be sent as `MMIO HXG Message`_. + * + * Available since GuC VF compatibility 1.27.0. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MARKER - can't be zero | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_RESFIX_START` = 0x550F | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_VF2GUC_RESFIX_START 0x550Fu + +#define VF2GUC_RESFIX_START_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN +#define VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0 + +#define VF2GUC_RESFIX_START_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define VF2GUC_RESFIX_START_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_capture_abi.h b/drivers/gpu/drm/xe/abi/guc_capture_abi.h new file mode 100644 index 000000000..ff9c0ae34 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_capture_abi.h @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _ABI_GUC_CAPTURE_ABI_H +#define _ABI_GUC_CAPTURE_ABI_H + +#include + +/* Capture List Index */ +enum guc_capture_list_index_type { + GUC_CAPTURE_LIST_INDEX_PF = 0, + GUC_CAPTURE_LIST_INDEX_VF = 1, +}; + +#define GUC_CAPTURE_LIST_INDEX_MAX (GUC_CAPTURE_LIST_INDEX_VF + 1) + +/* Register-types of GuC capture register lists */ +enum guc_state_capture_type { + GUC_STATE_CAPTURE_TYPE_GLOBAL = 0, + GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE +}; + +#define GUC_STATE_CAPTURE_TYPE_MAX (GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE + 1) + +/* Class indices for capture_class and capture_instance arrays */ +enum guc_capture_list_class_type { + GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE = 0, + GUC_CAPTURE_LIST_CLASS_VIDEO = 1, + GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE = 2, + GUC_CAPTURE_LIST_CLASS_BLITTER = 3, + GUC_CAPTURE_LIST_CLASS_GSC_OTHER = 4, + GUC_CAPTURE_LIST_CLASS_PAGING = 5, +}; + +#define GUC_CAPTURE_LIST_CLASS_MAX (GUC_CAPTURE_LIST_CLASS_PAGING + 1) + +/** + * struct guc_mmio_reg - GuC MMIO reg state struct + * + * GuC MMIO reg state struct + */ +struct guc_mmio_reg { + /** @offset: MMIO Offset - filled in by Host */ + u32 offset; + /** @value: MMIO Value - Used by Firmware to store value */ + u32 value; + /** @flags: Flags for accessing the MMIO */ + u32 flags; + /** @mask: Value of a mask to apply if mask with value is set */ + u32 mask; +#define GUC_REGSET_MASKED BIT(0) +#define GUC_REGSET_STEERING_NEEDED BIT(1) +#define GUC_REGSET_MASKED_WITH_VALUE BIT(2) +#define GUC_REGSET_RESTORE_ONLY BIT(3) +#define GUC_REGSET_STEERING_GROUP GENMASK(16, 12) +#define GUC_REGSET_STEERING_INSTANCE GENMASK(23, 20) +} __packed; + +/** + * struct guc_mmio_reg_set - GuC register sets + * + * GuC register sets + */ +struct guc_mmio_reg_set { + /** @address: register address */ + u32 address; + /** @count: register count */ + u16 count; + /** @reserved: reserved */ + u16 reserved; +} __packed; + +/** + * struct guc_debug_capture_list_header - Debug capture list header. + * + * Debug capture list header. + */ +struct guc_debug_capture_list_header { + /** @info: contains number of MMIO descriptors in the capture list. */ + u32 info; +#define GUC_CAPTURELISTHDR_NUMDESCR GENMASK(15, 0) +} __packed; + +/** + * struct guc_debug_capture_list - Debug capture list + * + * As part of ADS registration, these header structures (followed by + * an array of 'struct guc_mmio_reg' entries) are used to register with + * GuC microkernel the list of registers we want it to dump out prior + * to a engine reset. + */ +struct guc_debug_capture_list { + /** @header: Debug capture list header. */ + struct guc_debug_capture_list_header header; + /** @regs: MMIO descriptors in the capture list. */ + struct guc_mmio_reg regs[]; +} __packed; + +/** + * struct guc_state_capture_header_t - State capture header. + * + * Prior to resetting engines that have hung or faulted, GuC microkernel + * reports the engine error-state (register values that was read) by + * logging them into the shared GuC log buffer using these hierarchy + * of structures. + */ +struct guc_state_capture_header_t { + /** + * @owner: VFID + * BR[ 7: 0] MBZ when SRIOV is disabled. When SRIOV is enabled + * VFID is an integer in range [0, 63] where 0 means the state capture + * is corresponding to the PF and an integer N in range [1, 63] means + * the state capture is for VF N. + */ + u32 owner; +#define GUC_STATE_CAPTURE_HEADER_VFID GENMASK(7, 0) + /** @info: Engine class/instance and capture type info */ + u32 info; +#define GUC_STATE_CAPTURE_HEADER_CAPTURE_TYPE GENMASK(3, 0) /* see guc_state_capture_type */ +#define GUC_STATE_CAPTURE_HEADER_ENGINE_CLASS GENMASK(7, 4) /* see guc_capture_list_class_type */ +#define GUC_STATE_CAPTURE_HEADER_ENGINE_INSTANCE GENMASK(11, 8) + /** + * @lrca: logical ring context address. + * if type-instance, LRCA (address) that hung, else set to ~0 + */ + u32 lrca; + /** + * @guc_id: context_index. + * if type-instance, context index of hung context, else set to ~0 + */ + u32 guc_id; + /** @num_mmio_entries: Number of captured MMIO entries. */ + u32 num_mmio_entries; +#define GUC_STATE_CAPTURE_HEADER_NUM_MMIO_ENTRIES GENMASK(9, 0) +} __packed; + +/** + * struct guc_state_capture_t - State capture. + * + * State capture + */ +struct guc_state_capture_t { + /** @header: State capture header. */ + struct guc_state_capture_header_t header; + /** @mmio_entries: Array of captured guc_mmio_reg entries. */ + struct guc_mmio_reg mmio_entries[]; +} __packed; + +/* State Capture Group Type */ +enum guc_state_capture_group_type { + GUC_STATE_CAPTURE_GROUP_TYPE_FULL = 0, + GUC_STATE_CAPTURE_GROUP_TYPE_PARTIAL +}; + +#define GUC_STATE_CAPTURE_GROUP_TYPE_MAX (GUC_STATE_CAPTURE_GROUP_TYPE_PARTIAL + 1) + +/** + * struct guc_state_capture_group_header_t - State capture group header + * + * State capture group header. + */ +struct guc_state_capture_group_header_t { + /** @owner: VFID */ + u32 owner; +#define GUC_STATE_CAPTURE_GROUP_HEADER_VFID GENMASK(7, 0) + /** @info: Engine class/instance and capture type info */ + u32 info; +#define GUC_STATE_CAPTURE_GROUP_HEADER_NUM_CAPTURES GENMASK(7, 0) +#define GUC_STATE_CAPTURE_GROUP_HEADER_CAPTURE_GROUP_TYPE GENMASK(15, 8) +} __packed; + +/** + * struct guc_state_capture_group_t - State capture group. + * + * this is the top level structure where an error-capture dump starts + */ +struct guc_state_capture_group_t { + /** @grp_header: State capture group header. */ + struct guc_state_capture_group_header_t grp_header; + /** @capture_entries: Array of state captures */ + struct guc_state_capture_t capture_entries[]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h b/drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h new file mode 100644 index 000000000..f58198cf2 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_COMMUNICATION_CTB_ABI_H +#define _ABI_GUC_COMMUNICATION_CTB_ABI_H + +#include +#include + +#include "guc_messages_abi.h" + +/** + * DOC: CT Buffer + * + * Circular buffer used to send `CTB Message`_ + */ + +/** + * DOC: CTB Descriptor + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:0 | **HEAD** - offset (in dwords) to the last dword that was | + * | | | read from the `CT Buffer`_. | + * | | | It can only be updated by the receiver. | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **TAIL** - offset (in dwords) to the last dword that was | + * | | | written to the `CT Buffer`_. | + * | | | It can only be updated by the sender. | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **STATUS** - status of the CTB | + * | | | | + * | | | - _`GUC_CTB_STATUS_NO_ERROR` = 0 (normal operation) | + * | | | - _`GUC_CTB_STATUS_OVERFLOW` = 1 (head/tail too large) | + * | | | - _`GUC_CTB_STATUS_UNDERFLOW` = 2 (truncated message) | + * | | | - _`GUC_CTB_STATUS_MISMATCH` = 4 (head/tail modified) | + * +---+-------+--------------------------------------------------------------+ + * |...| | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + * | 15| 31:0 | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ + +struct guc_ct_buffer_desc { + u32 head; + u32 tail; + u32 status; +#define GUC_CTB_STATUS_NO_ERROR 0 +#define GUC_CTB_STATUS_OVERFLOW (1 << 0) +#define GUC_CTB_STATUS_UNDERFLOW (1 << 1) +#define GUC_CTB_STATUS_MISMATCH (1 << 2) +#define GUC_CTB_STATUS_DISABLED (1 << 3) + u32 reserved[13]; +} __packed; +static_assert(sizeof(struct guc_ct_buffer_desc) == 64); + +/** + * DOC: CTB Message + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | **FENCE** - message identifier | + * | +-------+--------------------------------------------------------------+ + * | | 15:12 | **FORMAT** - format of the CTB message | + * | | | - _`GUC_CTB_FORMAT_HXG` = 0 - see `CTB HXG Message`_ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | **RESERVED** | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | **NUM_DWORDS** - length of the CTB message (w/o header) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | optional (depends on FORMAT) | + * +---+-------+ | + * |...| | | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_CTB_HDR_LEN 1u +#define GUC_CTB_MSG_MIN_LEN GUC_CTB_HDR_LEN +#define GUC_CTB_MSG_MAX_LEN (GUC_CTB_MSG_MIN_LEN + GUC_CTB_MAX_DWORDS) +#define GUC_CTB_MSG_0_FENCE (0xffffu << 16) +#define GUC_CTB_MSG_0_FORMAT (0xfu << 12) +#define GUC_CTB_FORMAT_HXG 0u +#define GUC_CTB_MSG_0_RESERVED (0xfu << 8) +#define GUC_CTB_MSG_0_NUM_DWORDS (0xffu << 0) +#define GUC_CTB_MAX_DWORDS 255 + +/** + * DOC: CTB HXG Message + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | FENCE | + * | +-------+--------------------------------------------------------------+ + * | | 15:12 | FORMAT = GUC_CTB_FORMAT_HXG_ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | RESERVED = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | NUM_DWORDS = length (in dwords) of the embedded HXG message | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | [Embedded `HXG Message`_] | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_CTB_HXG_MSG_MIN_LEN (GUC_CTB_MSG_MIN_LEN + GUC_HXG_MSG_MIN_LEN) +#define GUC_CTB_HXG_MSG_MAX_LEN GUC_CTB_MSG_MAX_LEN + +/** + * DOC: CTB based communication + * + * The CTB (command transport buffer) communication between Host and GuC + * is based on u32 data stream written to the shared buffer. One buffer can + * be used to transmit data only in one direction (one-directional channel). + * + * Current status of the each buffer is maintained in the `CTB Descriptor`_. + * Each message in data stream is encoded as `CTB HXG Message`_. + */ + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h b/drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h new file mode 100644 index 000000000..ef538e34f --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_COMMUNICATION_MMIO_ABI_H +#define _ABI_GUC_COMMUNICATION_MMIO_ABI_H + +/** + * DOC: GuC MMIO based communication + * + * The MMIO based communication between Host and GuC relies on special + * hardware registers which format could be defined by the software + * (so called scratch registers). + * + * Each MMIO based message, both Host to GuC (H2G) and GuC to Host (G2H) + * messages, which maximum length depends on number of available scratch + * registers, is directly written into those scratch registers. + * + * For Gen9+, there are 16 software scratch registers 0xC180-0xC1B8, + * but no H2G command takes more than 4 parameters and the GuC firmware + * itself uses an 4-element array to store the H2G message. + * + * For Gen11+, there are additional 4 registers 0x190240-0x19024C, which + * are, regardless on lower count, preferred over legacy ones. + * + * The MMIO based communication is mainly used during driver initialization + * phase to setup the `CTB based communication`_ that will be used afterwards. + */ + +#define GUC_MAX_MMIO_MSG_LEN 4 + +/** + * DOC: MMIO HXG Message + * + * Format of the MMIO messages follows definitions of `HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:0 | | + * +---+-------+ | + * |...| | [Embedded `HXG Message`_] | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_errors_abi.h b/drivers/gpu/drm/xe/abi/guc_errors_abi.h new file mode 100644 index 000000000..ad76b4baf --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_errors_abi.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_ERRORS_ABI_H +#define _ABI_GUC_ERRORS_ABI_H + +enum xe_guc_response { + XE_GUC_RESPONSE_ERROR_PROTOCOL = 0x04, + XE_GUC_RESPONSE_INVALID_STATE = 0x0A, + XE_GUC_RESPONSE_UNSUPPORTED_VERSION = 0x0B, + XE_GUC_RESPONSE_INVALID_VFID = 0x0C, + XE_GUC_RESPONSE_UNPROVISIONED_VF = 0x0D, + XE_GUC_RESPONSE_INVALID_EVENT = 0x0E, + XE_GUC_RESPONSE_NOT_SUPPORTED = 0x20, + XE_GUC_RESPONSE_UNKNOWN_ACTION = 0x30, + XE_GUC_RESPONSE_ACTION_ABORTED = 0x31, + XE_GUC_RESPONSE_NO_PERMISSION = 0x40, + XE_GUC_RESPONSE_CANNOT_COMPLETE_ACTION = 0x41, + XE_GUC_RESPONSE_INVALID_KLV_DATA = 0x50, + XE_GUC_RESPONSE_INVALID_PARAMS = 0x60, + XE_GUC_RESPONSE_INVALID_CONTEXT_INDEX = 0x61, + XE_GUC_RESPONSE_INVALID_CONTEXT_REGISTRATION = 0x62, + XE_GUC_RESPONSE_INVALID_DOORBELL_ID = 0x63, + XE_GUC_RESPONSE_INVALID_ENGINE_ID = 0x64, + XE_GUC_RESPONSE_INVALID_BUFFER_RANGE = 0x70, + XE_GUC_RESPONSE_INVALID_BUFFER = 0x71, + XE_GUC_RESPONSE_BUFFER_ALREADY_REGISTERED = 0x72, + XE_GUC_RESPONSE_INVALID_GGTT_ADDRESS = 0x80, + XE_GUC_RESPONSE_PENDING_ACTION = 0x90, + XE_GUC_RESPONSE_CONTEXT_NOT_REGISTERED = 0x100, + XE_GUC_RESPONSE_CONTEXT_ALREADY_REGISTERED = 0X101, + XE_GUC_RESPONSE_INVALID_SIZE = 0x102, + XE_GUC_RESPONSE_MALFORMED_KLV = 0x103, + XE_GUC_RESPONSE_INVALID_CONTEXT = 0x104, + XE_GUC_RESPONSE_INVALID_KLV_KEY = 0x105, + XE_GUC_RESPONSE_DATA_TOO_LARGE = 0x106, + XE_GUC_RESPONSE_VF_MIGRATED = 0x107, + XE_GUC_RESPONSE_NO_ATTRIBUTE_TABLE = 0x201, + XE_GUC_RESPONSE_NO_DECRYPTION_KEY = 0x202, + XE_GUC_RESPONSE_DECRYPTION_FAILED = 0x204, + XE_GUC_RESPONSE_VGT_DISABLED = 0x300, + XE_GUC_RESPONSE_CTB_FULL = 0x301, + XE_GUC_RESPONSE_VGT_UNAUTHORIZED_REQUEST = 0x302, + XE_GUC_RESPONSE_CTB_INVALID = 0x303, + XE_GUC_RESPONSE_CTB_NOT_REGISTERED = 0x304, + XE_GUC_RESPONSE_CTB_IN_USE = 0x305, + XE_GUC_RESPONSE_CTB_INVALID_DESC = 0x306, + XE_GUC_RESPONSE_HW_TIMEOUT = 0x30C, + XE_GUC_RESPONSE_CTB_SOURCE_INVALID_DESCRIPTOR = 0x30D, + XE_GUC_RESPONSE_CTB_DESTINATION_INVALID_DESCRIPTOR = 0x30E, + XE_GUC_RESPONSE_INVALID_CONFIG_STATE = 0x30F, + XE_GUC_RESPONSE_GENERIC_FAIL = 0xF000, +}; + +enum xe_guc_load_status { + XE_GUC_LOAD_STATUS_DEFAULT = 0x00, + XE_GUC_LOAD_STATUS_START = 0x01, + XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH = 0x02, + XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH = 0x03, + XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE = 0x04, + XE_GUC_LOAD_STATUS_HWCONFIG_START = 0x05, + XE_GUC_LOAD_STATUS_HWCONFIG_DONE = 0x06, + XE_GUC_LOAD_STATUS_HWCONFIG_ERROR = 0x07, + XE_GUC_LOAD_STATUS_BOOTROM_VERSION_MISMATCH = 0x08, + XE_GUC_LOAD_STATUS_GDT_DONE = 0x10, + XE_GUC_LOAD_STATUS_IDT_DONE = 0x20, + XE_GUC_LOAD_STATUS_LAPIC_DONE = 0x30, + XE_GUC_LOAD_STATUS_GUCINT_DONE = 0x40, + XE_GUC_LOAD_STATUS_DPC_READY = 0x50, + XE_GUC_LOAD_STATUS_DPC_ERROR = 0x60, + XE_GUC_LOAD_STATUS_EXCEPTION = 0x70, + XE_GUC_LOAD_STATUS_INIT_DATA_INVALID = 0x71, + XE_GUC_LOAD_STATUS_PXP_TEARDOWN_CTRL_ENABLED = 0x72, + XE_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_START, + XE_GUC_LOAD_STATUS_MPU_DATA_INVALID = 0x73, + XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID = 0x74, + XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR = 0x75, + XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG = 0x76, + XE_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_END, + + XE_GUC_LOAD_STATUS_READY = 0xF0, +}; + +enum xe_bootrom_load_status { + XE_BOOTROM_STATUS_NO_KEY_FOUND = 0x13, + XE_BOOTROM_STATUS_AES_PROD_KEY_FOUND = 0x1A, + XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE = 0x2B, + XE_BOOTROM_STATUS_RSA_FAILED = 0x50, + XE_BOOTROM_STATUS_PAVPC_FAILED = 0x73, + XE_BOOTROM_STATUS_WOPCM_FAILED = 0x74, + XE_BOOTROM_STATUS_LOADLOC_FAILED = 0x75, + XE_BOOTROM_STATUS_JUMP_PASSED = 0x76, + XE_BOOTROM_STATUS_JUMP_FAILED = 0x77, + XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED = 0x79, + XE_BOOTROM_STATUS_MPUMAP_INCORRECT = 0x7A, + XE_BOOTROM_STATUS_EXCEPTION = 0x7E, +}; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h new file mode 100644 index 000000000..685c4ef17 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -0,0 +1,546 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _ABI_GUC_KLVS_ABI_H +#define _ABI_GUC_KLVS_ABI_H + +#include + +#include "abi/guc_scheduler_abi.h" + +/** + * DOC: GuC KLV + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | **KEY** - KLV key identifier | + * | | | - `GuC Self Config KLVs`_ | + * | | | - `GuC Opt In Feature KLVs`_ | + * | | | - `GuC Scheduling Policies KLVs`_ | + * | | | - `GuC VGT Policy KLVs`_ | + * | | | - `GuC VF Configuration KLVs`_ | + * | | | - `GuC Reserved KLVs`_ | + * | | | | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **LEN** - length of VALUE (in 32bit dwords) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VALUE** - actual value of the KLV (format depends on KEY) | + * +---+-------+ | + * |...| | | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_KLV_LEN_MIN 1u +#define GUC_KLV_0_KEY (0xffffu << 16) +#define GUC_KLV_0_LEN (0xffffu << 0) +#define GUC_KLV_n_VALUE (0xffffffffu << 0) + +/** + * DOC: GuC Global Config KLVs + * + * `GuC KLV`_ keys available for use with HOST2GUC_SELF_CFG_. + * + * _`GUC_KLV_GLOBAL_CFG_GMD_ID` : 0x3000 + * Refers to 32 bit architecture version as reported by the HW IP. + * This key is supported on MTL+ platforms only. + * Requires GuC ABI 1.2+. + * + * _`GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE` : 0x3001 + * Tells the driver whether scheduler groups are enabled or not. + * Requires GuC ABI 1.26+ + * + * _`GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES` : 0x3003 + * Tells the driver the paging engine configuration. + * Paging engine logical instances are guaranteed to be dense starting at + * index 0. + * Requires GuC ABI 1.36+ + */ + +#define GUC_KLV_GLOBAL_CFG_GMD_ID_KEY 0x3000u +#define GUC_KLV_GLOBAL_CFG_GMD_ID_LEN 1u + +#define GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY 0x3001u +#define GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_LEN 1u + +#define GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_KEY 0x3003u +#define GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_LEN 1u + +/** + * DOC: GuC Self Config KLVs + * + * `GuC KLV`_ keys available for use with HOST2GUC_SELF_CFG_. + * + * _`GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR` : 0x0900 + * Refers to 64 bit Global Gfx address (in bytes) of memory based interrupts + * status vector for use by the GuC. + * + * _`GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR` : 0x0901 + * Refers to 64 bit Global Gfx address (in bytes) of memory based interrupts + * source vector for use by the GuC. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_ADDR` : 0x0902 + * Refers to 64 bit Global Gfx address of H2G `CT Buffer`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR : 0x0903 + * Refers to 64 bit Global Gfx address of H2G `CTB Descriptor`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_SIZE : 0x0904 + * Refers to size of H2G `CT Buffer`_ in bytes. + * Should be a multiple of 4K. + * + * _`GUC_KLV_SELF_CFG_G2H_CTB_ADDR : 0x0905 + * Refers to 64 bit Global Gfx address of G2H `CT Buffer`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR : 0x0906 + * Refers to 64 bit Global Gfx address of G2H `CTB Descriptor`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _GUC_KLV_SELF_CFG_G2H_CTB_SIZE : 0x0907 + * Refers to size of G2H `CT Buffer`_ in bytes. + * Should be a multiple of 4K. + */ + +#define GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY 0x0900 +#define GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY 0x0901 +#define GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY 0x0902 +#define GUC_KLV_SELF_CFG_H2G_CTB_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY 0x0903 +#define GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY 0x0904 +#define GUC_KLV_SELF_CFG_H2G_CTB_SIZE_LEN 1u + +#define GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY 0x0905 +#define GUC_KLV_SELF_CFG_G2H_CTB_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY 0x0906 +#define GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY 0x0907 +#define GUC_KLV_SELF_CFG_G2H_CTB_SIZE_LEN 1u + +/* + * Per context scheduling policy update keys. + */ +enum { + GUC_CONTEXT_POLICIES_KLV_ID_EXECUTION_QUANTUM = 0x2001, + GUC_CONTEXT_POLICIES_KLV_ID_PREEMPTION_TIMEOUT = 0x2002, + GUC_CONTEXT_POLICIES_KLV_ID_SCHEDULING_PRIORITY = 0x2003, + GUC_CONTEXT_POLICIES_KLV_ID_PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY = 0x2004, + GUC_CONTEXT_POLICIES_KLV_ID_SLPM_GT_FREQUENCY = 0x2005, + + GUC_CONTEXT_POLICIES_KLV_NUM_IDS = 5, +}; + +/** + * DOC: GuC Opt In Feature KLVs + * + * `GuC KLV`_ keys available for use with OPT_IN_FEATURE_KLV + * + * _`GUC_KLV_OPT_IN_FEATURE_EXT_CAT_ERR_TYPE` : 0x4001 + * Adds an extra dword to the XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR G2H + * containing the type of the CAT error. On HW that does not support + * reporting the CAT error type, the extra dword is set to 0xdeadbeef. + * + * _`GUC_KLV_OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH` : 0x4003 + * This KLV enables the Dynamic Inhibit Context Switch optimization, which + * consists in the GuC setting the CTX_CTRL_INHIBIT_SYN_CTX_SWITCH bit to + * zero in the CTX_CONTEXT_CONTROL register of LRCs that are submitted + * to an oversubscribed engine. This will cause those contexts to be + * switched out immediately if they hit an unsatisfied semaphore wait + * (instead of waiting the full timeslice duration). The bit is instead set + * to one if a single context is queued on the engine, to avoid it being + * switched out if there isn't another context that can run in its place. + * + * _`GUC_KLV_OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION` : 0x4004 + * This flag will enable notification from GuC to KMD via G2H message + * GUC_ACTION_GUC2HOST_NOTIFY_UNCORRECTABLE_LOCAL_ERROR upon receiving the + * same interrupt from the CS. + */ + +#define GUC_KLV_OPT_IN_FEATURE_EXT_CAT_ERR_TYPE_KEY 0x4001 +#define GUC_KLV_OPT_IN_FEATURE_EXT_CAT_ERR_TYPE_LEN 0u + +#define GUC_KLV_OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH_KEY 0x4003 +#define GUC_KLV_OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH_LEN 0u + +#define GUC_KLV_OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION_KEY 0x4004 +#define GUC_KLV_OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION_LEN 0u + +/** + * DOC: GuC Scheduling Policies KLVs + * + * `GuC KLV`_ keys available for use with UPDATE_SCHEDULING_POLICIES_KLV. + * + * _`GUC_KLV_SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD` : 0x1001 + * Some platforms do not allow concurrent execution of RCS and CCS + * workloads from different address spaces. By default, the GuC prioritizes + * RCS submissions over CCS ones, which can lead to CCS workloads being + * significantly (or completely) starved of execution time. This KLV allows + * the driver to specify a quantum (in ms) and a ratio (percentage value + * between 0 and 100), and the GuC will prioritize the CCS for that + * percentage of each quantum. For example, specifying 100ms and 30% will + * make the GuC prioritize the CCS for 30ms of every 100ms. + * Note that this does not necessarly mean that RCS and CCS engines will + * only be active for their percentage of the quantum, as the restriction + * only kicks in if both classes are fully busy with non-compatible address + * spaces; i.e., if one engine is idle or running the same address space, + * a pending job on the other engine will still be submitted to the HW no + * matter what the ratio is + */ +#define GUC_KLV_SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD_KEY 0x1001 +#define GUC_KLV_SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD_LEN 2u + +/** + * DOC: GuC VGT Policy KLVs + * + * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VGT_POLICY. + * + * _`GUC_KLV_VGT_POLICY_SCHED_IF_IDLE` : 0x8001 + * [From 70.12.0] + * This config allows to update scheduling priority of PF and all VFs at once. + * Setting this policy to 0 updates all VFs scheduling priorities to LOW, and + * setting this policy to 1 updates all VFs scheduling priorities to NORMAL. + * Those changes will take effect on the next VF-Switch event. + * + * [Before 70.12.0] + * This config sets whether strict scheduling is enabled whereby any VF + * that doesn’t have work to submit is still allocated a fixed execution + * time-slice to ensure active VFs execution is always consistent even + * during other VF reprovisiong / rebooting events. Changing this KLV + * impacts all VFs and takes effect on the next VF-Switch event. + * + * :0: don't schedule idle (default) + * :1: schedule if idle + * + * _`GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD` : 0x8002 + * This config sets the sample period for tracking adverse event counters. + * A sample period is the period in millisecs during which events are counted. + * This is applicable for all the VFs. + * + * :0: adverse events are not counted (default) + * :n: sample period in milliseconds + * + * _`GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG` : 0x8004 + * This config allows the PF to split the engines across scheduling groups. + * Each group is independently timesliced across VFs, allowing different + * VFs to be active on the HW at the same time. When enabling this feature, + * all engines must be assigned to a group (and only one group), or they + * will be excluded from scheduling after this KLV is sent. To enable + * the groups, the driver must provide a masks array with + * GUC_MAX_ENGINE_CLASSES entries for each group, with each mask indicating + * which logical instances of that class belong to the group. Therefore, + * the length of this KLV when enabling groups is + * num_groups * GUC_MAX_ENGINE_CLASSES. To disable the groups, the driver + * must send the KLV without any payload (i.e. len = 0). The maximum + * number of groups is 8. + * + * _`GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH` : 0x8D00 + * This enum is to reset utilized HW engine after VF Switch (i.e to clean + * up Stale HW register left behind by previous VF) + * + * :0: don't reset (default) + * :1: reset + */ + +#define GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY 0x8001 +#define GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_LEN 1u + +#define GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY 0x8002 +#define GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_LEN 1u + +#define GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_KEY 0x8004 +#define GUC_KLV_VGT_POLICY_ENGINE_GROUP_MAX_COUNT GUC_MAX_SCHED_GROUPS +#define GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_MIN_LEN 0 +#define GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_MAX_LEN \ + (GUC_KLV_VGT_POLICY_ENGINE_GROUP_MAX_COUNT * GUC_MAX_ENGINE_CLASSES) + +#define GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY 0x8D00 +#define GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_LEN 1u + +/** + * DOC: GuC VF Configuration KLVs + * + * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VF_CFG. + * + * _`GUC_KLV_VF_CFG_GGTT_START` : 0x0001 + * A 4K aligned start GTT address/offset assigned to VF. + * Value is 64 bits. + * + * _`GUC_KLV_VF_CFG_GGTT_SIZE` : 0x0002 + * A 4K aligned size of GGTT assigned to VF. + * Value is 64 bits. + * + * _`GUC_KLV_VF_CFG_LMEM_SIZE` : 0x0003 + * A 2M aligned size of local memory assigned to VF. + * Value is 64 bits. + * + * _`GUC_KLV_VF_CFG_NUM_CONTEXTS` : 0x0004 + * Refers to the number of contexts allocated to this VF. + * + * :0: no contexts (default) + * :1-65535: number of contexts (Gen12) + * + * _`GUC_KLV_VF_CFG_TILE_MASK` : 0x0005 + * For multi-tiled products, this field contains the bitwise-OR of tiles + * assigned to the VF. Bit-0-set means VF has access to Tile-0, + * Bit-31-set means VF has access to Tile-31, and etc. + * At least one tile will always be allocated. + * If all bits are zero, VF KMD should treat this as a fatal error. + * For, single-tile products this KLV config is ignored. + * + * _`GUC_KLV_VF_CFG_NUM_DOORBELLS` : 0x0006 + * Refers to the number of doorbells allocated to this VF. + * + * :0: no doorbells (default) + * :1-255: number of doorbells (Gen12) + * + * _`GUC_KLV_VF_CFG_EXEC_QUANTUM` : 0x8A01 + * This config sets the VFs-execution-quantum in milliseconds. + * GUC will attempt to obey the maximum values as much as HW is capable + * of and this will never be perfectly-exact (accumulated nano-second + * granularity) since the GPUs clock time runs off a different crystal + * from the CPUs clock. Changing this KLV on a VF that is currently + * running a context won't take effect until a new context is scheduled in. + * That said, when the PF is changing this value from 0x0 to + * a non-zero value, it might never take effect if the VF is running an + * infinitely long compute or shader kernel. In such a scenario, the + * PF would need to trigger a VM PAUSE and then change the KLV to force + * it to take effect. Such cases might typically happen on a 1PF+1VF + * Virtualization config enabled for heavier workloads like AI/ML. + * + * If scheduling groups are supported, the provided value is applied to all + * groups (even if they've not yet been enabled). Support for this feature + * is available from GuC 70.53.0. + * + * The max value for this KLV is 100 seconds, anything exceeding that + * will be clamped to the max. + * + * :0: infinite exec quantum (default) + * :100000: maximum exec quantum (100000ms == 100s) + * + * _`GUC_KLV_VF_CFG_PREEMPT_TIMEOUT` : 0x8A02 + * This config sets the VF-preemption-timeout in microseconds. + * GUC will attempt to obey the minimum and maximum values as much as + * HW is capable and this will never be perfectly-exact (accumulated + * nano-second granularity) since the GPUs clock time runs off a + * different crystal from the CPUs clock. Changing this KLV on a VF + * that is currently running a context won't take effect until a new + * context is scheduled in. + * That said, when the PF is changing this value from 0x0 to + * a non-zero value, it might never take effect if the VF is running an + * infinitely long compute or shader kernel. + * In this case, the PF would need to trigger a VM PAUSE and then change + * the KLV to force it to take effect. Such cases might typically happen + * on a 1PF+1VF Virtualization config enabled for heavier workloads like + * AI/ML. + * + * If scheduling groups are supported, the provided value is applied to all + * groups (even if they've not yet been enabled). Support for this feature + * is available from GuC 70.53.0. + * + * The max value for this KLV is 100 seconds, anything exceeding that + * will be clamped to the max. + * + * :0: no preemption timeout (default) + * :100000000: maximum preemption timeout (100000000us == 100s) + * + * _`GUC_KLV_VF_CFG_THRESHOLD_CAT_ERR` : 0x8A03 + * This config sets threshold for CAT errors caused by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: event occurrence count per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_ENGINE_RESET` : 0x8A04 + * This config sets threshold for engine reset caused by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: event occurrence count per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_PAGE_FAULT` : 0x8A05 + * This config sets threshold for page fault errors caused by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: event occurrence count per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_H2G_STORM` : 0x8A06 + * This config sets threshold for H2G interrupts triggered by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: time (us) per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_IRQ_STORM` : 0x8A07 + * This config sets threshold for GT interrupts triggered by the VF's + * workloads. + * + * :0: adverse events or error will not be reported (default) + * :n: time (us) per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_DOORBELL_STORM` : 0x8A08 + * This config sets threshold for doorbell's ring triggered by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: time (us) per sampling interval + * + * _`GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID` : 0x8A0A + * Refers to the start index of doorbell assigned to this VF. + * + * :0: (default) + * :1-255: number of doorbells (Gen12) + * + * _`GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID` : 0x8A0B + * Refers to the start index in context array allocated to this VF’s use. + * + * :0: (default) + * :1-65535: number of contexts (Gen12) + * + * _`GUC_KLV_VF_CFG_SCHED_PRIORITY` : 0x8A0C + * This config controls VF’s scheduling priority. + * + * :0: LOW = schedule VF only if it has active work (default) + * :1: NORMAL = schedule VF always, irrespective of whether it has work or not + * :2: HIGH = schedule VF in the next time-slice after current active + * time-slice completes if it has active work + * + * _`GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT` : 0x8A0D + * Given that multi-LRC contexts are incompatible with SRIOV scheduler + * groups and cause the latter to be turned off when registered with the + * GuC, this config allows the PF to set a threshold for multi-LRC context + * registrations by VFs to monitor their behavior. + * + * _`GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM' : 0x8A0E + * This config sets the VFs-execution-quantum for each scheduling group in + * milliseconds. The driver must provide an array of values, with each of + * them matching the respective group index (first value goes to group 0, + * second to group 1, etc). The setting of group values follows the same + * behavior and rules as setting via GUC_KLV_VF_CFG_EXEC_QUANTUM. Note that + * the GuC always sets the EQ for all groups (even the non-enabled ones), + * so if we provide fewer values than the max the GuC will use 0 for the + * remaining groups. This KLV is available starting from GuC 70.53.0. + * + * _`GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT' : 0x8A0F + * This config sets the VFs-preemption-timeout for each scheduling group in + * microseconds. The driver must provide an array of values, with each of + * them matching the respective group index (first value goes to group 0, + * second to group 1, etc). The setting of group values follows the same + * behavior and rules as setting via GUC_KLV_VF_CFG_PREEMPT_TIMEOUT. Note + * that the GuC always sets the EQ for all groups (even the non-enabled + * ones), so if we provide fewer values than the max the GuC will use 0 for + * the remaining groups. This KLV is available starting from GuC 70.53.0. + */ + +#define GUC_KLV_VF_CFG_GGTT_START_KEY 0x0001 +#define GUC_KLV_VF_CFG_GGTT_START_LEN 2u + +#define GUC_KLV_VF_CFG_GGTT_SIZE_KEY 0x0002 +#define GUC_KLV_VF_CFG_GGTT_SIZE_LEN 2u + +#define GUC_KLV_VF_CFG_LMEM_SIZE_KEY 0x0003 +#define GUC_KLV_VF_CFG_LMEM_SIZE_LEN 2u + +#define GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY 0x0004 +#define GUC_KLV_VF_CFG_NUM_CONTEXTS_LEN 1u + +#define GUC_KLV_VF_CFG_TILE_MASK_KEY 0x0005 +#define GUC_KLV_VF_CFG_TILE_MASK_LEN 1u + +#define GUC_KLV_VF_CFG_NUM_DOORBELLS_KEY 0x0006 +#define GUC_KLV_VF_CFG_NUM_DOORBELLS_LEN 1u + +#define GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY 0x8a01 +#define GUC_KLV_VF_CFG_EXEC_QUANTUM_LEN 1u +#define GUC_KLV_VF_CFG_EXEC_QUANTUM_MAX_VALUE 100000u + +#define GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY 0x8a02 +#define GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_LEN 1u +#define GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_MAX_VALUE 100000000u + +#define GUC_KLV_VF_CFG_THRESHOLD_CAT_ERR_KEY 0x8a03 +#define GUC_KLV_VF_CFG_THRESHOLD_CAT_ERR_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_ENGINE_RESET_KEY 0x8a04 +#define GUC_KLV_VF_CFG_THRESHOLD_ENGINE_RESET_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_PAGE_FAULT_KEY 0x8a05 +#define GUC_KLV_VF_CFG_THRESHOLD_PAGE_FAULT_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_H2G_STORM_KEY 0x8a06 +#define GUC_KLV_VF_CFG_THRESHOLD_H2G_STORM_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_IRQ_STORM_KEY 0x8a07 +#define GUC_KLV_VF_CFG_THRESHOLD_IRQ_STORM_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_DOORBELL_STORM_KEY 0x8a08 +#define GUC_KLV_VF_CFG_THRESHOLD_DOORBELL_STORM_LEN 1u + +#define GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID_KEY 0x8a0a +#define GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID_LEN 1u + +#define GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_KEY 0x8a0b +#define GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_LEN 1u + +#define GUC_KLV_VF_CFG_SCHED_PRIORITY_KEY 0x8a0c +#define GUC_KLV_VF_CFG_SCHED_PRIORITY_LEN 1u +#define GUC_SCHED_PRIORITY_LOW 0u +#define GUC_SCHED_PRIORITY_NORMAL 1u +#define GUC_SCHED_PRIORITY_HIGH 2u + +#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_KEY 0x8a0d +#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_LEN 1u + +#define GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY 0x8a0e +#define GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MIN_LEN 1u +#define GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MAX_LEN GUC_MAX_SCHED_GROUPS + +#define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY 0x8a0f +#define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MIN_LEN 1u +#define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN GUC_MAX_SCHED_GROUPS +/* + * Feature and Workaround keys: + */ +enum xe_guc_klv_ids { + GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE = 0x5001, + GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED = 0x9002, + GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT = 0x9004, + GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING = 0x9005, + GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE = 0x9007, + GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE = 0x9008, + GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET = 0x9009, + GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO = 0x900a, + GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH = 0x900b, + GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG = 0x900c, + GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG = 0x900e, + GUC_WA_KLV_REMAP_RANGED_TLB_INV = 0x900f, + GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64 = 0x9010, +}; + +/** + * DOC: GuC Reserved KLVs + * + * Range of `GuC KLV`_ keys reserved for internal use by the GuC that will + * never be part of the offcial GuC ABI and can be reused by the drivers. + * + * Currently this range includes 1024 keys starting from: + * + * _`GUC_KLV_RESERVED_RANGE_START` : 0xF000 + * + * See `Xe Driver KLVs`_ for the KLVs that the Xe driver is currently using. + */ + +#define GUC_KLV_RESERVED_RANGE_START 0xf000u +#define GUC_KLV_RESERVED_RANGE_LEN 1024u + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_lfd_abi.h b/drivers/gpu/drm/xe/abi/guc_lfd_abi.h new file mode 100644 index 000000000..5c48459f3 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_lfd_abi.h @@ -0,0 +1,170 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _ABI_GUC_LFD_ABI_H_ +#define _ABI_GUC_LFD_ABI_H_ + +#include + +#include "guc_lic_abi.h" + +/* The current major version of GuC-Log-File format. */ +#define GUC_LFD_FORMAT_VERSION_MAJOR 0x0001 +/* The current minor version of GuC-Log-File format. */ +#define GUC_LFD_FORMAT_VERSION_MINOR 0x0000 + +/** enum guc_lfd_type - Log format descriptor type */ +enum guc_lfd_type { + /** + * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_START: Start of range for + * required LFDs from GuC + * @GUC_LFD_TYPE_FW_VERSION: GuC Firmware Version structure. + * @GUC_LFD_TYPE_GUC_DEVICE_ID: GuC microcontroller device ID. + * @GUC_LFD_TYPE_TSC_FREQUENCY: Frequency of GuC timestamps. + * @GUC_LFD_TYPE_GMD_ID: HW GMD ID. + * @GUC_LFD_TYPE_BUILD_PLATFORM_ID: GuC build platform ID. + * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_END: End of range for + * required LFDs from GuC + */ + GUC_LFD_TYPE_FW_REQUIRED_RANGE_START = 0x1, + GUC_LFD_TYPE_FW_VERSION = 0x1, + GUC_LFD_TYPE_GUC_DEVICE_ID = 0x2, + GUC_LFD_TYPE_TSC_FREQUENCY = 0x3, + GUC_LFD_TYPE_GMD_ID = 0x4, + GUC_LFD_TYPE_BUILD_PLATFORM_ID = 0x5, + GUC_LFD_TYPE_FW_REQUIRED_RANGE_END = 0x1FFF, + + /** + * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START: Start of range for + * optional LFDs from GuC + * @GUC_LFD_TYPE_LOG_EVENTS_BUFFER: Log-event-entries buffer. + * @GUC_LFD_TYPE_FW_CRASH_DUMP: GuC generated crash-dump blob. + * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END: End of range for + * optional LFDs from GuC + */ + GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START = 0x2000, + GUC_LFD_TYPE_LOG_EVENTS_BUFFER = 0x2000, + GUC_LFD_TYPE_FW_CRASH_DUMP = 0x2001, + GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END = 0x3FFF, + + /** + * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START: Start of range for + * required KMD LFDs + * @GUC_LFD_TYPE_OS_ID: An identifier for the OS. + * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END: End of this range for + * required KMD LFDs + */ + GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START = 0x4000, + GUC_LFD_TYPE_OS_ID = 0x4000, + GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END = 0x5FFF, + + /** + * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START: Start of range for + * optional KMD LFDs + * @GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT: Binary representation of + * GuC log-events schema. + * @GUC_LFD_TYPE_HOST_COMMENT: ASCII string containing comments + * from the host/KMD. + * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR: A timestamp anchor, to convert + * between host and GuC timestamp. + * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG: Timestamp anchor + * configuration, definition of timestamp frequency and bit width. + * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END: End of this range for + * optional KMD LFDs + */ + GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START = 0x6000, + GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT = 0x6000, + GUC_LFD_TYPE_HOST_COMMENT = 0x6001, + GUC_LFD_TYPE_TIMESTAMP_ANCHOR = 0x6002, + GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG = 0x6003, + GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END = 0x7FFF, + + /* + * @GUC_LFD_TYPE_RESERVED_RANGE_START: Start of reserved range + * @GUC_LFD_TYPE_RESERVED_RANGE_END: End of reserved range + */ + GUC_LFD_TYPE_RESERVED_RANGE_START = 0x8000, + GUC_LFD_TYPE_RESERVED_RANGE_END = 0xFFFF, +}; + +/** enum guc_lfd_os_type - OS Type LFD-ID */ +enum guc_lfd_os_type { + /** @GUC_LFD_OS_TYPE_OSID_WIN: Windows OS */ + GUC_LFD_OS_TYPE_OSID_WIN = 0x1, + /** @GUC_LFD_OS_TYPE_OSID_LIN: Linux OS */ + GUC_LFD_OS_TYPE_OSID_LIN = 0x2, + /** @GUC_LFD_OS_TYPE_OSID_VMW: VMWare OS */ + GUC_LFD_OS_TYPE_OSID_VMW = 0x3, + /** @GUC_LFD_OS_TYPE_OSID_OTHER: Other */ + GUC_LFD_OS_TYPE_OSID_OTHER = 0x4, +}; + +/** struct guc_lfd_data - A generic header structure for all LFD blocks */ +struct guc_lfd_data { + /** @header: A 32 bits dword, contains multiple bit fields */ + u32 header; + /* LFD type. See guc_lfd_type */ +#define GUC_LFD_DATA_HEADER_MASK_TYPE GENMASK(31, 16) +#define GUC_LFD_DATA_HEADER_MASK_MAGIC GENMASK(15, 0) + + /** @data_count: Number of dwords the `data` field contains. */ + u32 data_count; + /** @data: Data defined by GUC_LFD_DATA_HEADER_MASK_TYPE */ + u32 data[] __counted_by(data_count); +} __packed; + +/** + * struct guc_lfd_data_log_events_buf - GuC Log Events Buffer. + * This is optional fw LFD data + */ +struct guc_lfd_data_log_events_buf { + /** + * @log_events_format_version: version of GuC log format of buffer + */ + u32 log_events_format_version; + /** + * @log_event: The log event data. + * Size in dwords is LFD block size - 1. + */ + u32 log_event[]; +} __packed; + +/** struct guc_lfd_data_os_info - OS Version Information. */ +struct guc_lfd_data_os_info { + /** + * @os_id: enum values to identify the OS brand. + * See guc_lfd_os_type for the range of types + */ + u32 os_id; + /** + * @build_version: ASCII string containing OS build version + * information based on os_id. String is padded with null + * characters to ensure its DWORD aligned. + * Size in dwords is LFD block size - 1. + */ + char build_version[]; +} __packed; + +/** + * struct guc_lfd_file_header - Header of GuC Log Streaming-LFD-File Format. + * This structure encapsulates the layout of the guc-log-file format + */ +struct guc_lfd_file_header { + /** + * @magic: A magic number set by producer of a GuC log file to + * identify that file is a valid guc-log-file containing a stream + * of LFDs. + */ + u64 magic; + /** @version: Version of this file format layout */ + u32 version; +#define GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR GENMASK(31, 16) +#define GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR GENMASK(15, 0) + + /** @stream: A stream of one or more guc_lfd_data LFD blocks */ + u32 stream[]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_lic_abi.h b/drivers/gpu/drm/xe/abi/guc_lic_abi.h new file mode 100644 index 000000000..916964409 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_lic_abi.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _ABI_GUC_LIC_ABI_H_ +#define _ABI_GUC_LIC_ABI_H_ + +#include + +/** + * enum guc_lic_type - Log Init Config KLV IDs. + */ +enum guc_lic_type { + /** + * @GUC_LIC_TYPE_GUC_SW_VERSION: GuC firmware version. Value + * is a 32 bit number represented by guc_sw_version. + */ + GUC_LIC_TYPE_GUC_SW_VERSION = 0x1, + /** + * @GUC_LIC_TYPE_GUC_DEVICE_ID: GuC device id. Value is a 32 + * bit. + */ + GUC_LIC_TYPE_GUC_DEVICE_ID = 0x2, + /** + * @GUC_LIC_TYPE_TSC_FREQUENCY: GuC timestamp counter + * frequency. Value is a 32 bit number representing frequency in + * kHz. This timestamp is utilized in log entries, timer and + * for engine utilization tracking. + */ + GUC_LIC_TYPE_TSC_FREQUENCY = 0x3, + /** + * @GUC_LIC_TYPE_GMD_ID: HW GMD ID. Value is a 32 bit number + * representing graphics, media and display HW architecture IDs. + */ + GUC_LIC_TYPE_GMD_ID = 0x4, + /** + * @GUC_LIC_TYPE_BUILD_PLATFORM_ID: GuC build platform ID. + * Value is 32 bits. + */ + GUC_LIC_TYPE_BUILD_PLATFORM_ID = 0x5, +}; + +/** + * struct guc_lic - GuC LIC (Log-Init-Config) structure. + * + * This is populated by the GUC at log init time and is located in the log + * buffer memory allocation. + */ +struct guc_lic { + /** + * @magic: A magic number set by GuC to identify that this + * structure contains valid information: magic = GUC_LIC_MAGIC. + */ + u32 magic; +#define GUC_LIC_MAGIC 0x8086900D + /** + * @version: The version of the this structure. + * Major and minor version number are represented as bit fields. + */ + u32 version; +#define GUC_LIC_VERSION_MASK_MAJOR GENMASK(31, 16) +#define GUC_LIC_VERSION_MASK_MINOR GENMASK(15, 0) + +#define GUC_LIC_VERSION_MAJOR 1u +#define GUC_LIC_VERSION_MINOR 0u + + /** @data_count: Number of dwords the `data` array contains. */ + u32 data_count; + /** + * @data: Array of dwords representing a list of LIC KLVs of + * type guc_klv_generic with keys represented by guc_lic_type + */ + u32 data[] __counted_by(data_count); +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_log_abi.h b/drivers/gpu/drm/xe/abi/guc_log_abi.h new file mode 100644 index 000000000..fbf212d59 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_log_abi.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _ABI_GUC_LOG_ABI_H +#define _ABI_GUC_LOG_ABI_H + +#include + +/** + * DOC: GuC Log buffer Layout + * + * The in-memory log buffer layout is as follows:: + * + * +===============================+ 0000h + * | Crash dump state header | ^ + * +-------------------------------+ 32B | + * | Debug state header | | + * +-------------------------------+ 64B 4KB + * | Capture state header | | + * +-------------------------------+ 96B | + * | | v + * +===============================+ <--- EVENT_DATA_OFFSET + * | Event logs(raw data) | ^ + * | | | + * | | EVENT_DATA_BUFFER_SIZE + * | | | + * | | v + * +===============================+ <--- CRASH_DUMP_OFFSET + * | Crash Dump(raw data) | ^ + * | | | + * | | CRASH_DUMP_BUFFER_SIZE + * | | | + * | | v + * +===============================+ <--- STATE_CAPTURE_OFFSET + * | Error state capture(raw data) | ^ + * | | | + * | | STATE_CAPTURE_BUFFER_SIZE + * | | | + * | | v + * +===============================+ Total: GUC_LOG_SIZE + */ + +/* GuC logging buffer types */ +enum guc_log_type { + GUC_LOG_TYPE_EVENT_DATA, + GUC_LOG_TYPE_CRASH_DUMP, + GUC_LOG_TYPE_STATE_CAPTURE, +}; + +#define GUC_LOG_BUFFER_TYPE_MAX 3 + +/** + * struct guc_log_buffer_state - GuC log buffer state + * + * Below state structure is used for coordination of retrieval of GuC firmware + * logs. Separate state is maintained for each log buffer type. + * read_ptr points to the location where Xe read last in log buffer and + * is read only for GuC firmware. write_ptr is incremented by GuC with number + * of bytes written for each log entry and is read only for Xe. + * When any type of log buffer becomes half full, GuC sends a flush interrupt. + * GuC firmware expects that while it is writing to 2nd half of the buffer, + * first half would get consumed by Host and then get a flush completed + * acknowledgment from Host, so that it does not end up doing any overwrite + * causing loss of logs. So when buffer gets half filled & Xe has requested + * for interrupt, GuC will set flush_to_file field, set the sampled_write_ptr + * to the value of write_ptr and raise the interrupt. + * On receiving the interrupt Xe should read the buffer, clear flush_to_file + * field and also update read_ptr with the value of sample_write_ptr, before + * sending an acknowledgment to GuC. marker & version fields are for internal + * usage of GuC and opaque to Xe. buffer_full_cnt field is incremented every + * time GuC detects the log buffer overflow. + */ +struct guc_log_buffer_state { + /** @marker: buffer state start marker */ + u32 marker[2]; + /** @read_ptr: the last byte offset that was read by KMD previously */ + u32 read_ptr; + /** + * @write_ptr: the next byte offset location that will be written by + * GuC + */ + u32 write_ptr; + /** @size: Log buffer size */ + u32 size; + /** + * @sampled_write_ptr: Log buffer write pointer + * This is written by GuC to the byte offset of the next free entry in + * the buffer on log buffer half full or state capture notification + */ + u32 sampled_write_ptr; + /** + * @wrap_offset: wraparound offset + * This is the byte offset of location 1 byte after last valid guc log + * event entry written by Guc firmware before there was a wraparound. + * This field is updated by guc firmware and should be used by Host + * when copying buffer contents to file. + */ + u32 wrap_offset; + /** @flags: Flush to file flag and buffer full count */ + u32 flags; +#define GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE GENMASK(0, 0) +#define GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT GENMASK(4, 1) + /** @version: The Guc-Log-Entry format version */ + u32 version; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_messages_abi.h b/drivers/gpu/drm/xe/abi/guc_messages_abi.h new file mode 100644 index 000000000..f6ed4dfd2 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_messages_abi.h @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_MESSAGES_ABI_H +#define _ABI_GUC_MESSAGES_ABI_H + +/** + * DOC: HXG Message + * + * All messages exchanged with GuC are defined using 32 bit dwords. + * First dword is treated as a message header. Remaining dwords are optional. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | | | | + * | 0 | 31 | **ORIGIN** - originator of the message | + * | | | - _`GUC_HXG_ORIGIN_HOST` = 0 | + * | | | - _`GUC_HXG_ORIGIN_GUC` = 1 | + * | | | | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | **TYPE** - message type | + * | | | - _`GUC_HXG_TYPE_REQUEST` = 0 | + * | | | - _`GUC_HXG_TYPE_EVENT` = 1 | + * | | | - _`GUC_HXG_TYPE_FAST_REQUEST` = 2 | + * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_BUSY` = 3 | + * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_RETRY` = 5 | + * | | | - _`GUC_HXG_TYPE_RESPONSE_FAILURE` = 6 | + * | | | - _`GUC_HXG_TYPE_RESPONSE_SUCCESS` = 7 | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **AUX** - auxiliary data (depends on TYPE) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **PAYLOAD** - optional payload (depends on TYPE) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_MSG_MIN_LEN 1u +#define GUC_HXG_MSG_0_ORIGIN (0x1u << 31) +#define GUC_HXG_ORIGIN_HOST 0u +#define GUC_HXG_ORIGIN_GUC 1u +#define GUC_HXG_MSG_0_TYPE (0x7u << 28) +#define GUC_HXG_TYPE_REQUEST 0u +#define GUC_HXG_TYPE_EVENT 1u +#define GUC_HXG_TYPE_FAST_REQUEST 2u +#define GUC_HXG_TYPE_NO_RESPONSE_BUSY 3u +#define GUC_HXG_TYPE_NO_RESPONSE_RETRY 5u +#define GUC_HXG_TYPE_RESPONSE_FAILURE 6u +#define GUC_HXG_TYPE_RESPONSE_SUCCESS 7u +#define GUC_HXG_MSG_0_AUX (0xfffffffu << 0) +#define GUC_HXG_MSG_n_PAYLOAD (0xffffffffu << 0) + +/** + * DOC: HXG Request + * + * The `HXG Request`_ message should be used to initiate synchronous activity + * for which confirmation or return data is expected. + * + * The recipient of this message shall use `HXG Response`_, `HXG Failure`_ + * or `HXG Retry`_ message as a definite reply, and may use `HXG Busy`_ + * message as a intermediate reply. + * + * Format of @DATA0 and all @DATAn fields depends on the @ACTION code. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **DATA0** - request data (depends on ACTION) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ACTION** - requested action code | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - optional data (depends on ACTION) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_REQUEST_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_REQUEST_MSG_0_DATA0 (0xfffu << 16) +#define GUC_HXG_REQUEST_MSG_0_ACTION (0xffffu << 0) +#define GUC_HXG_REQUEST_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/** + * DOC: HXG Fast Request + * + * The `HXG Request`_ message should be used to initiate asynchronous activity + * for which confirmation or return data is not expected. + * + * If confirmation is required then `HXG Request`_ shall be used instead. + * + * The recipient of this message may only use `HXG Failure`_ message if it was + * unable to accept this request (like invalid data). + * + * Format of `HXG Fast Request`_ message is same as `HXG Request`_ except @TYPE. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN - see `HXG Message`_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = `GUC_HXG_TYPE_FAST_REQUEST`_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 - see `HXG Request`_ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION - see `HXG Request`_ | + * +---+-------+--------------------------------------------------------------+ + * |...| | DATAn - see `HXG Request`_ | + * +---+-------+--------------------------------------------------------------+ + */ + +/** + * DOC: HXG Event + * + * The `HXG Event`_ message should be used to initiate asynchronous activity + * that does not involves immediate confirmation nor data. + * + * Format of @DATA0 and all @DATAn fields depends on the @ACTION code. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **DATA0** - event data (depends on ACTION) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ACTION** - event action code | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - optional event data (depends on ACTION) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_EVENT_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_EVENT_MSG_0_DATA0 (0xfffu << 16) +#define GUC_HXG_EVENT_MSG_0_ACTION (0xffffu << 0) +#define GUC_HXG_EVENT_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/** + * DOC: HXG Busy + * + * The `HXG Busy`_ message may be used to acknowledge reception of the `HXG Request`_ + * message if the recipient expects that it processing will be longer than default + * timeout. + * + * The @COUNTER field may be used as a progress indicator. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_NO_RESPONSE_BUSY_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **COUNTER** - progress indicator | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_BUSY_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_BUSY_MSG_0_COUNTER GUC_HXG_MSG_0_AUX + +/** + * DOC: HXG Retry + * + * The `HXG Retry`_ message should be used by recipient to indicate that the + * `HXG Request`_ message was dropped and it should be resent again. + * + * The @REASON field may be used to provide additional information. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_NO_RESPONSE_RETRY_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **REASON** - reason for retry | + * | | | - _`GUC_HXG_RETRY_REASON_UNSPECIFIED` = 0 | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_RETRY_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_RETRY_MSG_0_REASON GUC_HXG_MSG_0_AUX +#define GUC_HXG_RETRY_REASON_UNSPECIFIED 0u + +/** + * DOC: HXG Failure + * + * The `HXG Failure`_ message shall be used as a reply to the `HXG Request`_ + * message that could not be processed due to an error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_FAILURE_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **HINT** - additional error hint | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ERROR** - error/result code | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_FAILURE_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_FAILURE_MSG_0_HINT (0xfffu << 16) +#define GUC_HXG_FAILURE_MSG_0_ERROR (0xffffu << 0) + +/** + * DOC: HXG Response + * + * The `HXG Response`_ message shall be used as a reply to the `HXG Request`_ + * message that was successfully processed without an error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **DATA0** - data (depends on ACTION from `HXG Request`_) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - data (depends on ACTION from `HXG Request`_) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_RESPONSE_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_RESPONSE_MSG_0_DATA0 GUC_HXG_MSG_0_AUX +#define GUC_HXG_RESPONSE_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_relay_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_relay_actions_abi.h new file mode 100644 index 000000000..6c2834613 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_relay_actions_abi.h @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _ABI_GUC_RELAY_ACTIONS_ABI_H_ +#define _ABI_GUC_RELAY_ACTIONS_ABI_H_ + +#include "abi/guc_relay_communication_abi.h" + +/** + * DOC: GuC Relay VF/PF ABI Version + * + * The _`GUC_RELAY_VERSION_BASE` defines minimum VF/PF ABI version that + * drivers must support. Currently this is version 1.0. + * + * The _`GUC_RELAY_VERSION_LATEST` defines latest VF/PF ABI version that + * drivers may use. Currently this is version 1.0. + * + * Some platforms may require different base VF/PF ABI version. + * No supported VF/PF ABI version can be 0.0. + */ + +#define GUC_RELAY_VERSION_BASE_MAJOR 1 +#define GUC_RELAY_VERSION_BASE_MINOR 0 + +#define GUC_RELAY_VERSION_LATEST_MAJOR 1 +#define GUC_RELAY_VERSION_LATEST_MINOR 0 + +/** + * DOC: GuC Relay Actions + * + * The following actions are supported from VF/PF ABI version 1.0: + * + * * `VF2PF_HANDSHAKE`_ + * * `VF2PF_QUERY_RUNTIME`_ + */ + +/** + * DOC: VF2PF_HANDSHAKE + * + * This `Relay Message`_ is used by the VF to establish ABI version with the PF. + * + * Prior to exchanging any other messages, both VF driver and PF driver must + * negotiate the VF/PF ABI version that will be used in their communication. + * + * The VF driver shall use @MAJOR and @MINOR fields to pass requested ABI version. + * The VF driver may use special version 0.0 (both @MAJOR and @MINOR set to 0) + * to request latest (or any) ABI version that is supported by the PF driver. + * + * This message definition shall be supported by all future ABI versions. + * This message definition shall not be changed by future ABI versions. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_RELAY_ACTION_VF2PF_HANDSHAKE` = 0x0001 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:16 | **MAJOR** - requested major version of the VFPF interface | + * | | | (use MAJOR_ANY to request latest version supported by PF) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **MINOR** - requested minor version of the VFPF interface | + * | | | (use MINOR_ANY to request latest version supported by PF) | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:16 | **MAJOR** - agreed major version of the VFPF interface | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **MINOR** - agreed minor version of the VFPF interface | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_RELAY_ACTION_VF2PF_HANDSHAKE 0x0001u + +#define VF2PF_HANDSHAKE_REQUEST_MSG_LEN 2u +#define VF2PF_HANDSHAKE_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define VF2PF_HANDSHAKE_REQUEST_MSG_1_MAJOR (0xffffu << 16) +#define VF2PF_HANDSHAKE_MAJOR_ANY 0 +#define VF2PF_HANDSHAKE_REQUEST_MSG_1_MINOR (0xffffu << 0) +#define VF2PF_HANDSHAKE_MINOR_ANY 0 + +#define VF2PF_HANDSHAKE_RESPONSE_MSG_LEN 2u +#define VF2PF_HANDSHAKE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 +#define VF2PF_HANDSHAKE_RESPONSE_MSG_1_MAJOR (0xffffu << 16) +#define VF2PF_HANDSHAKE_RESPONSE_MSG_1_MINOR (0xffffu << 0) + +/** + * DOC: VF2PF_QUERY_RUNTIME + * + * This `Relay Message`_ is used by the VF to query values of runtime registers. + * + * On some platforms, VF drivers may not have access to the some fuse registers + * (referred here as 'runtime registers') and therefore VF drivers need to ask + * the PF driver to obtain their values. + * + * However, the list of such registers, and their values, is fully owned and + * maintained by the PF driver and the VF driver may only initiate the query + * sequence and indicate in the @START field the starting index of the next + * requested register from this predefined list. + * + * In the response, the PF driver will return tuple of 32-bit register offset and + * the 32-bit value of that register (respectively @REG_OFFSET and @REG_VALUE). + * + * The VF driver can use @LIMIT field to limit number of returned register tuples. + * If @LIMIT is unset then PF decides about number of returned register tuples. + * + * This message definition is supported from ABI version 1.0. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = **LIMIT** - limit number of returned entries | + * | | | (use zero to not enforce any limits on the response) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_RELAY_ACTION_VF2PF_QUERY_RUNTIME` = 0x0101 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | DATA1 = **START** - index of the first requested entry | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = **COUNT** - number of entries included in response | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | DATA1 = **REMAINING** - number of remaining entries | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | DATA2 = **REG_OFFSET** - offset of register[START] | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | DATA3 = **REG_VALUE** - value of register[START] | + * +---+-------+--------------------------------------------------------------+ + * | | | | + * +---+-------+--------------------------------------------------------------+ + * |n-1| 31:0 | REG_OFFSET - offset of register[START + x] | + * +---+-------+--------------------------------------------------------------+ + * | n | 31:0 | REG_VALUE - value of register[START + x] | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_RELAY_ACTION_VF2PF_QUERY_RUNTIME 0x0101u + +#define VF2PF_QUERY_RUNTIME_REQUEST_MSG_LEN 2u +#define VF2PF_QUERY_RUNTIME_REQUEST_MSG_0_LIMIT GUC_HXG_REQUEST_MSG_0_DATA0 +#define VF2PF_QUERY_RUNTIME_NO_LIMIT 0u +#define VF2PF_QUERY_RUNTIME_REQUEST_MSG_1_START GUC_HXG_REQUEST_MSG_n_DATAn + +#define VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN (GUC_HXG_MSG_MIN_LEN + 1u) +#define VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MAX_LEN \ + (VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN + VF2PF_QUERY_RUNTIME_MAX_COUNT * 2) +#define VF2PF_QUERY_RUNTIME_RESPONSE_MSG_0_COUNT GUC_HXG_RESPONSE_MSG_0_DATA0 +#define VF2PF_QUERY_RUNTIME_MIN_COUNT 0 +#define VF2PF_QUERY_RUNTIME_MAX_COUNT \ + ((GUC_RELAY_MSG_MAX_LEN - VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN) / 2) +#define VF2PF_QUERY_RUNTIME_RESPONSE_MSG_1_REMAINING GUC_HXG_RESPONSE_MSG_n_DATAn +#define VF2PF_QUERY_RUNTIME_RESPONSE_DATAn_REG_OFFSETx GUC_HXG_RESPONSE_MSG_n_DATAn +#define VF2PF_QUERY_RUNTIME_RESPONSE_DATAn_REG_VALUEx GUC_HXG_RESPONSE_MSG_n_DATAn + +/** + * DOC: GuC Relay Debug Actions + * + * This range of action codes is reserved for debugging purposes only and should + * be used only on debug builds. These actions may not be supported by the + * production drivers. Their definitions could be changed in the future. + * + * _`GUC_RELAY_ACTION_DEBUG_ONLY_START` = 0xDEB0 + * _`GUC_RELAY_ACTION_DEBUG_ONLY_END` = 0xDEFF + */ + +#define GUC_RELAY_ACTION_DEBUG_ONLY_START 0xDEB0 +#define GUC_RELAY_ACTION_DEBUG_ONLY_END 0xDEFF + +/** + * DOC: VFXPF_TESTLOOP + * + * This `Relay Message`_ is used to selftest the `GuC Relay Communication`_. + * + * The following opcodes are defined: + * VFXPF_TESTLOOP_OPCODE_NOP_ will return no data. + * VFXPF_TESTLOOP_OPCODE_BUSY_ will reply with BUSY response first. + * VFXPF_TESTLOOP_OPCODE_RETRY_ will reply with RETRY response instead. + * VFXPF_TESTLOOP_OPCODE_ECHO_ will return same data as received. + * VFXPF_TESTLOOP_OPCODE_FAIL_ will always fail with error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ or GUC_HXG_TYPE_FAST_REQUEST_ | + * | | | or GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **OPCODE** | + * | | | - _`VFXPF_TESTLOOP_OPCODE_NOP` = 0x0 | + * | | | - _`VFXPF_TESTLOOP_OPCODE_BUSY` = 0xB | + * | | | - _`VFXPF_TESTLOOP_OPCODE_RETRY` = 0xD | + * | | | - _`VFXPF_TESTLOOP_OPCODE_ECHO` = 0xE | + * | | | - _`VFXPF_TESTLOOP_OPCODE_FAIL` = 0xF | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`IOV_ACTION_SELFTEST_RELAY` | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **DATA1** = optional, depends on **OPCODE**: | + * | | | for VFXPF_TESTLOOP_OPCODE_BUSY_: time in ms for reply | + * | | | for VFXPF_TESTLOOP_OPCODE_FAIL_: expected error | + * | | | for VFXPF_TESTLOOP_OPCODE_ECHO_: payload | + * +---+-------+--------------------------------------------------------------+ + * |...| 31:0 | **DATAn** = only for **OPCODE** VFXPF_TESTLOOP_OPCODE_ECHO_ | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + * |...| 31:0 | DATAn = only for **OPCODE** VFXPF_TESTLOOP_OPCODE_ECHO_ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_RELAY_ACTION_VFXPF_TESTLOOP (GUC_RELAY_ACTION_DEBUG_ONLY_START + 1) +#define VFXPF_TESTLOOP_OPCODE_NOP 0x0 +#define VFXPF_TESTLOOP_OPCODE_BUSY 0xB +#define VFXPF_TESTLOOP_OPCODE_RETRY 0xD +#define VFXPF_TESTLOOP_OPCODE_ECHO 0xE +#define VFXPF_TESTLOOP_OPCODE_FAIL 0xF + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_relay_communication_abi.h b/drivers/gpu/drm/xe/abi/guc_relay_communication_abi.h new file mode 100644 index 000000000..f92625f04 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_relay_communication_abi.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _ABI_GUC_RELAY_COMMUNICATION_ABI_H +#define _ABI_GUC_RELAY_COMMUNICATION_ABI_H + +#include + +#include "guc_actions_sriov_abi.h" +#include "guc_communication_ctb_abi.h" +#include "guc_messages_abi.h" + +/** + * DOC: GuC Relay Communication + * + * The communication between Virtual Function (VF) drivers and Physical Function + * (PF) drivers is based on the GuC firmware acting as a proxy (relay) agent. + * + * To communicate with the PF driver, VF's drivers use `VF2GUC_RELAY_TO_PF`_ + * action that takes the `Relay Message`_ as opaque payload and requires the + * relay message identifier (RID) as additional parameter. + * + * This identifier is used by the drivers to match related messages. + * + * The GuC forwards this `Relay Message`_ and its identifier to the PF driver + * in `GUC2PF_RELAY_FROM_VF`_ action. This event message additionally contains + * the identifier of the origin VF (VFID). + * + * Likewise, to communicate with the VF drivers, PF driver use + * `VF2GUC_RELAY_TO_PF`_ action that in addition to the `Relay Message`_ + * and the relay message identifier (RID) also takes the target VF identifier. + * + * The GuC uses this target VFID from the message to select where to send the + * `GUC2VF_RELAY_FROM_PF`_ with the embedded `Relay Message`_ with response:: + * + * VF GuC PF + * | | | + * [ ] VF2GUC_RELAY_TO_PF | | + * [ ]---------------------------> [ ] | + * [ ] { rid, msg } [ ] | + * [ ] [ ] GUC2PF_RELAY_FROM_VF | + * [ ] [ ]---------------------------> [ ] + * [ ] | { VFID, rid, msg } [ ] + * [ ] | [ ] + * [ ] | PF2GUC_RELAY_TO_VF [ ] + * [ ] [ ] <---------------------------[ ] + * [ ] [ ] { VFID, rid, reply } | + * [ ] GUC2VF_RELAY_FROM_PF [ ] | + * [ ] <---------------------------[ ] | + * | { rid, reply } | | + * | | | + * + * It is also possible that PF driver will initiate communication with the + * selected VF driver. The same GuC action messages will be used:: + * + * VF GuC PF + * | | | + * | | PF2GUC_RELAY_TO_VF [ ] + * | [ ] <---------------------------[ ] + * | [ ] { VFID, rid, msg } [ ] + * | GUC2VF_RELAY_FROM_PF [ ] [ ] + * [ ] <---------------------------[ ] [ ] + * [ ] { rid, msg } | [ ] + * [ ] | [ ] + * [ ] VF2GUC_RELAY_TO_PF | [ ] + * [ ]---------------------------> [ ] [ ] + * | { rid, reply } [ ] [ ] + * | [ ] GUC2PF_RELAY_FROM_VF [ ] + * | [ ]---------------------------> [ ] + * | | { VFID, rid, reply } | + * | | | + */ + +/** + * DOC: Relay Message + * + * The `Relay Message`_ is used by Physical Function (PF) driver and Virtual + * Function (VF) drivers to communicate using `GuC Relay Communication`_. + * + * Format of the `Relay Message`_ follows format of the generic `HXG Message`_. + * + * +--------------------------------------------------------------------------+ + * | `Relay Message`_ | + * +==========================================================================+ + * | `HXG Message`_ | + * +--------------------------------------------------------------------------+ + * + * Maximum length of the `Relay Message`_ is limited by the maximum length of + * the `CTB HXG Message`_ and format of the `GUC2PF_RELAY_FROM_VF`_ message. + */ + +#define GUC_RELAY_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_RELAY_MSG_MAX_LEN \ + (GUC_CTB_MAX_DWORDS - GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN) + +static_assert(PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN > + VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN); + +/** + * DOC: Relay Error Codes + * + * The `GuC Relay Communication`_ can be used to pass `Relay Message`_ between + * drivers that run on different Operating Systems. To help in troubleshooting, + * `GuC Relay Communication`_ uses error codes that mostly match errno values. + */ + +#define GUC_RELAY_ERROR_UNDISCLOSED 0 +#define GUC_RELAY_ERROR_OPERATION_NOT_PERMITTED 1 /* EPERM */ +#define GUC_RELAY_ERROR_PERMISSION_DENIED 13 /* EACCES */ +#define GUC_RELAY_ERROR_INVALID_ARGUMENT 22 /* EINVAL */ +#define GUC_RELAY_ERROR_INVALID_REQUEST_CODE 56 /* EBADRQC */ +#define GUC_RELAY_ERROR_NO_DATA_AVAILABLE 61 /* ENODATA */ +#define GUC_RELAY_ERROR_PROTOCOL_ERROR 71 /* EPROTO */ +#define GUC_RELAY_ERROR_MESSAGE_SIZE 90 /* EMSGSIZE */ + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h b/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h new file mode 100644 index 000000000..85bba3427 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _ABI_GUC_SCHEDULER_ABI_H +#define _ABI_GUC_SCHEDULER_ABI_H + +#include + +/* + * Generic defines required for registration with and submissions to the GuC + * scheduler. Includes engine class/instance defines and context attributes + * (id, priority, etc) + */ + +/* Engine classes/instances */ +#define GUC_RENDER_CLASS 0 +#define GUC_VIDEO_CLASS 1 +#define GUC_VIDEOENHANCE_CLASS 2 +#define GUC_BLITTER_CLASS 3 +#define GUC_COMPUTE_CLASS 4 +#define GUC_GSC_OTHER_CLASS 5 +#define GUC_PAGING_CLASS 6 +#define GUC_LAST_ENGINE_CLASS GUC_PAGING_CLASS +#define GUC_MAX_ENGINE_CLASSES 16 +#define GUC_MAX_INSTANCES_PER_CLASS 32 + +/* context priority values */ +#define GUC_CLIENT_PRIORITY_KMD_HIGH 0 +#define GUC_CLIENT_PRIORITY_HIGH 1 +#define GUC_CLIENT_PRIORITY_KMD_NORMAL 2 +#define GUC_CLIENT_PRIORITY_NORMAL 3 +#define GUC_CLIENT_PRIORITY_NUM 4 + +/* Context registration */ +#define GUC_ID_MAX 65535 +#define GUC_ID_UNKNOWN 0xffffffff + +#define CONTEXT_REGISTRATION_FLAG_KMD BIT(0) +#define CONTEXT_REGISTRATION_FLAG_TYPE GENMASK(2, 1) +#define GUC_CONTEXT_NORMAL 0 +#define GUC_CONTEXT_COMPRESSION_SAVE 1 +#define GUC_CONTEXT_COMPRESSION_RESTORE 2 +#define GUC_CONTEXT_COUNT (GUC_CONTEXT_COMPRESSION_RESTORE + 1) + +/* context enable/disable */ +#define GUC_CONTEXT_DISABLE 0 +#define GUC_CONTEXT_ENABLE 1 + +/* scheduler groups */ +#define GUC_MAX_SCHED_GROUPS 8 + +struct guc_sched_group { + u32 engines[GUC_MAX_ENGINE_CLASSES]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/uc_fw_abi.h b/drivers/gpu/drm/xe/abi/uc_fw_abi.h new file mode 100644 index 000000000..198e94966 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/uc_fw_abi.h @@ -0,0 +1,405 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _ABI_UC_FW_ABI_H +#define _ABI_UC_FW_ABI_H + +#include +#include + +/** + * DOC: CSS-based Firmware Layout + * + * The CSS-based firmware structure is used for GuC releases on all platforms + * and for HuC releases up to DG1. Starting from DG2/MTL the HuC uses the GSC + * layout instead. + * The CSS firmware layout looks like this:: + * + * +======================================================================+ + * | Firmware blob | + * +===============+===============+============+============+============+ + * | CSS header | uCode | RSA key | modulus | exponent | + * +===============+===============+============+============+============+ + * <-header size-> <---header size continued -----------> + * <--- size -----------------------------------------------------------> + * <-key size-> + * <-mod size-> + * <-exp size-> + * + * The firmware may or may not have modulus key and exponent data. The header, + * uCode and RSA signature are must-have components that will be used by driver. + * Length of each components, which is all in dwords, can be found in header. + * In the case that modulus and exponent are not present in fw, a.k.a truncated + * image, the length value still appears in header. + * + * Driver will do some basic fw size validation based on the following rules: + * + * 1. Header, uCode and RSA are must-have components. + * 2. All firmware components, if they present, are in the sequence illustrated + * in the layout table above. + * 3. Length info of each component can be found in header, in dwords. + * 4. Modulus and exponent key are not required by driver. They may not appear + * in fw. So driver will load a truncated firmware in this case. + */ + +struct uc_css_rsa_info { + u32 key_size_dw; + u32 modulus_size_dw; + u32 exponent_size_dw; +} __packed; + +struct uc_css_guc_info { + u32 time; +#define CSS_TIME_HOUR (0xFF << 0) +#define CSS_TIME_MIN (0xFF << 8) +#define CSS_TIME_SEC (0xFFFF << 16) + u32 reserved0[5]; + u32 sw_version; +#define CSS_SW_VERSION_UC_MAJOR (0xFF << 16) +#define CSS_SW_VERSION_UC_MINOR (0xFF << 8) +#define CSS_SW_VERSION_UC_PATCH (0xFF << 0) + u32 submission_version; + u32 reserved1[11]; + u32 header_info; +#define CSS_HEADER_INFO_SVN (0xFF) +#define CSS_HEADER_INFO_COPY_VALID (0x1 << 31) + u32 private_data_size; + u32 ukernel_info; +#define CSS_UKERNEL_INFO_DEVICEID (0xFFFF << 16) +#define CSS_UKERNEL_INFO_PRODKEY (0xFF << 8) +#define CSS_UKERNEL_INFO_BUILDTYPE (0x3 << 2) +#define CSS_UKERNEL_INFO_BUILDTYPE_PROD 0 +#define CSS_UKERNEL_INFO_BUILDTYPE_PREPROD 1 +#define CSS_UKERNEL_INFO_BUILDTYPE_DEBUG 2 +#define CSS_UKERNEL_INFO_ENCSTATUS (0x1 << 1) +#define CSS_UKERNEL_INFO_COPY_VALID (0x1 << 0) +} __packed; + +struct uc_css_header { + u32 module_type; + /* + * header_size includes all non-uCode bits, including css_header, rsa + * key, modulus key and exponent data. + */ + u32 header_size_dw; + u32 header_version; + u32 reserved0; + u32 module_vendor; + u32 date; +#define CSS_DATE_DAY (0xFF << 0) +#define CSS_DATE_MONTH (0xFF << 8) +#define CSS_DATE_YEAR (0xFFFF << 16) + u32 size_dw; /* uCode plus header_size_dw */ + union { + u32 reserved1[3]; + struct uc_css_rsa_info rsa_info; + }; + union { + u32 reserved2[22]; + struct uc_css_guc_info guc_info; + }; +} __packed; +static_assert(sizeof(struct uc_css_header) == 128); + +/** + * DOC: GSC-based Firmware Layout + * + * The GSC-based firmware structure is used for GSC releases on all platforms + * and for HuC releases starting from DG2/MTL. Older HuC releases use the + * CSS-based layout instead. Differently from the CSS headers, the GSC headers + * uses a directory + entries structure (i.e., there is array of addresses + * pointing to specific header extensions identified by a name). Although the + * header structures are the same, some of the entries are specific to GSC while + * others are specific to HuC. The manifest header entry, which includes basic + * information about the binary (like the version) is always present, but it is + * named differently based on the binary type. + * + * The HuC binary starts with a Code Partition Directory (CPD) header. The + * entries we're interested in for use in the driver are: + * + * 1. "HUCP.man": points to the manifest header for the HuC. + * 2. "huc_fw": points to the FW code. On platforms that support load via DMA + * and 2-step HuC authentication (i.e. MTL+) this is a full CSS-based binary, + * while if the GSC is the one doing the load (which only happens on DG2) + * this section only contains the uCode. + * + * The GSC-based HuC firmware layout looks like this:: + * + * +================================================+ + * | CPD Header | + * +================================================+ + * | CPD entries[] | + * | entry1 | + * | ... | + * | entryX | + * | "HUCP.man" | + * | ... | + * | offset >----------------------------|------o + * | ... | | + * | entryY | | + * | "huc_fw" | | + * | ... | | + * | offset >----------------------------|----------o + * +================================================+ | | + * | | + * +================================================+ | | + * | Manifest Header |<-----o | + * | ... | | + * | FW version | | + * | ... | | + * +================================================+ | + * | + * +================================================+ | + * | FW binary |<---------o + * | CSS (MTL+ only) | + * | uCode | + * | RSA Key (MTL+ only) | + * | ... | + * +================================================+ + * + * The GSC binary starts instead with a layout header, which contains the + * locations of the various partitions of the binary. The one we're interested + * in is the boot1 partition, where we can find a BPDT header followed by + * entries, one of which points to the RBE sub-section of the partition, which + * contains the CPD. The GSC blob does not contain a CSS-based binary, so we + * only need to look for the manifest, which is under the "RBEP.man" CPD entry. + * Note that we have no need to find where the actual FW code is inside the + * image because the GSC ROM will itself parse the headers to find it and load + * it. + * The GSC firmware header layout looks like this:: + * + * +================================================+ + * | Layout Pointers | + * | ... | + * | Boot1 offset >---------------------------|------o + * | ... | | + * +================================================+ | + * | + * +================================================+ | + * | BPDT header |<-----o + * +================================================+ + * | BPDT entries[] | + * | entry1 | + * | ... | + * | entryX | + * | type == GSC_RBE | + * | offset >-----------------------------|------o + * | ... | | + * +================================================+ | + * | + * +================================================+ | + * | CPD Header |<-----o + * +================================================+ + * | CPD entries[] | + * | entry1 | + * | ... | + * | entryX | + * | "RBEP.man" | + * | ... | + * | offset >----------------------------|------o + * | ... | | + * +================================================+ | + * | + * +================================================+ | + * | Manifest Header |<-----o + * | ... | + * | FW version | + * | ... | + * | Security version | + * | ... | + * +================================================+ + */ + +struct gsc_version { + u16 major; + u16 minor; + u16 hotfix; + u16 build; +} __packed; + +struct gsc_partition { + u32 offset; + u32 size; +} __packed; + +struct gsc_layout_pointers { + u8 rom_bypass_vector[16]; + + /* size of this header section, not including ROM bypass vector */ + u16 size; + + /* + * bit0: Backup copy of layout pointers exists + * bits1-15: reserved + */ + u8 flags; + + u8 reserved; + + u32 crc32; + + struct gsc_partition datap; + struct gsc_partition boot1; + struct gsc_partition boot2; + struct gsc_partition boot3; + struct gsc_partition boot4; + struct gsc_partition boot5; + struct gsc_partition temp_pages; +} __packed; + +/* Boot partition structures */ +struct gsc_bpdt_header { + u32 signature; +#define GSC_BPDT_HEADER_SIGNATURE 0x000055AA + + u16 descriptor_count; /* num of entries after the header */ + + u8 version; + u8 configuration; + + u32 crc32; + + u32 build_version; + struct gsc_version tool_version; +} __packed; + +struct gsc_bpdt_entry { + /* + * Bits 0-15: BPDT entry type + * Bits 16-17: reserved + * Bit 18: code sub-partition + * Bits 19-31: reserved + */ + u32 type; +#define GSC_BPDT_ENTRY_TYPE_MASK GENMASK(15, 0) +#define GSC_BPDT_ENTRY_TYPE_GSC_RBE 0x1 + + u32 sub_partition_offset; /* from the base of the BPDT header */ + u32 sub_partition_size; +} __packed; + +/* Code partition directory (CPD) structures */ +struct gsc_cpd_header_v2 { + u32 header_marker; +#define GSC_CPD_HEADER_MARKER 0x44504324 + + u32 num_of_entries; + u8 header_version; + u8 entry_version; + u8 header_length; /* in bytes */ + u8 flags; + u32 partition_name; + u32 crc32; +} __packed; + +struct gsc_cpd_entry { + u8 name[12]; + + /* + * Bits 0-24: offset from the beginning of the code partition + * Bit 25: huffman compressed + * Bits 26-31: reserved + */ + u32 offset; +#define GSC_CPD_ENTRY_OFFSET_MASK GENMASK(24, 0) +#define GSC_CPD_ENTRY_HUFFMAN_COMP BIT(25) + + /* + * Module/Item length, in bytes. For Huffman-compressed modules, this + * refers to the uncompressed size. For software-compressed modules, + * this refers to the compressed size. + */ + u32 length; + + u8 reserved[4]; +} __packed; + +struct gsc_manifest_header { + u32 header_type; /* 0x4 for manifest type */ + u32 header_length; /* in dwords */ + u32 header_version; + u32 flags; + u32 vendor; + u32 date; + u32 size; /* In dwords, size of entire manifest (header + extensions) */ + u32 header_id; + u32 internal_data; + struct gsc_version fw_version; + u32 security_version; + struct gsc_version meu_kit_version; + u32 meu_manifest_version; + u8 general_data[4]; + u8 reserved3[56]; + u32 modulus_size; /* in dwords */ + u32 exponent_size; /* in dwords */ +} __packed; + +/** + * DOC: Late binding Firmware Layout + * + * The Late binding binary starts with FPT header, which contains locations + * of various partitions of the binary. Here we're interested in finding out + * manifest version. To the manifest version, we need to locate CPD header + * one of the entry in CPD header points to manifest header. Manifest header + * contains the version. + * + * +================================================+ + * | FPT Header | + * +================================================+ + * | FPT entries[] | + * | entry1 | + * | ... | + * | entryX | + * | "LTES" | + * | ... | + * | offset >-----------------------------|------o + * +================================================+ | + * | + * +================================================+ | + * | CPD Header |<-----o + * +================================================+ + * | CPD entries[] | + * | entry1 | + * | ... | + * | entryX | + * | "LTES.man" | + * | ... | + * | offset >----------------------------|------o + * +================================================+ | + * | + * +================================================+ | + * | Manifest Header |<-----o + * | ... | + * | FW version | + * | ... | + * +================================================+ + */ + +/* FPT Headers */ +struct csc_fpt_header { + u32 header_marker; +#define CSC_FPT_HEADER_MARKER 0x54504624 + u32 num_of_entries; + u8 header_version; + u8 entry_version; + u8 header_length; /* in bytes */ + u8 flags; + u16 ticks_to_add; + u16 tokens_to_add; + u32 uma_size; + u32 crc32; + struct gsc_version fitc_version; +} __packed; + +struct csc_fpt_entry { + u8 name[4]; /* partition name */ + u32 reserved1; + u32 offset; /* offset from beginning of CSE region */ + u32 length; /* partition length in bytes */ + u32 reserved2[3]; + u32 partition_flags; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h b/drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h new file mode 100644 index 000000000..3b557e568 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _ABI_XE_DRIVER_KLVS_ABI_H +#define _ABI_XE_DRIVER_KLVS_ABI_H + +#include "abi/guc_klvs_abi.h" + +/** + * DOC: Xe Driver KLVs + * + * The Xe driver uses the following keys from the `GuC Reserved KLVs`_ range: + * + * _`MIGRATION_KLV_DEVICE_DEVID_KEY` : + * PCI device ID of the migrated VF. + * _`MIGRATION_KLV_DEVICE_REVID_KEY` : + * PCI device revision ID of the migrated VF. + */ + +#define MIGRATION_KLV_DEVICE_DEVID_KEY 0xf001u +#define MIGRATION_KLV_DEVICE_DEVID_LEN 1u +#define MIGRATION_KLV_DEVICE_REVID_KEY 0xf002u +#define MIGRATION_KLV_DEVICE_REVID_LEN 1u + +#endif diff --git a/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h new file mode 100644 index 000000000..4cbde267a --- /dev/null +++ b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_ABI_H_ +#define _XE_SYSCTRL_ABI_H_ + +#include + +/** + * DOC: System Controller ABI + * + * This header defines the Application Binary Interface (ABI) used by + * drm/xe to communicate with System Controller firmware on Intel Xe3p + * discrete GPU platforms. + * + * System Controller (sysctrl) is a firmware-managed entity on Intel + * dGPUs responsible for certain low-level platform management + * functions. + * + * Communication protocol: + * + * Communication uses a mailbox interface with messages composed of: + * + * - Application message header (struct xe_sysctrl_app_msg_hdr) + * containing group_id, command, and version + * - Variable-length, command-specific payload + * + * Message header format: + * + * The 32-bit application message header is packed as: + * + * - Bits [7:0] : Group ID identifying command group + * - Bits [15:8] : Command identifier within group + * - Bits [23:16] : Command version for interface compatibility + * - Bits [31:24] : Reserved, must be zero + * + * This header defines firmware ABI message formats and constants shared + * between driver and System Controller firmware. + */ + +/** + * struct xe_sysctrl_app_msg_hdr - Application layer message header + * @data: 32-bit header data + * + * Header structure for application-level messages. + */ +struct xe_sysctrl_app_msg_hdr { + u32 data; +} __packed; + +#define SYSCTRL_HDR_GROUP_ID_MASK GENMASK(7, 0) +#define SYSCTRL_HDR_COMMAND_MASK GENMASK(14, 8) +#define SYSCTRL_HDR_COMMAND_MAX 0x7f +#define SYSCTRL_HDR_IS_RESPONSE BIT(15) +#define SYSCTRL_HDR_RESERVED_MASK GENMASK(23, 16) +#define SYSCTRL_HDR_RESULT_MASK GENMASK(31, 24) + +#define APP_HDR_GROUP_ID_MASK GENMASK(7, 0) +#define APP_HDR_COMMAND_MASK GENMASK(15, 8) +#define APP_HDR_VERSION_MASK GENMASK(23, 16) +#define APP_HDR_RESERVED_MASK GENMASK(31, 24) + +#endif diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_config.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_config.h new file mode 100644 index 000000000..d4522203e --- /dev/null +++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_config.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef __I915_CONFIG_H__ +#define __I915_CONFIG_H__ + +#include + +static inline unsigned long i915_fence_timeout(void) +{ + return MAX_SCHEDULE_TIMEOUT; +} + +#endif /* __I915_CONFIG_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_gtt_view_types.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_gtt_view_types.h new file mode 100644 index 000000000..b261910cd --- /dev/null +++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_gtt_view_types.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2025 Intel Corporation */ + +#include "../../i915/i915_gtt_view_types.h" + +/* Partial view not supported in xe, fail build if used. */ +#define I915_GTT_VIEW_PARTIAL diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_reg_defs.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_reg_defs.h new file mode 100644 index 000000000..723279c97 --- /dev/null +++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_reg_defs.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#include "../../i915/i915_reg_defs.h" diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h new file mode 100644 index 000000000..552975a30 --- /dev/null +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef __INTEL_CLOCK_GATING_H__ +#define __INTEL_CLOCK_GATING_H__ + +struct drm_device; + +static inline void intel_clock_gating_init(struct drm_device *drm) {} +static inline void intel_clock_gating_hooks_init(struct drm_device *drm) {} + +#endif /* __INTEL_CLOCK_GATING_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h new file mode 100644 index 000000000..08d7ab933 --- /dev/null +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef __INTEL_UNCORE_H__ +#define __INTEL_UNCORE_H__ + +#include "i915_reg_defs.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_mmio.h" + +static inline struct intel_uncore *to_intel_uncore(struct drm_device *drm) +{ + return &to_xe_device(drm)->uncore; +} + +static inline struct xe_mmio *__compat_uncore_to_mmio(struct intel_uncore *uncore) +{ + struct xe_device *xe = container_of(uncore, struct xe_device, uncore); + + return xe_root_tile_mmio(xe); +} + +static inline u32 intel_uncore_read(struct intel_uncore *uncore, + i915_reg_t i915_reg) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + return xe_mmio_read32(__compat_uncore_to_mmio(uncore), reg); +} + +static inline u8 intel_uncore_read8(struct intel_uncore *uncore, + i915_reg_t i915_reg) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + return xe_mmio_read8(__compat_uncore_to_mmio(uncore), reg); +} + +static inline void intel_uncore_write8(struct intel_uncore *uncore, + i915_reg_t i915_reg, u8 val) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + xe_mmio_write8(__compat_uncore_to_mmio(uncore), reg, val); +} + +static inline u16 intel_uncore_read16(struct intel_uncore *uncore, + i915_reg_t i915_reg) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + return xe_mmio_read16(__compat_uncore_to_mmio(uncore), reg); +} + +static inline u64 +intel_uncore_read64_2x32(struct intel_uncore *uncore, + i915_reg_t i915_lower_reg, i915_reg_t i915_upper_reg) +{ + struct xe_reg lower_reg = XE_REG(i915_mmio_reg_offset(i915_lower_reg)); + struct xe_reg upper_reg = XE_REG(i915_mmio_reg_offset(i915_upper_reg)); + u32 upper, lower, old_upper; + int loop = 0; + + upper = xe_mmio_read32(__compat_uncore_to_mmio(uncore), upper_reg); + do { + old_upper = upper; + lower = xe_mmio_read32(__compat_uncore_to_mmio(uncore), lower_reg); + upper = xe_mmio_read32(__compat_uncore_to_mmio(uncore), upper_reg); + } while (upper != old_upper && loop++ < 2); + + return (u64)upper << 32 | lower; +} + +static inline void intel_uncore_posting_read(struct intel_uncore *uncore, + i915_reg_t i915_reg) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + xe_mmio_read32(__compat_uncore_to_mmio(uncore), reg); +} + +static inline void intel_uncore_write(struct intel_uncore *uncore, + i915_reg_t i915_reg, u32 val) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + xe_mmio_write32(__compat_uncore_to_mmio(uncore), reg, val); +} + +static inline u32 intel_uncore_rmw(struct intel_uncore *uncore, + i915_reg_t i915_reg, u32 clear, u32 set) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + return xe_mmio_rmw32(__compat_uncore_to_mmio(uncore), reg, clear, set); +} + +static inline u32 intel_uncore_read_fw(struct intel_uncore *uncore, + i915_reg_t i915_reg) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + return xe_mmio_read32(__compat_uncore_to_mmio(uncore), reg); +} + +static inline void intel_uncore_write_fw(struct intel_uncore *uncore, + i915_reg_t i915_reg, u32 val) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + xe_mmio_write32(__compat_uncore_to_mmio(uncore), reg, val); +} + +static inline u32 intel_uncore_read_notrace(struct intel_uncore *uncore, + i915_reg_t i915_reg) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + return xe_mmio_read32(__compat_uncore_to_mmio(uncore), reg); +} + +static inline void intel_uncore_write_notrace(struct intel_uncore *uncore, + i915_reg_t i915_reg, u32 val) +{ + struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg)); + + xe_mmio_write32(__compat_uncore_to_mmio(uncore), reg, val); +} + +static inline bool +intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore) +{ + return false; +} + +#endif /* __INTEL_UNCORE_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore_trace.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore_trace.h new file mode 100644 index 000000000..d429d421a --- /dev/null +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore_trace.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#define trace_i915_reg_rw(a...) do { } while (0) diff --git a/drivers/gpu/drm/xe/display/tests/Makefile b/drivers/gpu/drm/xe/display/tests/Makefile new file mode 100644 index 000000000..c71686458 --- /dev/null +++ b/drivers/gpu/drm/xe/display/tests/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 + +subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \ + -I$(srctree)/drivers/gpu/drm/i915/display/ + +# Rule to build display code shared with i915 +$(obj)/i915-display/tests/%.o: $(srctree)/drivers/gpu/drm/i915/display/tests/%.c FORCE + $(call cmd,force_checksrc) + $(call if_changed_rule,cc_o_c) + +obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_display_test.o +xe_display_test-y = \ + i915-display/tests/intel_dp_link_test.o diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c new file mode 100644 index 000000000..8da4c457a --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_display.h" +#include "regs/xe_irq_regs.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "intel_acpi.h" +#include "intel_audio.h" +#include "intel_display.h" +#include "intel_display_core.h" +#include "intel_display_device.h" +#include "intel_display_driver.h" +#include "intel_display_irq.h" +#include "intel_display_types.h" +#include "intel_dmc.h" +#include "intel_dmc_wl.h" +#include "intel_dp.h" +#include "intel_fbdev.h" +#include "intel_hdcp.h" +#include "intel_hotplug.h" +#include "intel_opregion.h" +#include "skl_watermark.h" +#include "xe_device.h" +#include "xe_display_bo.h" +#include "xe_display_pcode.h" +#include "xe_display_rpm.h" +#include "xe_dsb_buffer.h" +#include "xe_fb_pin.h" +#include "xe_frontbuffer.h" +#include "xe_hdcp_gsc.h" +#include "xe_initial_plane.h" +#include "xe_module.h" +#include "xe_panic.h" +#include "xe_stolen.h" + +/* Ensure drm and display members are placed properly. */ +INTEL_DISPLAY_MEMBER_STATIC_ASSERT(struct xe_device, drm, display); + +/* Xe device functions */ + +/** + * xe_display_driver_probe_defer - Detect if we need to wait for other drivers + * early on + * @pdev: PCI device + * + * Note: This is called before xe or display device creation. + * + * Returns: true if probe needs to be deferred, false otherwise + */ +bool xe_display_driver_probe_defer(struct pci_dev *pdev) +{ + if (!xe_modparam.probe_display) + return 0; + + return intel_display_driver_probe_defer(pdev); +} + +static void unset_display_features(struct xe_device *xe) +{ + xe->drm.driver_features &= ~XE_DISPLAY_DRIVER_FEATURES; +} + +static void xe_display_fini_early(void *arg) +{ + struct xe_device *xe = arg; + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_driver_remove_noirq(display); + intel_display_driver_remove_nogem(display); + intel_display_power_cleanup(display); +} + +int xe_display_init_early(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + int err; + + if (!xe->info.probe_display) + return 0; + + /* Fake uncore lock */ + spin_lock_init(&xe->uncore.lock); + + intel_display_driver_early_probe(display); + + intel_display_device_info_runtime_init(display); + + /* Display may have been disabled at runtime init */ + if (!intel_display_device_present(display)) { + xe->info.probe_display = false; + unset_display_features(xe); + return 0; + } + + err = intel_display_driver_probe_noirq(display); + if (err) + return err; + + err = intel_display_driver_probe_nogem(display); + if (err) + goto err_noirq; + + return devm_add_action_or_reset(xe->drm.dev, xe_display_fini_early, xe); +err_noirq: + intel_display_driver_remove_noirq(display); + intel_display_power_cleanup(display); + + return err; +} + +static void xe_display_fini(void *arg) +{ + struct xe_device *xe = arg; + struct intel_display *display = xe->display; + + intel_hpd_poll_fini(display); + intel_hdcp_component_fini(display); + intel_audio_deinit(display); + intel_display_driver_remove(display); +} + +int xe_display_init(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + int err; + + if (!xe->info.probe_display) + return 0; + + err = intel_display_driver_probe(display); + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, xe_display_fini, xe); +} + +void xe_display_register(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_driver_register(display); + intel_display_driver_runtime_pm_enable(display); +} + +void xe_display_unregister(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_driver_runtime_pm_disable(display); + intel_display_driver_unregister(display); +} + +void xe_display_shutdown(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_driver_shutdown(display); + + intel_opregion_suspend(display, PCI_D3cold); + + intel_dmc_suspend(display); +} + +void xe_display_shutdown_late(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_driver_shutdown_late(display); +} + +/* IRQ-related functions */ + +void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + if (master_ctl & DISPLAY_IRQ) + intel_display_irq_handler(display, NULL); +} + +void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + if (gu_misc_iir & GU_MISC_GSE) + intel_opregion_asle_intr(display); +} + +void xe_display_irq_reset(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_irq_reset(display); +} + +void xe_display_irq_postinstall(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_irq_postinstall(display); +} + +static bool suspend_to_idle(void) +{ +#if IS_ENABLED(CONFIG_ACPI_SLEEP) + if (acpi_target_system_state() < ACPI_STATE_S3) + return true; +#endif + return false; +} + +void xe_display_pm_suspend(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + bool s2idle = suspend_to_idle(); + + if (!xe->info.probe_display) + return; + + intel_display_driver_pm_suspend(display); + + intel_opregion_suspend(display, s2idle ? PCI_D1 : PCI_D3cold); + + intel_dmc_suspend(display); +} + +void xe_display_pm_suspend_late(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + bool s2idle = suspend_to_idle(); + + if (!xe->info.probe_display) + return; + + intel_display_driver_pm_suspend_late(display, s2idle); +} + +void xe_display_pm_resume_early(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_display_driver_pm_resume_early(display); +} + +void xe_display_pm_resume(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_dmc_resume(display); + + if (intel_display_device_present(display)) + drm_mode_config_reset(&xe->drm); + + intel_display_driver_init_hw(display); + + intel_display_driver_pm_resume(display); +} + +static void xe_display_enable_d3cold(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + /* + * We do a lot of poking in a lot of registers, make sure they work + * properly. + */ + intel_display_driver_runtime_pm_disable(display); + + intel_display_flush_cleanup_work(display); + + intel_opregion_suspend(display, PCI_D3cold); + + intel_dmc_suspend(display); + + if (intel_display_device_present(display)) + intel_hpd_poll_enable(display); +} + +static void xe_display_disable_d3cold(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + intel_dmc_resume(display); + + if (intel_display_device_present(display)) + drm_mode_config_reset(&xe->drm); + + intel_display_driver_init_hw(display); + + intel_hpd_init(display); + + if (intel_display_device_present(display)) + intel_hpd_poll_disable(display); + + intel_opregion_resume(display); + + intel_display_driver_runtime_pm_enable(display); +} + +/* before irq suspend */ +void xe_display_pm_runtime_suspend(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + if (xe->d3cold.allowed) { + xe_display_enable_d3cold(xe); + return; + } + + intel_display_driver_pm_runtime_suspend(display); +} + +/* after irq suspend */ +void xe_display_pm_runtime_suspend_late(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + if (xe->d3cold.allowed) { + xe_display_pm_suspend_late(xe); + /* Ensure the wakelock release work gets flushed */ + intel_dmc_wl_flush_release_work(display); + return; + } + + intel_display_driver_pm_runtime_suspend_late(display); +} + +/* before irq resume */ +void xe_display_pm_runtime_resume_early(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + if (xe->d3cold.allowed) + return; + + intel_display_driver_pm_runtime_resume_early(display); +} + +/* after irq resume */ +void xe_display_pm_runtime_resume(struct xe_device *xe) +{ + struct intel_display *display = xe->display; + + if (!xe->info.probe_display) + return; + + if (xe->d3cold.allowed) { + xe_display_disable_d3cold(xe); + return; + } + + intel_display_driver_pm_runtime_resume(display); +} + + +static void display_device_remove(struct drm_device *dev, void *arg) +{ + struct xe_device *xe = arg; + + intel_display_device_remove(xe->display); + xe->display = NULL; +} + +static bool irq_enabled(struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + + return atomic_read(&xe->irq.enabled); +} + +static void irq_synchronize(struct drm_device *drm) +{ + synchronize_irq(to_pci_dev(drm->dev)->irq); +} + +static const struct intel_display_irq_interface xe_display_irq_interface = { + .enabled = irq_enabled, + .synchronize = irq_synchronize, +}; + +static bool has_auxccs(struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + + return xe->info.platform == XE_ALDERLAKE_P; +} + +static const struct intel_display_parent_interface parent = { + .bo = &xe_display_bo_interface, + .dsb = &xe_display_dsb_interface, + .fb_pin = &xe_display_fb_pin_interface, + .frontbuffer = &xe_display_frontbuffer_interface, + .hdcp = &xe_display_hdcp_interface, + .initial_plane = &xe_display_initial_plane_interface, + .irq = &xe_display_irq_interface, + .panic = &xe_display_panic_interface, + .pcode = &xe_display_pcode_interface, + .rpm = &xe_display_rpm_interface, + .stolen = &xe_display_stolen_interface, + .has_auxccs = has_auxccs, +}; + +/** + * xe_display_probe - probe display and create display struct + * @xe: XE device instance + * + * Initialize all fields used by the display part. + * + * TODO: once everything can be inside a single struct, make the struct opaque + * to the rest of xe and return it to be xe->display. + * + * Returns: 0 on success + */ +int xe_display_probe(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct intel_display *display; + int err; + + if (!xe->info.probe_display) + goto no_display; + + display = intel_display_device_probe(pdev, &parent); + if (IS_ERR(display)) + return PTR_ERR(display); + + xe->display = display; + + err = drmm_add_action_or_reset(&xe->drm, display_device_remove, xe); + if (err) + return err; + + if (intel_display_device_present(display)) + return 0; + +no_display: + xe->info.probe_display = false; + unset_display_features(xe); + return 0; +} + +#ifdef CONFIG_DRM_FBDEV_EMULATION +int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh, + struct drm_fb_helper_surface_size *sizes) +{ + return intel_fbdev_driver_fbdev_probe(fbh, sizes); +} +#endif diff --git a/drivers/gpu/drm/xe/display/xe_display.h b/drivers/gpu/drm/xe/display/xe_display.h new file mode 100644 index 000000000..0babb50bf --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_DISPLAY_H_ +#define _XE_DISPLAY_H_ + +#include + +struct drm_driver; +struct drm_fb_helper; +struct drm_fb_helper_surface_size; +struct pci_dev; +struct xe_device; + +#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) + +bool xe_display_driver_probe_defer(struct pci_dev *pdev); + +int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh, + struct drm_fb_helper_surface_size *sizes); + +int xe_display_probe(struct xe_device *xe); + +int xe_display_init_early(struct xe_device *xe); +int xe_display_init(struct xe_device *xe); + +void xe_display_register(struct xe_device *xe); +void xe_display_unregister(struct xe_device *xe); + +void xe_display_shutdown(struct xe_device *xe); +void xe_display_shutdown_late(struct xe_device *xe); + +void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl); +void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir); +void xe_display_irq_reset(struct xe_device *xe); +void xe_display_irq_postinstall(struct xe_device *xe); + +void xe_display_pm_suspend(struct xe_device *xe); +void xe_display_pm_suspend_late(struct xe_device *xe); +void xe_display_pm_resume_early(struct xe_device *xe); +void xe_display_pm_resume(struct xe_device *xe); +void xe_display_pm_runtime_suspend(struct xe_device *xe); +void xe_display_pm_runtime_suspend_late(struct xe_device *xe); +void xe_display_pm_runtime_resume_early(struct xe_device *xe); +void xe_display_pm_runtime_resume(struct xe_device *xe); + +#define XE_DISPLAY_DRIVER_FEATURES (DRIVER_MODESET | DRIVER_ATOMIC) +#define XE_DISPLAY_DRIVER_OPS \ + .fbdev_probe = PTR_IF(IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION), \ + xe_display_driver_fbdev_probe) + +#else + +#define XE_DISPLAY_DRIVER_FEATURES 0 +#define XE_DISPLAY_DRIVER_OPS \ + .fbdev_probe = NULL + +static inline int xe_display_driver_probe_defer(struct pci_dev *pdev) { return 0; } + +static inline int xe_display_probe(struct xe_device *xe) { return 0; } + +static inline int xe_display_init_early(struct xe_device *xe) { return 0; } +static inline int xe_display_init(struct xe_device *xe) { return 0; } + +static inline void xe_display_register(struct xe_device *xe) {} +static inline void xe_display_unregister(struct xe_device *xe) {} + +static inline void xe_display_shutdown(struct xe_device *xe) {} +static inline void xe_display_shutdown_late(struct xe_device *xe) {} + +static inline void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl) {} +static inline void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir) {} +static inline void xe_display_irq_reset(struct xe_device *xe) {} +static inline void xe_display_irq_postinstall(struct xe_device *xe) {} + +static inline void xe_display_pm_suspend(struct xe_device *xe) {} +static inline void xe_display_pm_suspend_late(struct xe_device *xe) {} +static inline void xe_display_pm_resume_early(struct xe_device *xe) {} +static inline void xe_display_pm_resume(struct xe_device *xe) {} +static inline void xe_display_pm_runtime_suspend(struct xe_device *xe) {} +static inline void xe_display_pm_runtime_suspend_late(struct xe_device *xe) {} +static inline void xe_display_pm_runtime_resume_early(struct xe_device *xe) {} +static inline void xe_display_pm_runtime_resume(struct xe_device *xe) {} + +#endif /* CONFIG_DRM_XE_DISPLAY */ +#endif /* _XE_DISPLAY_H_ */ diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c new file mode 100644 index 000000000..f5102cd0e --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_bo.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: MIT +/* Copyright © 2024 Intel Corporation */ + +#include + +#include +#include + +#include "intel_fb.h" +#include "xe_bo.h" +#include "xe_display_bo.h" +#include "xe_pxp.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_wa.h" + +#include + +static bool xe_display_bo_is_protected(struct drm_gem_object *obj) +{ + return xe_bo_is_protected(gem_to_xe_bo(obj)); +} + +static int xe_display_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + + return xe_bo_read(bo, offset, dst, size); +} + +static int xe_display_bo_framebuffer_init(struct drm_gem_object *obj, + struct drm_mode_fb_cmd2 *mode_cmd) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + struct xe_device *xe = to_xe_device(bo->ttm.base.dev); + int ret; + + /* + * Some modifiers require physical alignment of 64KiB VRAM pages; + * require that the BO in those cases is created correctly. + */ + if (XE_IOCTL_DBG(xe, intel_fb_needs_64k_phys(mode_cmd->modifier[0]) && + !(bo->flags & XE_BO_FLAG_NEEDS_64K))) + return -EINVAL; + + xe_bo_get(bo); + + ret = ttm_bo_reserve(&bo->ttm, true, false, NULL); + if (ret) + goto err; + + if (!(bo->flags & XE_BO_FLAG_FORCE_WC) && + bo->ttm.type != ttm_bo_type_sg) { + /* + * XE_BO_FLAG_FORCE_WC should ideally be set at creation, or is + * automatically set when creating FB. We cannot change caching + * mode when the bo is VM_BINDed, so we can only set + * coherency with display when unbound. + */ + if (XE_IOCTL_DBG(xe, xe_bo_is_vm_bound(bo))) { + ttm_bo_unreserve(&bo->ttm); + ret = -EINVAL; + goto err; + } + bo->flags |= XE_BO_FLAG_FORCE_WC; + } + ttm_bo_unreserve(&bo->ttm); + return 0; + +err: + xe_bo_put(bo); + return ret; +} + +static void xe_display_bo_framebuffer_fini(struct drm_gem_object *obj) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + + if (bo->flags & XE_BO_FLAG_PINNED) { + /* Unpin our kernel fb first */ + xe_bo_lock(bo, false); + xe_bo_unpin(bo); + xe_bo_unlock(bo); + } + xe_bo_put(bo); +} + +static struct drm_gem_object * +xe_display_bo_framebuffer_lookup(struct drm_device *drm, + struct drm_file *filp, + const struct drm_mode_fb_cmd2 *mode_cmd) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_bo *bo; + struct drm_gem_object *gem = drm_gem_object_lookup(filp, mode_cmd->handles[0]); + + if (!gem) + return ERR_PTR(-ENOENT); + + bo = gem_to_xe_bo(gem); + /* Require vram placement or dma-buf import */ + if (IS_DGFX(xe) && + !xe_bo_can_migrate(bo, XE_PL_VRAM0) && + bo->ttm.type != ttm_bo_type_sg) { + drm_gem_object_put(gem); + return ERR_PTR(-EREMOTE); + } + + return gem; +} + +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) +/* + * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride + * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to + * have driver specific values. However, dropping the stride alignment to 64 + * leads to underflowing the bo pin count in the atomic cleanup work. + */ +static u32 xe_display_bo_fbdev_pitch_align(u32 stride) +{ + return ALIGN(stride, XE_PAGE_SIZE); +} + +bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size) +{ + struct ttm_resource_manager *stolen; + + stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); + if (!stolen) + return false; + + if (IS_DGFX(xe)) + return false; + + if (XE_DEVICE_WA(xe, 22019338487_display)) + return false; + + /* + * If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. + */ + return stolen->size >= (size * 2) >> PAGE_SHIFT; +} + +static struct drm_gem_object *xe_display_bo_fbdev_create(struct drm_device *drm, int size) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_bo *obj; + + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, + ttm_bo_type_kernel, + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_FLAG_GGTT, + false); + if (IS_ERR(obj)) { + drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj); + return ERR_PTR(-ENOMEM); + } + + return &obj->ttm.base; +} + +static void xe_display_bo_fbdev_destroy(struct drm_gem_object *obj) +{ + xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj)); +} + +static int xe_display_bo_fbdev_fill_info(struct drm_gem_object *_obj, struct fb_info *info, + struct i915_vma *vma) +{ + struct xe_bo *obj = gem_to_xe_bo(_obj); + struct pci_dev *pdev = to_pci_dev(_obj->dev->dev); + + if (!(obj->flags & XE_BO_FLAG_SYSTEM)) { + if (obj->flags & XE_BO_FLAG_STOLEN) + info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0); + else + info->fix.smem_start = + pci_resource_start(pdev, 2) + + xe_bo_addr(obj, 0, XE_PAGE_SIZE); + + info->fix.smem_len = obj->ttm.base.size; + } else { + /* XXX: Pure fiction, as the BO may not be physically accessible.. */ + info->fix.smem_start = 0; + info->fix.smem_len = obj->ttm.base.size; + } + XE_WARN_ON(iosys_map_is_null(&obj->vmap)); + + info->screen_base = obj->vmap.vaddr_iomem; + info->screen_size = obj->ttm.base.size; + + return 0; +} +#endif + +const struct intel_display_bo_interface xe_display_bo_interface = { + .is_protected = xe_display_bo_is_protected, + .key_check = xe_pxp_obj_key_check, + .fb_mmap = drm_gem_prime_mmap, + .read_from_page = xe_display_bo_read_from_page, + .framebuffer_init = xe_display_bo_framebuffer_init, + .framebuffer_fini = xe_display_bo_framebuffer_fini, + .framebuffer_lookup = xe_display_bo_framebuffer_lookup, +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) + .fbdev_create = xe_display_bo_fbdev_create, + .fbdev_destroy = xe_display_bo_fbdev_destroy, + .fbdev_fill_info = xe_display_bo_fbdev_fill_info, + .fbdev_pitch_align = xe_display_bo_fbdev_pitch_align, +#endif +}; diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.h b/drivers/gpu/drm/xe/display/xe_display_bo.h new file mode 100644 index 000000000..c72056884 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_bo.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef __XE_DISPLAY_BO_H__ +#define __XE_DISPLAY_BO_H__ + +#include + +struct xe_device; + +bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size); + +extern const struct intel_display_bo_interface xe_display_bo_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_display_pcode.c b/drivers/gpu/drm/xe/display/xe_display_pcode.c new file mode 100644 index 000000000..f6820ef7e --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_pcode.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT +/* Copyright © 2026 Intel Corporation */ + +#include + +#include "xe_device.h" +#include "xe_pcode.h" + +static int xe_display_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_tile *tile = xe_device_get_root_tile(xe); + + return xe_pcode_read(tile, mbox, val, val1); +} + +static int xe_display_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_tile *tile = xe_device_get_root_tile(xe); + + return xe_pcode_write_timeout(tile, mbox, val, timeout_ms); +} + +static int xe_display_pcode_request(struct drm_device *drm, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_tile *tile = xe_device_get_root_tile(xe); + + return xe_pcode_request(tile, mbox, request, reply_mask, reply, timeout_base_ms); +} + +const struct intel_display_pcode_interface xe_display_pcode_interface = { + .read = xe_display_pcode_read, + .write = xe_display_pcode_write_timeout, + .request = xe_display_pcode_request, +}; diff --git a/drivers/gpu/drm/xe/display/xe_display_pcode.h b/drivers/gpu/drm/xe/display/xe_display_pcode.h new file mode 100644 index 000000000..58bd2fb7f --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_pcode.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef __XE_DISPLAY_PCODE_H__ +#define __XE_DISPLAY_PCODE_H__ + +extern const struct intel_display_pcode_interface xe_display_pcode_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_display_rpm.c b/drivers/gpu/drm/xe/display/xe_display_rpm.c new file mode 100644 index 000000000..b3db40035 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_rpm.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT +/* Copyright © 2025 Intel Corporation */ + +#include + +#include "intel_display_core.h" +#include "intel_display_rpm.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_pm.h" + +/* -ENOENT means we got the ref, but there's no tracking */ +#define INTEL_WAKEREF_DEF ERR_PTR(-ENOENT) + +static struct ref_tracker *xe_display_rpm_get(const struct drm_device *drm) +{ + return xe_pm_runtime_resume_and_get(to_xe_device(drm)) ? INTEL_WAKEREF_DEF : NULL; +} + +static struct ref_tracker *xe_display_rpm_get_if_in_use(const struct drm_device *drm) +{ + return xe_pm_runtime_get_if_in_use(to_xe_device(drm)) ? INTEL_WAKEREF_DEF : NULL; +} + +static struct ref_tracker *xe_display_rpm_get_noresume(const struct drm_device *drm) +{ + xe_pm_runtime_get_noresume(to_xe_device(drm)); + + return INTEL_WAKEREF_DEF; +} + +static void xe_display_rpm_put(const struct drm_device *drm, struct ref_tracker *wakeref) +{ + if (wakeref) + xe_pm_runtime_put(to_xe_device(drm)); +} + +static void xe_display_rpm_put_unchecked(const struct drm_device *drm) +{ + xe_pm_runtime_put(to_xe_device(drm)); +} + +static bool xe_display_rpm_suspended(const struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + + return pm_runtime_suspended(xe->drm.dev); +} + +static void xe_display_rpm_assert_held(const struct drm_device *drm) +{ + /* FIXME */ +} + +static void xe_display_rpm_assert_block(const struct drm_device *drm) +{ + /* FIXME */ +} + +static void xe_display_rpm_assert_unblock(const struct drm_device *drm) +{ + /* FIXME */ +} + +const struct intel_display_rpm_interface xe_display_rpm_interface = { + .get = xe_display_rpm_get, + .get_raw = xe_display_rpm_get, + .get_if_in_use = xe_display_rpm_get_if_in_use, + .get_noresume = xe_display_rpm_get_noresume, + .put = xe_display_rpm_put, + .put_raw = xe_display_rpm_put, + .put_unchecked = xe_display_rpm_put_unchecked, + .suspended = xe_display_rpm_suspended, + .assert_held = xe_display_rpm_assert_held, + .assert_block = xe_display_rpm_assert_block, + .assert_unblock = xe_display_rpm_assert_unblock +}; diff --git a/drivers/gpu/drm/xe/display/xe_display_rpm.h b/drivers/gpu/drm/xe/display/xe_display_rpm.h new file mode 100644 index 000000000..0bf9d31e8 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_rpm.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_DISPLAY_RPM_H_ +#define _XE_DISPLAY_RPM_H_ + +extern const struct intel_display_rpm_interface xe_display_rpm_interface; + +#endif /* _XE_DISPLAY_RPM_H_ */ diff --git a/drivers/gpu/drm/xe/display/xe_display_vma.h b/drivers/gpu/drm/xe/display/xe_display_vma.h new file mode 100644 index 000000000..28267be61 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_vma.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef _XE_DISPLAY_VMA_H_ +#define _XE_DISPLAY_VMA_H_ + +#include + +struct xe_bo; +struct xe_ggtt_node; + +struct i915_vma { + refcount_t ref; + struct xe_bo *bo, *dpt; + struct xe_ggtt_node *node; +}; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_display_wa.c b/drivers/gpu/drm/xe/display/xe_display_wa.c new file mode 100644 index 000000000..2aa1b8c03 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_display_wa.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include "intel_display_core.h" +#include "intel_display_wa.h" +#include "xe_device.h" +#include "xe_wa.h" + +#include + +bool intel_display_needs_wa_16023588340(struct intel_display *display) +{ + struct xe_device *xe = to_xe_device(display->drm); + struct xe_gt *wa_gt = xe_root_mmio_gt(xe); + + return wa_gt && XE_GT_WA(wa_gt, 16023588340); +} diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c new file mode 100644 index 000000000..a7158c73a --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2023, Intel Corporation. + */ + +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_dsb_buffer.h" + +struct intel_dsb_buffer { + u32 *cmd_buf; + struct xe_bo *bo; + size_t buf_size; +}; + +static u32 xe_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf) +{ + return xe_bo_ggtt_addr(dsb_buf->bo); +} + +static void xe_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val) +{ + iosys_map_wr(&dsb_buf->bo->vmap, idx * 4, u32, val); +} + +static u32 xe_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx) +{ + return iosys_map_rd(&dsb_buf->bo->vmap, idx * 4, u32); +} + +static void xe_dsb_buffer_fill(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size) +{ + WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf)); + + iosys_map_memset(&dsb_buf->bo->vmap, idx * 4, val, size); +} + +static struct intel_dsb_buffer *xe_dsb_buffer_create(struct drm_device *drm, size_t size) +{ + struct xe_device *xe = to_xe_device(drm); + struct intel_dsb_buffer *dsb_buf; + struct xe_bo *obj; + int ret; + + dsb_buf = kzalloc_obj(*dsb_buf); + if (!dsb_buf) + return ERR_PTR(-ENOMEM); + + /* Set scanout flag for WC mapping */ + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), + PAGE_ALIGN(size), + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_GGTT, + false); + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); + goto err_pin_map; + } + + dsb_buf->bo = obj; + dsb_buf->buf_size = size; + + return dsb_buf; + +err_pin_map: + kfree(dsb_buf); + + return ERR_PTR(ret); +} + +static void xe_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf) +{ + xe_bo_unpin_map_no_vm(dsb_buf->bo); + kfree(dsb_buf); +} + +static void xe_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf) +{ + struct xe_device *xe = dsb_buf->bo->tile->xe; + + /* + * The memory barrier here is to ensure coherency of DSB vs MMIO, + * both for weak ordering archs and discrete cards. + */ + xe_device_wmb(xe); + xe_device_l2_flush(xe); +} + +const struct intel_display_dsb_interface xe_display_dsb_interface = { + .ggtt_offset = xe_dsb_buffer_ggtt_offset, + .write = xe_dsb_buffer_write, + .read = xe_dsb_buffer_read, + .fill = xe_dsb_buffer_fill, + .create = xe_dsb_buffer_create, + .cleanup = xe_dsb_buffer_cleanup, + .flush_map = xe_dsb_buffer_flush_map, +}; diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.h b/drivers/gpu/drm/xe/display/xe_dsb_buffer.h new file mode 100644 index 000000000..2e4772187 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef __XE_DSB_BUFFER_H__ +#define __XE_DSB_BUFFER_H__ + +extern const struct intel_display_dsb_interface xe_display_dsb_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c new file mode 100644 index 000000000..73469ea5f --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include +#include + +/* FIXME move the types to parent interface? */ +#include "i915_gtt_view_types.h" + +/* FIXME move intel_remapped_info_size() & co. to parent interface? */ +#include "intel_fb.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_display_vma.h" +#include "xe_fb_pin.h" +#include "xe_ggtt.h" +#include "xe_pat.h" +#include "xe_pm.h" +#include "xe_vram_types.h" + +static void +write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, + u32 width, u32 height, u32 src_stride, u32 dst_stride) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; + u32 column, row; + u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); + + /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, + * by writing dpt/ggtt in a different order? + */ + + for (column = 0; column < width; column++) { + u32 src_idx = src_stride * (height - 1) + column + bo_ofs; + + for (row = 0; row < height; row++) { + u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); + + iosys_map_wr(map, *dpt_ofs, u64, pte | addr); + *dpt_ofs += 8; + src_idx -= src_stride; + } + + /* The DE ignores the PTEs for the padding tiles */ + *dpt_ofs += (dst_stride - height) * 8; + } + + /* Align to next page */ + *dpt_ofs = ALIGN(*dpt_ofs, 4096); +} + +static unsigned int +write_dpt_padding(struct iosys_map *map, unsigned int dest, unsigned int pad) +{ + /* The DE ignores the PTEs for the padding tiles */ + return dest + pad * sizeof(u64); +} + +static unsigned int +write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map, + unsigned int dest, + const struct intel_remapped_plane_info *plane) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; + const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, + xe_cache_pat_idx(xe, XE_CACHE_NONE)); + unsigned int offset = plane->offset * XE_PAGE_SIZE; + unsigned int size = plane->size; + + while (size--) { + u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE); + + iosys_map_wr(map, dest, u64, addr | pte); + dest += sizeof(u64); + offset += XE_PAGE_SIZE; + } + + return dest; +} + +static unsigned int +write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map, + unsigned int dest, + const struct intel_remapped_plane_info *plane) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; + const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, + xe_cache_pat_idx(xe, XE_CACHE_NONE)); + unsigned int offset, column, row; + + for (row = 0; row < plane->height; row++) { + offset = (plane->offset + plane->src_stride * row) * + XE_PAGE_SIZE; + + for (column = 0; column < plane->width; column++) { + u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE); + + iosys_map_wr(map, dest, u64, addr | pte); + dest += sizeof(u64); + offset += XE_PAGE_SIZE; + } + + dest = write_dpt_padding(map, dest, + plane->dst_stride - plane->width); + } + + return dest; +} + +static void +write_dpt_remapped(struct xe_bo *bo, + const struct intel_remapped_info *remap_info, + struct iosys_map *map) +{ + unsigned int i, dest = 0; + + for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) { + const struct intel_remapped_plane_info *plane = + &remap_info->plane[i]; + + if (!plane->linear && !plane->width && !plane->height) + continue; + + if (dest && remap_info->plane_alignment) { + const unsigned int index = dest / sizeof(u64); + const unsigned int pad = + ALIGN(index, remap_info->plane_alignment) - + index; + + dest = write_dpt_padding(map, dest, pad); + } + + if (plane->linear) + dest = write_dpt_remapped_linear(bo, map, dest, plane); + else + dest = write_dpt_remapped_tiled(bo, map, dest, plane); + } +} + +static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma *vma) +{ + struct xe_device *xe = to_xe_device(obj->dev); + struct xe_tile *tile0 = xe_device_get_root_tile(xe); + struct xe_ggtt *ggtt = tile0->mem.ggtt; + const struct i915_gtt_view *view = pin_params->view; + struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; + u32 dpt_size, size = bo->ttm.base.size; + + if (view->type == I915_GTT_VIEW_NORMAL) + dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); + else if (view->type == I915_GTT_VIEW_REMAPPED) + dpt_size = ALIGN(intel_remapped_info_size(&view->remapped) * 8, + XE_PAGE_SIZE); + else + /* display uses 4K tiles instead of bytes here, convert to entries.. */ + dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, + XE_PAGE_SIZE); + + dpt = xe_bo_create_pin_map_at_novm(xe, tile0, + dpt_size, ~0ull, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile0) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_PAGETABLE | + XE_BO_FLAG_FORCE_WC, + pin_params->alignment, false); + if (IS_ERR(dpt)) + return PTR_ERR(dpt); + + if (view->type == I915_GTT_VIEW_NORMAL) { + u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); + u32 x; + + for (x = 0; x < size / XE_PAGE_SIZE; x++) { + u64 addr = xe_bo_addr(bo, x * XE_PAGE_SIZE, XE_PAGE_SIZE); + + iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr); + } + } else if (view->type == I915_GTT_VIEW_REMAPPED) { + write_dpt_remapped(bo, &view->remapped, &dpt->vmap); + } else { + const struct intel_rotation_info *rot_info = &view->rotated; + u32 i, dpt_ofs = 0; + + for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) + write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, + rot_info->plane[i].offset, + rot_info->plane[i].width, + rot_info->plane[i].height, + rot_info->plane[i].src_stride, + rot_info->plane[i].dst_stride); + } + + vma->dpt = dpt; + vma->node = dpt->ggtt_node[tile0->id]; + + /* Ensure DPT writes are flushed */ + xe_device_l2_flush(xe); + return 0; +} + +static void +write_ggtt_rotated(struct xe_ggtt *ggtt, u32 *ggtt_ofs, + u64 pte_flags, + xe_ggtt_set_pte_fn write_pte, + struct xe_bo *bo, u32 bo_ofs, + u32 width, u32 height, u32 src_stride, u32 dst_stride) +{ + u32 column, row; + + for (column = 0; column < width; column++) { + u32 src_idx = src_stride * (height - 1) + column + bo_ofs; + + for (row = 0; row < height; row++) { + u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); + + write_pte(ggtt, *ggtt_ofs, pte_flags | addr); + *ggtt_ofs += XE_PAGE_SIZE; + src_idx -= src_stride; + } + + /* The DE ignores the PTEs for the padding tiles */ + *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; + } +} + +struct fb_rotate_args { + const struct i915_gtt_view *view; + struct xe_bo *bo; +}; + +static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, + u64 pte_flags, xe_ggtt_set_pte_fn write_pte, void *data) +{ + struct fb_rotate_args *args = data; + struct xe_bo *bo = args->bo; + const struct intel_rotation_info *rot_info = &args->view->rotated; + u32 ggtt_ofs = xe_ggtt_node_addr(node); + + for (u32 i = 0; i < ARRAY_SIZE(rot_info->plane); i++) + write_ggtt_rotated(ggtt, &ggtt_ofs, pte_flags, write_pte, + bo, rot_info->plane[i].offset, + rot_info->plane[i].width, + rot_info->plane[i].height, + rot_info->plane[i].src_stride, + rot_info->plane[i].dst_stride); +} + +static int __xe_pin_fb_vma_ggtt(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma *vma) +{ + const struct i915_gtt_view *view = pin_params->view; + struct xe_bo *bo = gem_to_xe_bo(obj); + struct xe_device *xe = to_xe_device(obj->dev); + struct xe_tile *tile0 = xe_device_get_root_tile(xe); + struct xe_ggtt *ggtt = tile0->mem.ggtt; + u64 pte, size; + u32 align; + int ret = 0; + + /* TODO: Consider sharing framebuffer mapping? + * embed i915_vma inside intel_framebuffer + */ + guard(xe_pm_runtime_noresume)(xe); + + align = max(XE_PAGE_SIZE, pin_params->alignment); + if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + align = max(align, SZ_64K); + + /* Fast case, preallocated GGTT view? */ + if (bo->ggtt_node[tile0->id] && view->type == I915_GTT_VIEW_NORMAL) { + vma->node = bo->ggtt_node[tile0->id]; + return 0; + } + + /* TODO: Consider sharing framebuffer mapping? + * embed i915_vma inside intel_framebuffer + */ + if (view->type == I915_GTT_VIEW_NORMAL) + size = xe_bo_size(bo); + else + /* display uses tiles instead of bytes here, so convert it back.. */ + size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE; + + pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); + vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte, + ALIGN(size, align), align, + view->type == I915_GTT_VIEW_NORMAL ? + NULL : write_ggtt_rotated_node, + &(struct fb_rotate_args){view, bo}); + if (IS_ERR(vma->node)) + ret = PTR_ERR(vma->node); + + return ret; +} + +static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, + const struct intel_fb_pin_params *pin_params) +{ + struct xe_device *xe = to_xe_device(obj->dev); + struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); + struct xe_bo *bo = gem_to_xe_bo(obj); + struct xe_validation_ctx ctx; + struct drm_exec exec; + int ret = 0; + + /* We reject creating !SCANOUT fb's, so this is weird.. */ + drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC) && + bo->ttm.type != ttm_bo_type_sg); + + if (!vma) + return ERR_PTR(-ENODEV); + + refcount_set(&vma->ref, 1); + if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && + pin_params->needs_cpu_lmem_access && + !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { + struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; + + /* + * If we need to able to access the clear-color value stored in + * the buffer, then we require that such buffers are also CPU + * accessible. This is important on small-bar systems where + * only some subset of VRAM is CPU accessible. + */ + if (xe_vram_region_io_size(vram) < xe_vram_region_usable_size(vram)) { + ret = -EINVAL; + goto err; + } + } + + /* + * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the + * assumptions are incorrect for framebuffers + */ + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, + ret) { + ret = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + if (ret) + break; + + if (IS_DGFX(xe)) + ret = xe_bo_migrate(bo, XE_PL_VRAM0, NULL, &exec); + else + ret = xe_bo_validate(bo, NULL, true, &exec); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &ret); + if (!ret) + ttm_bo_pin(&bo->ttm); + } + if (ret) + goto err; + + vma->bo = bo; + if (is_dpt) + ret = __xe_pin_fb_vma_dpt(obj, pin_params, vma); + else + ret = __xe_pin_fb_vma_ggtt(obj, pin_params, vma); + if (ret) + goto err_unpin; + + return vma; + +err_unpin: + ttm_bo_reserve(&bo->ttm, false, false, NULL); + ttm_bo_unpin(&bo->ttm); + ttm_bo_unreserve(&bo->ttm); +err: + kfree(vma); + return ERR_PTR(ret); +} + +static void __xe_unpin_fb_vma(struct i915_vma *vma) +{ + u8 tile_id = xe_device_get_root_tile(xe_bo_device(vma->bo))->id; + + if (!refcount_dec_and_test(&vma->ref)) + return; + + if (vma->dpt) + xe_bo_unpin_map_no_vm(vma->dpt); + else if (vma->bo->ggtt_node[tile_id] != vma->node) + xe_ggtt_node_remove(vma->node, false); + + ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); + ttm_bo_unpin(&vma->bo->ttm); + ttm_bo_unreserve(&vma->bo->ttm); + kfree(vma); +} + +int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_ggtt_vma, + u32 *out_offset, + int *out_fence_id) +{ + struct i915_vma *ggtt_vma; + + ggtt_vma = __xe_pin_fb_vma(obj, false, pin_params); + if (IS_ERR(ggtt_vma)) + return PTR_ERR(ggtt_vma); + + *out_ggtt_vma = ggtt_vma; + *out_offset = xe_ggtt_node_addr(ggtt_vma->node); + if (out_fence_id) + *out_fence_id = -1; + + return 0; +} + +static void xe_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma, + int fence_id) +{ + WARN_ON(fence_id >= 0); + + __xe_unpin_fb_vma(ggtt_vma); +} + +static int xe_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_dpt_vma, + struct i915_vma **out_ggtt_vma, + u32 *out_offset) +{ + struct i915_vma *ggtt_vma; + + WARN_ON(dpt); + + ggtt_vma = __xe_pin_fb_vma(obj, true, pin_params); + if (IS_ERR(ggtt_vma)) + return PTR_ERR(ggtt_vma); + + *out_dpt_vma = NULL; /* not used on xe */ + *out_ggtt_vma = ggtt_vma; + *out_offset = xe_ggtt_node_addr(ggtt_vma->node); + + return 0; +} + +static void xe_fb_pin_dpt_unpin(struct intel_dpt *dpt, + struct i915_vma *dpt_vma, + struct i915_vma *ggtt_vma) +{ + WARN_ON(dpt || dpt_vma); + + __xe_unpin_fb_vma(ggtt_vma); +} + +static struct i915_vma * +xe_fb_pin_reuse_vma(struct i915_vma *old_ggtt_vma, + struct drm_gem_object *old_obj, + const struct i915_gtt_view *old_view, + struct drm_gem_object *new_obj, + const struct i915_gtt_view *new_view, + u32 *out_offset) +{ + if (old_ggtt_vma && old_obj == new_obj && + !memcmp(old_view, new_view, sizeof(*new_view))) { + refcount_inc(&old_ggtt_vma->ref); + + *out_offset = xe_ggtt_node_addr(old_ggtt_vma->node); + + return old_ggtt_vma; + } + + return NULL; +} + +static void xe_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map) +{ + *map = vma->bo->vmap; +} + +const struct intel_display_fb_pin_interface xe_display_fb_pin_interface = { + .ggtt_pin = xe_fb_pin_ggtt_pin, + .ggtt_unpin = xe_fb_pin_ggtt_unpin, + .dpt_pin = xe_fb_pin_dpt_pin, + .dpt_unpin = xe_fb_pin_dpt_unpin, + .reuse_vma = xe_fb_pin_reuse_vma, + .get_map = xe_fb_pin_get_map, +}; diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.h b/drivers/gpu/drm/xe/display/xe_fb_pin.h new file mode 100644 index 000000000..20dd8a99a --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef __XE_FB_PIN_H__ +#define __XE_FB_PIN_H__ + +#include + +struct drm_gem_object; +struct i915_vma; +struct intel_fb_pin_params; + +int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_ggtt_vma, + u32 *out_offset, + int *out_fence_id); + +extern const struct intel_display_fb_pin_interface xe_display_fb_pin_interface; + +#endif /* __XE_FB_PIN_H__ */ diff --git a/drivers/gpu/drm/xe/display/xe_frontbuffer.c b/drivers/gpu/drm/xe/display/xe_frontbuffer.c new file mode 100644 index 000000000..113fc017e --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_frontbuffer.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT +/* Copyright © 2026 Intel Corporation */ + +#include +#include + +#include "intel_frontbuffer.h" +#include "xe_frontbuffer.h" + +struct xe_frontbuffer { + struct intel_frontbuffer base; + struct drm_gem_object *obj; + struct kref ref; +}; + +static struct intel_frontbuffer *xe_frontbuffer_get(struct drm_gem_object *obj) +{ + struct xe_frontbuffer *front; + + front = kmalloc_obj(*front); + if (!front) + return NULL; + + intel_frontbuffer_init(&front->base, obj->dev); + + kref_init(&front->ref); + + drm_gem_object_get(obj); + front->obj = obj; + + return &front->base; +} + +static void xe_frontbuffer_ref(struct intel_frontbuffer *_front) +{ + struct xe_frontbuffer *front = + container_of(_front, typeof(*front), base); + + kref_get(&front->ref); +} + +static void frontbuffer_release(struct kref *ref) +{ + struct xe_frontbuffer *front = + container_of(ref, typeof(*front), ref); + + intel_frontbuffer_fini(&front->base); + + drm_gem_object_put(front->obj); + + kfree(front); +} + +static void xe_frontbuffer_put(struct intel_frontbuffer *_front) +{ + struct xe_frontbuffer *front = + container_of(_front, typeof(*front), base); + + kref_put(&front->ref, frontbuffer_release); +} + +static void xe_frontbuffer_flush_for_display(struct intel_frontbuffer *front) +{ +} + +const struct intel_display_frontbuffer_interface xe_display_frontbuffer_interface = { + .get = xe_frontbuffer_get, + .ref = xe_frontbuffer_ref, + .put = xe_frontbuffer_put, + .flush_for_display = xe_frontbuffer_flush_for_display, +}; diff --git a/drivers/gpu/drm/xe/display/xe_frontbuffer.h b/drivers/gpu/drm/xe/display/xe_frontbuffer.h new file mode 100644 index 000000000..6b4f59b42 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_frontbuffer.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef _XE_FRONTBUFFER_H_ +#define _XE_FRONTBUFFER_H_ + +extern const struct intel_display_frontbuffer_interface xe_display_frontbuffer_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c new file mode 100644 index 000000000..33494b862 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2023, Intel Corporation. + */ + +#include + +#include +#include +#include + +#include "abi/gsc_command_header_abi.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gsc_proxy.h" +#include "xe_gsc_submit.h" +#include "xe_hdcp_gsc.h" +#include "xe_map.h" +#include "xe_pm.h" +#include "xe_uc_fw.h" + +#define HECI_MEADDRESS_HDCP 18 + +struct intel_hdcp_gsc_context { + struct xe_device *xe; + struct xe_bo *hdcp_bo; + u64 hdcp_cmd_in; + u64 hdcp_cmd_out; +}; + +#define HDCP_GSC_HEADER_SIZE sizeof(struct intel_gsc_mtl_header) + +static bool intel_hdcp_gsc_check_status(struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_gt *gt = tile->media_gt; + struct xe_gsc *gsc; + + if (!gt) { + drm_dbg_kms(&xe->drm, + "not checking GSC status for HDCP2.x: media GT not present or disabled\n"); + return false; + } + + gsc = >->uc.gsc; + + if (!xe_uc_fw_is_available(&gsc->fw)) { + drm_dbg_kms(&xe->drm, + "GSC Components not ready for HDCP2.x\n"); + return false; + } + + guard(xe_pm_runtime)(xe); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + if (!fw_ref.domains) { + drm_dbg_kms(&xe->drm, + "failed to get forcewake to check proxy status\n"); + return false; + } + + return xe_gsc_proxy_init_done(gsc); +} + +/*This function helps allocate memory for the command that we will send to gsc cs */ +static int intel_hdcp_gsc_initialize_message(struct xe_device *xe, + struct intel_hdcp_gsc_context *gsc_context) +{ + struct xe_bo *bo = NULL; + u64 cmd_in, cmd_out; + int ret = 0; + + /* allocate object of two page for HDCP command memory and store it */ + bo = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), PAGE_SIZE * 2, + ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT, false); + + if (IS_ERR(bo)) { + drm_err(&xe->drm, "Failed to allocate bo for HDCP streaming command!\n"); + ret = PTR_ERR(bo); + goto out; + } + + cmd_in = xe_bo_ggtt_addr(bo); + cmd_out = cmd_in + PAGE_SIZE; + xe_map_memset(xe, &bo->vmap, 0, 0, xe_bo_size(bo)); + + gsc_context->hdcp_bo = bo; + gsc_context->hdcp_cmd_in = cmd_in; + gsc_context->hdcp_cmd_out = cmd_out; + gsc_context->xe = xe; + +out: + return ret; +} + +static struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + struct intel_hdcp_gsc_context *gsc_context; + int ret; + + gsc_context = kzalloc_obj(*gsc_context); + if (!gsc_context) + return ERR_PTR(-ENOMEM); + + /* + * NOTE: No need to lock the comp mutex here as it is already + * going to be taken before this function called + */ + ret = intel_hdcp_gsc_initialize_message(xe, gsc_context); + if (ret) { + drm_err(&xe->drm, "Could not initialize gsc_context\n"); + kfree(gsc_context); + gsc_context = ERR_PTR(ret); + } + + return gsc_context; +} + +static void intel_hdcp_gsc_context_free(struct intel_hdcp_gsc_context *gsc_context) +{ + if (!gsc_context) + return; + + xe_bo_unpin_map_no_vm(gsc_context->hdcp_bo); + kfree(gsc_context); +} + +static int xe_gsc_send_sync(struct xe_device *xe, + struct intel_hdcp_gsc_context *gsc_context, + u32 msg_size_in, u32 msg_size_out, + u32 addr_out_off) +{ + struct xe_gt *gt = gsc_context->hdcp_bo->tile->media_gt; + struct iosys_map *map = &gsc_context->hdcp_bo->vmap; + struct xe_gsc *gsc = >->uc.gsc; + int ret; + + ret = xe_gsc_pkt_submit_kernel(gsc, gsc_context->hdcp_cmd_in, msg_size_in, + gsc_context->hdcp_cmd_out, msg_size_out); + if (ret) { + drm_err(&xe->drm, "failed to send gsc HDCP msg (%d)\n", ret); + return ret; + } + + if (xe_gsc_check_and_update_pending(xe, map, 0, map, addr_out_off)) + return -EAGAIN; + + ret = xe_gsc_read_out_header(xe, map, addr_out_off, + sizeof(struct hdcp_cmd_header), NULL); + + return ret; +} + +static ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context, + void *msg_in, size_t msg_in_len, + void *msg_out, size_t msg_out_len) +{ + struct xe_device *xe = gsc_context->xe; + const size_t max_msg_size = PAGE_SIZE - HDCP_GSC_HEADER_SIZE; + u64 host_session_id; + u32 msg_size_in, msg_size_out; + u32 addr_out_off, addr_in_wr_off = 0; + int ret, tries = 0; + + if (msg_in_len > max_msg_size || msg_out_len > max_msg_size) + return -ENOSPC; + + msg_size_in = msg_in_len + HDCP_GSC_HEADER_SIZE; + msg_size_out = msg_out_len + HDCP_GSC_HEADER_SIZE; + addr_out_off = PAGE_SIZE; + + host_session_id = xe_gsc_create_host_session_id(); + guard(xe_pm_runtime_noresume)(xe); + addr_in_wr_off = xe_gsc_emit_header(xe, &gsc_context->hdcp_bo->vmap, + addr_in_wr_off, HECI_MEADDRESS_HDCP, + host_session_id, msg_in_len); + xe_map_memcpy_to(xe, &gsc_context->hdcp_bo->vmap, addr_in_wr_off, + msg_in, msg_in_len); + /* + * Keep sending request in case the pending bit is set no need to add + * message handle as we are using same address hence loc. of header is + * same and it will contain the message handle. we will send the message + * 20 times each message 50 ms apart + */ + do { + ret = xe_gsc_send_sync(xe, gsc_context, msg_size_in, msg_size_out, + addr_out_off); + + /* Only try again if gsc says so */ + if (ret != -EAGAIN) + break; + + msleep(50); + + } while (++tries < 20); + + if (ret) + return ret; + + xe_map_memcpy_from(xe, msg_out, &gsc_context->hdcp_bo->vmap, + addr_out_off + HDCP_GSC_HEADER_SIZE, + msg_out_len); + + return ret; +} + +const struct intel_display_hdcp_interface xe_display_hdcp_interface = { + .gsc_msg_send = intel_hdcp_gsc_msg_send, + .gsc_check_status = intel_hdcp_gsc_check_status, + .gsc_context_alloc = intel_hdcp_gsc_context_alloc, + .gsc_context_free = intel_hdcp_gsc_context_free, +}; diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.h b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.h new file mode 100644 index 000000000..c1062e4b6 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2025 Intel Corporation */ + +#ifndef __XE_HDCP_GSC_H__ +#define __XE_HDCP_GSC_H__ + +extern const struct intel_display_hdcp_interface xe_display_hdcp_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.c b/drivers/gpu/drm/xe/display/xe_initial_plane.c new file mode 100644 index 000000000..0f86b7303 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_initial_plane.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include + +#include "regs/xe_gtt_defs.h" + +/* FIXME move intel_remapped_info_size() & co. */ +#include "intel_fb.h" + +/* FIXME move intel_initial_plane_config */ +#include "intel_display_types.h" + +#include "xe_bo.h" +#include "xe_display_bo.h" +#include "xe_display_vma.h" +#include "xe_fb_pin.h" +#include "xe_ggtt.h" +#include "xe_mmio.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_vram_types.h" + +static bool is_pte_local(u64 pte) +{ + return pte & XE_GGTT_PTE_DM; +} + +static bool has_lmembar(struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) >= 1270; +} + +static bool need_pte_local(struct xe_device *xe) +{ + return IS_DGFX(xe) || has_lmembar(xe); +} + +static struct xe_bo * +initial_plane_bo(struct xe_device *xe, + struct intel_initial_plane_config *plane_config) +{ + struct xe_tile *tile0 = xe_device_get_root_tile(xe); + struct xe_bo *bo; + resource_size_t phys_base; + u32 base, size, flags; + u64 page_size = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; + + if (plane_config->size == 0) + return NULL; + + flags = XE_BO_FLAG_FORCE_WC | XE_BO_FLAG_GGTT; + + base = round_down(plane_config->base, page_size); + size = round_up(plane_config->base + plane_config->size, + page_size); + size -= base; + + if (IS_DGFX(xe)) { + u64 pte = xe_ggtt_read_pte(tile0->mem.ggtt, base); + + if (is_pte_local(pte) != need_pte_local(xe)) { + drm_err(&xe->drm, "Initial plane PTE has bad local memory bit\n"); + return NULL; + } + + phys_base = pte & ~(page_size - 1); + + flags |= XE_BO_FLAG_VRAM0; + + /* + * We don't currently expect this to ever be placed in the + * stolen portion. + */ + if (phys_base >= xe_vram_region_usable_size(tile0->mem.vram)) { + drm_err(&xe->drm, + "Initial plane programming using invalid range, phys_base=%pa\n", + &phys_base); + return NULL; + } + + drm_dbg_kms(&xe->drm, + "Using phys_base=%pa, based on initial plane programming\n", + &phys_base); + } else { + struct ttm_resource_manager *stolen; + u64 pte; + + stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); + if (!stolen) { + drm_dbg_kms(&xe->drm, "No stolen for initial FB\n"); + return NULL; + } + + pte = xe_ggtt_read_pte(tile0->mem.ggtt, base); + + if (is_pte_local(pte) != need_pte_local(xe)) { + drm_err(&xe->drm, "Initial plane PTE has bad local memory bit\n"); + return NULL; + } + + phys_base = base; + flags |= XE_BO_FLAG_STOLEN; + + if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) && + IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) && + !xe_display_bo_fbdev_prefer_stolen(xe, plane_config->size)) { + drm_info(&xe->drm, "Initial FB size exceeds half of stolen, discarding\n"); + return NULL; + } + } + + bo = xe_bo_create_pin_map_at_novm(xe, tile0, size, phys_base, + ttm_bo_type_kernel, flags, 0, false); + if (IS_ERR(bo)) { + drm_dbg_kms(&xe->drm, + "Failed to create bo phys_base=%pa size %u with flags %x: %li\n", + &phys_base, size, flags, PTR_ERR(bo)); + return NULL; + } + + return bo; +} + +static struct drm_gem_object * +xe_alloc_initial_plane_obj(struct drm_device *drm, + struct intel_initial_plane_config *plane_config) +{ + struct xe_device *xe = to_xe_device(drm); + struct drm_mode_fb_cmd2 mode_cmd = { 0 }; + struct drm_framebuffer *fb = plane_config->fb; + struct xe_bo *bo; + + mode_cmd.pixel_format = fb->format->format; + mode_cmd.width = fb->width; + mode_cmd.height = fb->height; + mode_cmd.pitches[0] = fb->pitches[0]; + mode_cmd.modifier[0] = fb->modifier; + mode_cmd.flags = DRM_MODE_FB_MODIFIERS; + + bo = initial_plane_bo(xe, plane_config); + if (!bo) + return NULL; + + if (intel_framebuffer_init(to_intel_framebuffer(fb), + &bo->ttm.base, fb->format, &mode_cmd)) { + drm_dbg_kms(&xe->drm, "intel fb init failed\n"); + goto err_bo; + } + /* Reference handed over to fb */ + xe_bo_put(bo); + + return &bo->ttm.base; + +err_bo: + xe_bo_unpin_map_no_vm(bo); + return NULL; +} + +static int +xe_initial_plane_setup(struct drm_plane_state *_plane_state, + struct intel_initial_plane_config *plane_config, + struct drm_framebuffer *fb, + struct i915_vma *_unused) +{ + struct intel_plane_state *plane_state = to_intel_plane_state(_plane_state); + struct i915_vma *vma; + struct intel_fb_pin_params pin_params = { + .view = &plane_state->view.gtt, + }; + u32 offset; + int ret; + + ret = xe_fb_pin_ggtt_pin(intel_fb_bo(fb), &pin_params, &vma, &offset, NULL); + if (ret) + return ret; + + plane_state->ggtt_vma = vma; + + plane_state->surf = offset; + + plane_config->vma = vma; + + return 0; +} + +static void xe_plane_config_fini(struct intel_initial_plane_config *plane_config) +{ +} + +const struct intel_display_initial_plane_interface xe_display_initial_plane_interface = { + .alloc_obj = xe_alloc_initial_plane_obj, + .setup = xe_initial_plane_setup, + .config_fini = xe_plane_config_fini, +}; diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.h b/drivers/gpu/drm/xe/display/xe_initial_plane.h new file mode 100644 index 000000000..399d15f14 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_initial_plane.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2025 Intel Corporation */ + +#ifndef __XE_INITIAL_PLANE_H__ +#define __XE_INITIAL_PLANE_H__ + +extern const struct intel_display_initial_plane_interface xe_display_initial_plane_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_panic.c b/drivers/gpu/drm/xe/display/xe_panic.c new file mode 100644 index 000000000..12c6fb990 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_panic.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MIT +/* Copyright © 2025 Intel Corporation */ + +#include +#include +#include + +#include "xe_bo.h" +#include "xe_panic.h" +#include "xe_res_cursor.h" + +struct intel_panic { + struct xe_res_cursor res; + struct iosys_map vmap; + + int page; + + struct xe_bo *bo; + unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width); +}; + +static void xe_panic_kunmap(struct intel_panic *panic) +{ + if (!panic->vmap.is_iomem && iosys_map_is_set(&panic->vmap)) { + drm_clflush_virt_range(panic->vmap.vaddr, PAGE_SIZE); + kunmap_local(panic->vmap.vaddr); + } + iosys_map_clear(&panic->vmap); + panic->page = -1; +} + +/* + * The scanout buffer pages are not mapped, so for each pixel, + * use kmap_local_page_try_from_panic() to map the page, and write the pixel. + * Try to keep the map from the previous pixel, to avoid too much map/unmap. + */ +static void xe_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int x, + unsigned int y, u32 color) +{ + struct intel_panic *panic = sb->private; + struct xe_bo *bo = panic->bo; + unsigned int new_page; + unsigned int offset; + + if (panic->tiling) + offset = panic->tiling(sb->width, x, y); + else + offset = y * sb->pitch[0] + x * sb->format->cpp[0]; + + new_page = offset >> PAGE_SHIFT; + offset = offset % PAGE_SIZE; + if (new_page != panic->page) { + if (xe_bo_is_vram(bo)) { + /* Display is always mapped on root tile */ + struct xe_vram_region *vram = xe_bo_device(bo)->mem.vram; + + if (panic->page < 0 || new_page < panic->page) { + xe_res_first(bo->ttm.resource, new_page * PAGE_SIZE, + bo->ttm.base.size - new_page * PAGE_SIZE, &panic->res); + } else { + xe_res_next(&panic->res, PAGE_SIZE * (new_page - panic->page)); + } + iosys_map_set_vaddr_iomem(&panic->vmap, + vram->mapping + panic->res.start); + } else { + xe_panic_kunmap(panic); + iosys_map_set_vaddr(&panic->vmap, + ttm_bo_kmap_try_from_panic(&bo->ttm, + new_page)); + } + panic->page = new_page; + } + + if (iosys_map_is_set(&panic->vmap)) + iosys_map_wr(&panic->vmap, offset, u32, color); +} + +static struct intel_panic *xe_panic_alloc(void) +{ + struct intel_panic *panic; + + panic = kzalloc_obj(*panic); + + return panic; +} + +static int xe_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb, + struct drm_gem_object *obj, + unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width)) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + + if (xe_bo_is_vram(bo) && !xe_bo_is_visible_vram(bo)) + return -ENODEV; + + panic->page = -1; + panic->bo = bo; + panic->tiling = tiling; + + sb->private = panic; + sb->set_pixel = xe_panic_page_set_pixel; + + return 0; +} + +const struct intel_display_panic_interface xe_display_panic_interface = { + .alloc = xe_panic_alloc, + .setup = xe_panic_setup, + .finish = xe_panic_kunmap, +}; diff --git a/drivers/gpu/drm/xe/display/xe_panic.h b/drivers/gpu/drm/xe/display/xe_panic.h new file mode 100644 index 000000000..3054b5110 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_panic.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2025 Intel Corporation */ + +#ifndef __XE_PANIC_H__ +#define __XE_PANIC_H__ + +extern const struct intel_display_panic_interface xe_display_panic_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_stolen.c b/drivers/gpu/drm/xe/display/xe_stolen.c new file mode 100644 index 000000000..5c7df67be --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_stolen.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: MIT +/* Copyright © 2025 Intel Corporation */ + +#include + +#include "xe_res_cursor.h" +#include "xe_stolen.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_validation.h" + +struct intel_stolen_node { + struct xe_device *xe; + struct xe_bo *bo; +}; + +static int xe_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size, + unsigned int align, u64 start, u64 end) +{ + struct xe_device *xe = node->xe; + + struct xe_bo *bo; + int err = 0; + u32 flags = XE_BO_FLAG_PINNED | XE_BO_FLAG_STOLEN; + + if (start < SZ_4K) + start = SZ_4K; + + if (align) { + size = ALIGN(size, align); + start = ALIGN(start, align); + } + + bo = xe_bo_create_pin_range_novm(xe, xe_device_get_root_tile(xe), + size, start, end, ttm_bo_type_kernel, flags); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + bo = NULL; + return err; + } + + node->bo = bo; + + return err; +} + +static void xe_stolen_remove_node(struct intel_stolen_node *node) +{ + xe_bo_unpin_map_no_vm(node->bo); + node->bo = NULL; +} + +static bool xe_stolen_initialized(struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + + return ttm_manager_type(&xe->ttm, XE_PL_STOLEN); +} + +static bool xe_stolen_node_allocated(const struct intel_stolen_node *node) +{ + return node->bo; +} + +static u64 xe_stolen_node_offset(const struct intel_stolen_node *node) +{ + struct xe_res_cursor res; + + xe_res_first(node->bo->ttm.resource, 0, 4096, &res); + return res.start; +} + +static u64 xe_stolen_node_address(const struct intel_stolen_node *node) +{ + struct xe_device *xe = node->xe; + + return xe_ttm_stolen_gpu_offset(xe) + xe_stolen_node_offset(node); +} + +static u64 xe_stolen_node_size(const struct intel_stolen_node *node) +{ + return xe_bo_size(node->bo); +} + +static struct intel_stolen_node *xe_stolen_node_alloc(struct drm_device *drm) +{ + struct xe_device *xe = to_xe_device(drm); + struct intel_stolen_node *node; + + node = kzalloc_obj(*node); + if (!node) + return NULL; + + node->xe = xe; + + return node; +} + +static void xe_stolen_node_free(const struct intel_stolen_node *node) +{ + kfree(node); +} + +const struct intel_display_stolen_interface xe_display_stolen_interface = { + .insert_node_in_range = xe_stolen_insert_node_in_range, + .remove_node = xe_stolen_remove_node, + .initialized = xe_stolen_initialized, + .node_allocated = xe_stolen_node_allocated, + .node_offset = xe_stolen_node_offset, + .node_address = xe_stolen_node_address, + .node_size = xe_stolen_node_size, + .node_alloc = xe_stolen_node_alloc, + .node_free = xe_stolen_node_free, +}; diff --git a/drivers/gpu/drm/xe/display/xe_stolen.h b/drivers/gpu/drm/xe/display/xe_stolen.h new file mode 100644 index 000000000..db86b9e01 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_stolen.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2025 Intel Corporation */ + +#ifndef __XE_STOLEN_H__ +#define __XE_STOLEN_H__ + +extern const struct intel_display_stolen_interface xe_display_stolen_interface; + +#endif diff --git a/drivers/gpu/drm/xe/display/xe_tdf.c b/drivers/gpu/drm/xe/display/xe_tdf.c new file mode 100644 index 000000000..78bda4c47 --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_tdf.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include "intel_display_core.h" +#include "intel_tdf.h" +#include "xe_device.h" + +void intel_td_flush(struct intel_display *display) +{ + struct xe_device *xe = to_xe_device(display->drm); + + xe_device_td_flush(xe); +} diff --git a/drivers/gpu/drm/xe/instructions/xe_alu_commands.h b/drivers/gpu/drm/xe/instructions/xe_alu_commands.h new file mode 100644 index 000000000..2987b10d3 --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_alu_commands.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_ALU_COMMANDS_H_ +#define _XE_ALU_COMMANDS_H_ + +#include "instructions/xe_instr_defs.h" + +/* Instruction Opcodes */ +#define CS_ALU_OPCODE_NOOP 0x000 +#define CS_ALU_OPCODE_FENCE_RD 0x001 +#define CS_ALU_OPCODE_FENCE_WR 0x002 +#define CS_ALU_OPCODE_LOAD 0x080 +#define CS_ALU_OPCODE_LOADINV 0x480 +#define CS_ALU_OPCODE_LOAD0 0x081 +#define CS_ALU_OPCODE_LOAD1 0x481 +#define CS_ALU_OPCODE_LOADIND 0x082 +#define CS_ALU_OPCODE_ADD 0x100 +#define CS_ALU_OPCODE_SUB 0x101 +#define CS_ALU_OPCODE_AND 0x102 +#define CS_ALU_OPCODE_OR 0x103 +#define CS_ALU_OPCODE_XOR 0x104 +#define CS_ALU_OPCODE_SHL 0x105 +#define CS_ALU_OPCODE_SHR 0x106 +#define CS_ALU_OPCODE_SAR 0x107 +#define CS_ALU_OPCODE_STORE 0x180 +#define CS_ALU_OPCODE_STOREINV 0x580 +#define CS_ALU_OPCODE_STOREIND 0x181 + +/* Instruction Operands */ +#define CS_ALU_OPERAND_REG(n) REG_FIELD_PREP(GENMASK(3, 0), (n)) +#define CS_ALU_OPERAND_REG0 0x0 +#define CS_ALU_OPERAND_REG1 0x1 +#define CS_ALU_OPERAND_REG2 0x2 +#define CS_ALU_OPERAND_REG3 0x3 +#define CS_ALU_OPERAND_REG4 0x4 +#define CS_ALU_OPERAND_REG5 0x5 +#define CS_ALU_OPERAND_REG6 0x6 +#define CS_ALU_OPERAND_REG7 0x7 +#define CS_ALU_OPERAND_REG8 0x8 +#define CS_ALU_OPERAND_REG9 0x9 +#define CS_ALU_OPERAND_REG10 0xa +#define CS_ALU_OPERAND_REG11 0xb +#define CS_ALU_OPERAND_REG12 0xc +#define CS_ALU_OPERAND_REG13 0xd +#define CS_ALU_OPERAND_REG14 0xe +#define CS_ALU_OPERAND_REG15 0xf +#define CS_ALU_OPERAND_SRCA 0x20 +#define CS_ALU_OPERAND_SRCB 0x21 +#define CS_ALU_OPERAND_ACCU 0x31 +#define CS_ALU_OPERAND_ZF 0x32 +#define CS_ALU_OPERAND_CF 0x33 +#define CS_ALU_OPERAND_NA 0 /* N/A operand */ + +/* Command Streamer ALU Instructions */ +#define CS_ALU_INSTR(opcode, op1, op2) (REG_FIELD_PREP(GENMASK(31, 20), (opcode)) | \ + REG_FIELD_PREP(GENMASK(19, 10), (op1)) | \ + REG_FIELD_PREP(GENMASK(9, 0), (op2))) + +#define __CS_ALU_INSTR(opcode, op1, op2) CS_ALU_INSTR(CS_ALU_OPCODE_##opcode, \ + CS_ALU_OPERAND_##op1, \ + CS_ALU_OPERAND_##op2) + +#define CS_ALU_INSTR_NOOP __CS_ALU_INSTR(NOOP, NA, NA) +#define CS_ALU_INSTR_LOAD(op1, op2) __CS_ALU_INSTR(LOAD, op1, op2) +#define CS_ALU_INSTR_LOADINV(op1, op2) __CS_ALU_INSTR(LOADINV, op1, op2) +#define CS_ALU_INSTR_LOAD0(op1) __CS_ALU_INSTR(LOAD0, op1, NA) +#define CS_ALU_INSTR_LOAD1(op1) __CS_ALU_INSTR(LOAD1, op1, NA) +#define CS_ALU_INSTR_ADD __CS_ALU_INSTR(ADD, NA, NA) +#define CS_ALU_INSTR_SUB __CS_ALU_INSTR(SUB, NA, NA) +#define CS_ALU_INSTR_AND __CS_ALU_INSTR(AND, NA, NA) +#define CS_ALU_INSTR_OR __CS_ALU_INSTR(OR, NA, NA) +#define CS_ALU_INSTR_XOR __CS_ALU_INSTR(XOR, NA, NA) +#define CS_ALU_INSTR_STORE(op1, op2) __CS_ALU_INSTR(STORE, op1, op2) +#define CS_ALU_INSTR_STOREINV(op1, op2) __CS_ALU_INSTR(STOREINV, op1, op2) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h b/drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h new file mode 100644 index 000000000..dca62af5a --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GFX_STATE_COMMANDS_H_ +#define _XE_GFX_STATE_COMMANDS_H_ + +#include "instructions/xe_instr_defs.h" + +#define GFX_STATE_OPCODE REG_GENMASK(28, 26) + +#define GFX_STATE_CMD(opcode) \ + (XE_INSTR_GFX_STATE | REG_FIELD_PREP(GFX_STATE_OPCODE, opcode)) + +#define STATE_WRITE_INLINE GFX_STATE_CMD(0x0) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h b/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h new file mode 100644 index 000000000..4d854c85e --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GFXPIPE_COMMANDS_H_ +#define _XE_GFXPIPE_COMMANDS_H_ + +#include "instructions/xe_instr_defs.h" + +#define GFXPIPE_PIPELINE REG_GENMASK(28, 27) +#define PIPELINE_COMMON REG_FIELD_PREP(GFXPIPE_PIPELINE, 0x0) +#define PIPELINE_SINGLE_DW REG_FIELD_PREP(GFXPIPE_PIPELINE, 0x1) +#define PIPELINE_COMPUTE REG_FIELD_PREP(GFXPIPE_PIPELINE, 0x2) +#define PIPELINE_3D REG_FIELD_PREP(GFXPIPE_PIPELINE, 0x3) + +#define GFXPIPE_OPCODE REG_GENMASK(26, 24) +#define GFXPIPE_SUBOPCODE REG_GENMASK(23, 16) + +#define GFXPIPE_MATCH_MASK (XE_INSTR_CMD_TYPE | \ + GFXPIPE_PIPELINE | \ + GFXPIPE_OPCODE | \ + GFXPIPE_SUBOPCODE) + +#define GFXPIPE_COMMON_CMD(opcode, subopcode) \ + (XE_INSTR_GFXPIPE | PIPELINE_COMMON | \ + REG_FIELD_PREP(GFXPIPE_OPCODE, opcode) | \ + REG_FIELD_PREP(GFXPIPE_SUBOPCODE, subopcode)) + +#define GFXPIPE_SINGLE_DW_CMD(opcode, subopcode) \ + (XE_INSTR_GFXPIPE | PIPELINE_SINGLE_DW | \ + REG_FIELD_PREP(GFXPIPE_OPCODE, opcode) | \ + REG_FIELD_PREP(GFXPIPE_SUBOPCODE, subopcode)) + +#define GFXPIPE_3D_CMD(opcode, subopcode) \ + (XE_INSTR_GFXPIPE | PIPELINE_3D | \ + REG_FIELD_PREP(GFXPIPE_OPCODE, opcode) | \ + REG_FIELD_PREP(GFXPIPE_SUBOPCODE, subopcode)) + +#define GFXPIPE_COMPUTE_CMD(opcode, subopcode) \ + (XE_INSTR_GFXPIPE | PIPELINE_COMPUTE | \ + REG_FIELD_PREP(GFXPIPE_OPCODE, opcode) | \ + REG_FIELD_PREP(GFXPIPE_SUBOPCODE, subopcode)) + +#define STATE_BASE_ADDRESS GFXPIPE_COMMON_CMD(0x1, 0x1) +#define STATE_SIP GFXPIPE_COMMON_CMD(0x1, 0x2) +#define GPGPU_CSR_BASE_ADDRESS GFXPIPE_COMMON_CMD(0x1, 0x4) +#define STATE_COMPUTE_MODE GFXPIPE_COMMON_CMD(0x1, 0x5) +#define CMD_3DSTATE_BTD GFXPIPE_COMMON_CMD(0x1, 0x6) +#define STATE_SYSTEM_MEM_FENCE_ADDRESS GFXPIPE_COMMON_CMD(0x1, 0x9) +#define STATE_CONTEXT_DATA_BASE_ADDRESS GFXPIPE_COMMON_CMD(0x1, 0xB) + +#define CMD_3DSTATE_VF_STATISTICS GFXPIPE_SINGLE_DW_CMD(0x0, 0xB) + +#define PIPELINE_SELECT GFXPIPE_SINGLE_DW_CMD(0x1, 0x4) + +#define CMD_3DSTATE_DRAWING_RECTANGLE_FAST GFXPIPE_3D_CMD(0x0, 0x0) +#define CMD_3DSTATE_CUSTOM_SAMPLE_PATTERN GFXPIPE_3D_CMD(0x0, 0x2) +#define CMD_3DSTATE_CLEAR_PARAMS GFXPIPE_3D_CMD(0x0, 0x4) +#define CMD_3DSTATE_DEPTH_BUFFER GFXPIPE_3D_CMD(0x0, 0x5) +#define CMD_3DSTATE_STENCIL_BUFFER GFXPIPE_3D_CMD(0x0, 0x6) +#define CMD_3DSTATE_HIER_DEPTH_BUFFER GFXPIPE_3D_CMD(0x0, 0x7) +#define CMD_3DSTATE_VERTEX_BUFFERS GFXPIPE_3D_CMD(0x0, 0x8) +#define CMD_3DSTATE_VERTEX_ELEMENTS GFXPIPE_3D_CMD(0x0, 0x9) +#define CMD_3DSTATE_INDEX_BUFFER GFXPIPE_3D_CMD(0x0, 0xA) +#define CMD_3DSTATE_VF GFXPIPE_3D_CMD(0x0, 0xC) +#define CMD_3DSTATE_MULTISAMPLE GFXPIPE_3D_CMD(0x0, 0xD) +#define CMD_3DSTATE_CC_STATE_POINTERS GFXPIPE_3D_CMD(0x0, 0xE) +#define CMD_3DSTATE_SCISSOR_STATE_POINTERS GFXPIPE_3D_CMD(0x0, 0xF) +#define CMD_3DSTATE_VS GFXPIPE_3D_CMD(0x0, 0x10) +#define CMD_3DSTATE_GS GFXPIPE_3D_CMD(0x0, 0x11) +#define CMD_3DSTATE_CLIP GFXPIPE_3D_CMD(0x0, 0x12) +#define CMD_3DSTATE_SF GFXPIPE_3D_CMD(0x0, 0x13) +#define CMD_3DSTATE_WM GFXPIPE_3D_CMD(0x0, 0x14) +#define CMD_3DSTATE_CONSTANT_VS GFXPIPE_3D_CMD(0x0, 0x15) +#define CMD_3DSTATE_CONSTANT_GS GFXPIPE_3D_CMD(0x0, 0x16) +#define CMD_3DSTATE_CONSTANT_PS GFXPIPE_3D_CMD(0x0, 0x17) +#define CMD_3DSTATE_SAMPLE_MASK GFXPIPE_3D_CMD(0x0, 0x18) +#define CMD_3DSTATE_CONSTANT_HS GFXPIPE_3D_CMD(0x0, 0x19) +#define CMD_3DSTATE_CONSTANT_DS GFXPIPE_3D_CMD(0x0, 0x1A) +#define CMD_3DSTATE_HS GFXPIPE_3D_CMD(0x0, 0x1B) +#define CMD_3DSTATE_TE GFXPIPE_3D_CMD(0x0, 0x1C) +#define CMD_3DSTATE_DS GFXPIPE_3D_CMD(0x0, 0x1D) +#define CMD_3DSTATE_STREAMOUT GFXPIPE_3D_CMD(0x0, 0x1E) +#define CMD_3DSTATE_SBE GFXPIPE_3D_CMD(0x0, 0x1F) +#define CMD_3DSTATE_PS GFXPIPE_3D_CMD(0x0, 0x20) +#define CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP GFXPIPE_3D_CMD(0x0, 0x21) +#define CMD_3DSTATE_CPS_POINTERS GFXPIPE_3D_CMD(0x0, 0x22) +#define CMD_3DSTATE_VIEWPORT_STATE_POINTERS_CC GFXPIPE_3D_CMD(0x0, 0x23) +#define CMD_3DSTATE_BLEND_STATE_POINTERS GFXPIPE_3D_CMD(0x0, 0x24) +#define CMD_3DSTATE_BINDING_TABLE_POINTERS_VS GFXPIPE_3D_CMD(0x0, 0x26) +#define CMD_3DSTATE_BINDING_TABLE_POINTERS_HS GFXPIPE_3D_CMD(0x0, 0x27) +#define CMD_3DSTATE_BINDING_TABLE_POINTERS_DS GFXPIPE_3D_CMD(0x0, 0x28) +#define CMD_3DSTATE_BINDING_TABLE_POINTERS_GS GFXPIPE_3D_CMD(0x0, 0x29) +#define CMD_3DSTATE_BINDING_TABLE_POINTERS_PS GFXPIPE_3D_CMD(0x0, 0x2A) +#define CMD_3DSTATE_SAMPLER_STATE_POINTERS_VS GFXPIPE_3D_CMD(0x0, 0x2B) +#define CMD_3DSTATE_SAMPLER_STATE_POINTERS_HS GFXPIPE_3D_CMD(0x0, 0x2C) +#define CMD_3DSTATE_SAMPLER_STATE_POINTERS_DS GFXPIPE_3D_CMD(0x0, 0x2D) +#define CMD_3DSTATE_SAMPLER_STATE_POINTERS_GS GFXPIPE_3D_CMD(0x0, 0x2E) +#define CMD_3DSTATE_SAMPLER_STATE_POINTERS_PS GFXPIPE_3D_CMD(0x0, 0x2F) +#define CMD_3DSTATE_VF_INSTANCING GFXPIPE_3D_CMD(0x0, 0x49) +#define CMD_3DSTATE_VF_SGVS GFXPIPE_3D_CMD(0x0, 0x4A) +#define CMD_3DSTATE_VF_TOPOLOGY GFXPIPE_3D_CMD(0x0, 0x4B) +#define CMD_3DSTATE_WM_CHROMAKEY GFXPIPE_3D_CMD(0x0, 0x4C) +#define CMD_3DSTATE_PS_BLEND GFXPIPE_3D_CMD(0x0, 0x4D) +#define CMD_3DSTATE_WM_DEPTH_STENCIL GFXPIPE_3D_CMD(0x0, 0x4E) +#define CMD_3DSTATE_PS_EXTRA GFXPIPE_3D_CMD(0x0, 0x4F) +#define CMD_3DSTATE_RASTER GFXPIPE_3D_CMD(0x0, 0x50) +#define CMD_3DSTATE_SBE_SWIZ GFXPIPE_3D_CMD(0x0, 0x51) +#define CMD_3DSTATE_WM_HZ_OP GFXPIPE_3D_CMD(0x0, 0x52) +#define CMD_3DSTATE_VF_COMPONENT_PACKING GFXPIPE_3D_CMD(0x0, 0x55) +#define CMD_3DSTATE_VF_SGVS_2 GFXPIPE_3D_CMD(0x0, 0x56) +#define CMD_3DSTATE_VFG GFXPIPE_3D_CMD(0x0, 0x57) +#define CMD_3DSTATE_URB_ALLOC_VS GFXPIPE_3D_CMD(0x0, 0x58) +#define CMD_3DSTATE_URB_ALLOC_HS GFXPIPE_3D_CMD(0x0, 0x59) +#define CMD_3DSTATE_URB_ALLOC_DS GFXPIPE_3D_CMD(0x0, 0x5A) +#define CMD_3DSTATE_URB_ALLOC_GS GFXPIPE_3D_CMD(0x0, 0x5B) +#define CMD_3DSTATE_SO_BUFFER_INDEX_0 GFXPIPE_3D_CMD(0x0, 0x60) +#define CMD_3DSTATE_SO_BUFFER_INDEX_1 GFXPIPE_3D_CMD(0x0, 0x61) +#define CMD_3DSTATE_SO_BUFFER_INDEX_2 GFXPIPE_3D_CMD(0x0, 0x62) +#define CMD_3DSTATE_SO_BUFFER_INDEX_3 GFXPIPE_3D_CMD(0x0, 0x63) +#define CMD_3DSTATE_PRIMITIVE_REPLICATION GFXPIPE_3D_CMD(0x0, 0x6C) +#define CMD_3DSTATE_TBIMR_TILE_PASS_INFO GFXPIPE_3D_CMD(0x0, 0x6E) +#define CMD_3DSTATE_AMFS GFXPIPE_3D_CMD(0x0, 0x6F) +#define CMD_3DSTATE_DEPTH_BOUNDS GFXPIPE_3D_CMD(0x0, 0x71) +#define CMD_3DSTATE_AMFS_TEXTURE_POINTERS GFXPIPE_3D_CMD(0x0, 0x72) +#define CMD_3DSTATE_CONSTANT_TS_POINTER GFXPIPE_3D_CMD(0x0, 0x73) +#define CMD_3DSTATE_MESH_CONTROL GFXPIPE_3D_CMD(0x0, 0x77) +#define CMD_3DSTATE_MESH_DISTRIB GFXPIPE_3D_CMD(0x0, 0x78) +#define CMD_3DSTATE_TASK_REDISTRIB GFXPIPE_3D_CMD(0x0, 0x79) +#define CMD_3DSTATE_MESH_SHADER GFXPIPE_3D_CMD(0x0, 0x7A) +#define CMD_3DSTATE_MESH_SHADER_DATA GFXPIPE_3D_CMD(0x0, 0x7B) +#define CMD_3DSTATE_TASK_CONTROL GFXPIPE_3D_CMD(0x0, 0x7C) +#define CMD_3DSTATE_TASK_SHADER GFXPIPE_3D_CMD(0x0, 0x7D) +#define CMD_3DSTATE_TASK_SHADER_DATA GFXPIPE_3D_CMD(0x0, 0x7E) +#define CMD_3DSTATE_URB_ALLOC_MESH GFXPIPE_3D_CMD(0x0, 0x7F) +#define CMD_3DSTATE_URB_ALLOC_TASK GFXPIPE_3D_CMD(0x0, 0x80) +#define CMD_3DSTATE_CLIP_MESH GFXPIPE_3D_CMD(0x0, 0x81) +#define CMD_3DSTATE_SBE_MESH GFXPIPE_3D_CMD(0x0, 0x82) +#define CMD_3DSTATE_CPSIZE_CONTROL_BUFFER GFXPIPE_3D_CMD(0x0, 0x83) +#define CMD_3DSTATE_COARSE_PIXEL GFXPIPE_3D_CMD(0x0, 0x89) +#define CMD_3DSTATE_MESH_SHADER_DATA_EXT GFXPIPE_3D_CMD(0x0, 0x8A) +#define CMD_3DSTATE_TASK_SHADER_DATA_EXT GFXPIPE_3D_CMD(0x0, 0x8B) +#define CMD_3DSTATE_VIEWPORT_STATE_POINTERS_CC_2 GFXPIPE_3D_CMD(0x0, 0x8D) +#define CMD_3DSTATE_CC_STATE_POINTERS_2 GFXPIPE_3D_CMD(0x0, 0x8E) +#define CMD_3DSTATE_SCISSOR_STATE_POINTERS_2 GFXPIPE_3D_CMD(0x0, 0x8F) +#define CMD_3DSTATE_BLEND_STATE_POINTERS_2 GFXPIPE_3D_CMD(0x0, 0xA0) +#define CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2 GFXPIPE_3D_CMD(0x0, 0xA1) + +#define CMD_3DSTATE_DRAWING_RECTANGLE GFXPIPE_3D_CMD(0x1, 0x0) +#define CMD_3DSTATE_URB_MEMORY GFXPIPE_3D_CMD(0x1, 0x1) +#define CMD_3DSTATE_CHROMA_KEY GFXPIPE_3D_CMD(0x1, 0x4) +#define CMD_3DSTATE_POLY_STIPPLE_OFFSET GFXPIPE_3D_CMD(0x1, 0x6) +#define CMD_3DSTATE_POLY_STIPPLE_PATTERN GFXPIPE_3D_CMD(0x1, 0x7) +#define CMD_3DSTATE_LINE_STIPPLE GFXPIPE_3D_CMD(0x1, 0x8) +#define CMD_3DSTATE_AA_LINE_PARAMETERS GFXPIPE_3D_CMD(0x1, 0xA) +#define CMD_3DSTATE_MONOFILTER_SIZE GFXPIPE_3D_CMD(0x1, 0x11) +#define CMD_3DSTATE_PUSH_CONSTANT_ALLOC_VS GFXPIPE_3D_CMD(0x1, 0x12) +#define CMD_3DSTATE_PUSH_CONSTANT_ALLOC_HS GFXPIPE_3D_CMD(0x1, 0x13) +#define CMD_3DSTATE_PUSH_CONSTANT_ALLOC_DS GFXPIPE_3D_CMD(0x1, 0x14) +#define CMD_3DSTATE_PUSH_CONSTANT_ALLOC_GS GFXPIPE_3D_CMD(0x1, 0x15) +#define CMD_3DSTATE_PUSH_CONSTANT_ALLOC_PS GFXPIPE_3D_CMD(0x1, 0x16) +#define CMD_3DSTATE_SO_DECL_LIST GFXPIPE_3D_CMD(0x1, 0x17) +#define CMD_3DSTATE_SO_DECL_LIST_DW_LEN REG_GENMASK(8, 0) +#define CMD_3DSTATE_SO_BUFFER GFXPIPE_3D_CMD(0x1, 0x18) +#define CMD_3DSTATE_BINDING_TABLE_POOL_ALLOC GFXPIPE_3D_CMD(0x1, 0x19) +#define CMD_3DSTATE_SAMPLE_PATTERN GFXPIPE_3D_CMD(0x1, 0x1C) +#define CMD_3DSTATE_3D_MODE GFXPIPE_3D_CMD(0x1, 0x1E) +#define CMD_3DSTATE_SUBSLICE_HASH_TABLE GFXPIPE_3D_CMD(0x1, 0x1F) +#define CMD_3DSTATE_SLICE_TABLE_STATE_POINTERS GFXPIPE_3D_CMD(0x1, 0x20) +#define CMD_3DSTATE_PTBR_TILE_PASS_INFO GFXPIPE_3D_CMD(0x1, 0x22) +#define CMD_3DSTATE_SLICE_TABLE_STATE_POINTER_2 GFXPIPE_3D_CMD(0x1, 0xA0) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h new file mode 100644 index 000000000..18d0fde8c --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GPU_COMMANDS_H_ +#define _XE_GPU_COMMANDS_H_ + +#include "regs/xe_reg_defs.h" + +#define XY_CTRL_SURF_COPY_BLT ((2 << 29) | (0x48 << 22) | 3) +#define SRC_ACCESS_TYPE_SHIFT 21 +#define DST_ACCESS_TYPE_SHIFT 20 +#define CCS_SIZE_MASK GENMASK(17, 8) +#define XE2_CCS_SIZE_MASK GENMASK(18, 9) +#define XY_CTRL_SURF_MOCS_MASK GENMASK(31, 26) +#define XE2_XY_CTRL_SURF_MOCS_INDEX_MASK GENMASK(31, 28) +#define NUM_CCS_BYTES_PER_BLOCK 256 +#define NUM_BYTES_PER_CCS_BYTE(_xe) (GRAPHICS_VER(_xe) >= 20 ? 512 : 256) + +#define XY_FAST_COLOR_BLT_CMD (2 << 29 | 0x44 << 22) +#define XY_FAST_COLOR_BLT_DEPTH_32 (2 << 19) +#define XY_FAST_COLOR_BLT_MOCS_MASK GENMASK(27, 22) +#define XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK GENMASK(27, 24) +#define XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT 31 + +#define XY_FAST_COPY_BLT_CMD (2 << 29 | 0x42 << 22) +#define XY_FAST_COPY_BLT_DEPTH_32 (3<<24) +#define XY_FAST_COPY_BLT_D1_SRC_TILE4 REG_BIT(31) +#define XY_FAST_COPY_BLT_D1_DST_TILE4 REG_BIT(30) +#define XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK GENMASK(23, 20) + +#define MEM_COPY_CMD (2 << 29 | 0x5a << 22) +#define MEM_COPY_PAGE_COPY_MODE REG_BIT(19) +#define MEM_COPY_MATRIX_COPY REG_BIT(17) +#define MEM_COPY_SRC_MOCS_INDEX_MASK GENMASK(31, 28) +#define MEM_COPY_DST_MOCS_INDEX_MASK GENMASK(6, 3) + +#define PVC_MEM_SET_CMD (2 << 29 | 0x5b << 22) +#define PVC_MEM_SET_MATRIX REG_BIT(17) +#define PVC_MEM_SET_DATA_FIELD GENMASK(31, 24) +/* Bspec lists field as [6:0], but index alone is from [6:1] */ +#define PVC_MEM_SET_MOCS_INDEX_MASK GENMASK(6, 1) +#define XE2_MEM_SET_MOCS_INDEX_MASK GENMASK(6, 3) + +#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2)) + +#define PIPE_CONTROL0_QUEUE_DRAIN_MODE BIT(12) +#define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */ +#define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */ + +#define PIPE_CONTROL_COMMAND_CACHE_INVALIDATE (1<<29) +#define PIPE_CONTROL_TILE_CACHE_FLUSH (1<<28) +#define PIPE_CONTROL_AMFS_FLUSH (1<<25) +#define PIPE_CONTROL_GLOBAL_GTT_IVB (1<<24) +#define PIPE_CONTROL_LRI_POST_SYNC BIT(23) +#define PIPE_CONTROL_STORE_DATA_INDEX (1<<21) +#define PIPE_CONTROL_CS_STALL (1<<20) +#define PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET (1<<19) +#define PIPE_CONTROL_TLB_INVALIDATE BIT(18) +#define PIPE_CONTROL_PSD_SYNC (1<<17) +#define PIPE_CONTROL_QW_WRITE (1<<14) +#define PIPE_CONTROL_DEPTH_STALL (1<<13) +#define PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH (1<<12) +#define PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE (1<<11) +#define PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE (1<<10) +#define PIPE_CONTROL_INDIRECT_STATE_DISABLE (1<<9) +#define PIPE_CONTROL_FLUSH_ENABLE (1<<7) +#define PIPE_CONTROL_DC_FLUSH_ENABLE (1<<5) +#define PIPE_CONTROL_VF_CACHE_INVALIDATE (1<<4) +#define PIPE_CONTROL_CONST_CACHE_INVALIDATE (1<<3) +#define PIPE_CONTROL_STATE_CACHE_INVALIDATE (1<<2) +#define PIPE_CONTROL_STALL_AT_SCOREBOARD (1<<1) +#define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1<<0) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_gsc_commands.h b/drivers/gpu/drm/xe/instructions/xe_gsc_commands.h new file mode 100644 index 000000000..f8949cad9 --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_gsc_commands.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GSC_COMMANDS_H_ +#define _XE_GSC_COMMANDS_H_ + +#include "instructions/xe_instr_defs.h" + +/* + * All GSCCS-specific commands have fixed length, so we can include it in the + * defines. Note that the generic GSC command header structure includes an + * optional data field in bits 9-21, but there are no commands that actually use + * it; some of the commands are instead defined as having an extended length + * field spanning bits 0-15, even if the extra bits are not required because the + * longest GSCCS command is only 8 dwords. To handle this, the defines below use + * a single field for both data and len. If we ever get a commands that does + * actually have data and this approach doesn't work for it we can re-work it + * at that point. + */ + +#define GSC_OPCODE REG_GENMASK(28, 22) +#define GSC_CMD_DATA_AND_LEN REG_GENMASK(21, 0) + +#define __GSC_INSTR(op, dl) \ + (XE_INSTR_GSC | \ + REG_FIELD_PREP(GSC_OPCODE, op) | \ + REG_FIELD_PREP(GSC_CMD_DATA_AND_LEN, dl)) + +#define GSC_HECI_CMD_PKT __GSC_INSTR(0, 6) + +#define GSC_FW_LOAD __GSC_INSTR(1, 2) +#define GSC_FW_LOAD_LIMIT_VALID REG_BIT(31) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_instr_defs.h b/drivers/gpu/drm/xe/instructions/xe_instr_defs.h new file mode 100644 index 000000000..e55996946 --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_instr_defs.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_INSTR_DEFS_H_ +#define _XE_INSTR_DEFS_H_ + +#include "regs/xe_reg_defs.h" + +/* + * The first dword of any GPU instruction is the "instruction header." Bits + * 31:29 identify the general type of the command and determine how exact + * opcodes and sub-opcodes will be encoded in the remaining bits. + */ +#define XE_INSTR_CMD_TYPE GENMASK(31, 29) +#define XE_INSTR_MI REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x0) +#define XE_INSTR_GSC REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x2) +#define XE_INSTR_VIDEOPIPE REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x3) +#define XE_INSTR_GFXPIPE REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x3) +#define XE_INSTR_GFX_STATE REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x4) + +/* + * Most (but not all) instructions have a "length" field in the instruction + * header. The value expected is the total number of dwords for the + * instruction, minus two. + * + * Some instructions have length fields longer or shorter than 8 bits, but + * those are rare. This definition can be used for the common case where + * the length field is from 7:0. + */ +#define XE_INSTR_LEN_MASK GENMASK(7, 0) +#define XE_INSTR_NUM_DW(x) REG_FIELD_PREP(XE_INSTR_LEN_MASK, (x) - 2) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_mfx_commands.h b/drivers/gpu/drm/xe/instructions/xe_mfx_commands.h new file mode 100644 index 000000000..3c0c97f78 --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_mfx_commands.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_MFX_COMMANDS_H_ +#define _XE_MFX_COMMANDS_H_ + +#include "instructions/xe_instr_defs.h" + +#define MFX_CMD_SUBTYPE REG_GENMASK(28, 27) /* A.K.A cmd pipe */ +#define MFX_CMD_OPCODE REG_GENMASK(26, 24) +#define MFX_CMD_SUB_OPCODE REG_GENMASK(23, 16) +#define MFX_FLAGS_AND_LEN REG_GENMASK(15, 0) + +#define XE_MFX_INSTR(subtype, op, sub_op) \ + (XE_INSTR_VIDEOPIPE | \ + REG_FIELD_PREP(MFX_CMD_SUBTYPE, subtype) | \ + REG_FIELD_PREP(MFX_CMD_OPCODE, op) | \ + REG_FIELD_PREP(MFX_CMD_SUB_OPCODE, sub_op)) + +#define MFX_WAIT XE_MFX_INSTR(1, 0, 0) +#define MFX_WAIT_DW0_PXP_SYNC_CONTROL_FLAG REG_BIT(9) +#define MFX_WAIT_DW0_MFX_SYNC_CONTROL_FLAG REG_BIT(8) + +#define CRYPTO_KEY_EXCHANGE XE_MFX_INSTR(2, 6, 9) + +#endif diff --git a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h new file mode 100644 index 000000000..ad7d98f2d --- /dev/null +++ b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_MI_COMMANDS_H_ +#define _XE_MI_COMMANDS_H_ + +#include "instructions/xe_instr_defs.h" + +/* + * MI (Memory Interface) commands are supported by all GT engines. They + * provide general memory operations and command streamer control. MI commands + * have a command type of 0x0 (MI_COMMAND) in bits 31:29 of the instruction + * header dword and a specific MI opcode in bits 28:23. + */ + +#define MI_OPCODE REG_GENMASK(28, 23) +#define MI_SUBOPCODE REG_GENMASK(22, 17) /* used with MI_EXPANSION */ + +#define __MI_INSTR(opcode) \ + (XE_INSTR_MI | REG_FIELD_PREP(MI_OPCODE, opcode)) + +#define MI_NOOP __MI_INSTR(0x0) +#define MI_USER_INTERRUPT __MI_INSTR(0x2) +#define MI_ARB_CHECK __MI_INSTR(0x5) + +#define MI_ARB_ON_OFF __MI_INSTR(0x8) +#define MI_ARB_ENABLE REG_BIT(0) +#define MI_ARB_DISABLE 0x0 + +#define MI_BATCH_BUFFER_END __MI_INSTR(0xA) +#define MI_TOPOLOGY_FILTER __MI_INSTR(0xD) +#define MI_FORCE_WAKEUP __MI_INSTR(0x1D) +#define MI_MATH(n) (__MI_INSTR(0x1A) | XE_INSTR_NUM_DW((n) + 1)) + +#define MI_SEMAPHORE_WAIT (__MI_INSTR(0x1c) | XE_INSTR_NUM_DW(5)) +#define MI_SEMW_GGTT REG_BIT(22) +#define MI_SEMW_POLL REG_BIT(15) +#define MI_SEMW_COMPARE_OP_MASK REG_GENMASK(14, 12) +#define COMPARE_OP_SAD_GT_SDD 0 +#define COMPARE_OP_SAD_GTE_SDD 1 +#define COMPARE_OP_SAD_LT_SDD 2 +#define COMPARE_OP_SAD_LTE_SDD 3 +#define COMPARE_OP_SAD_EQ_SDD 4 +#define COMPARE_OP_SAD_NEQ_SDD 5 +#define MI_SEMW_COMPARE(OP) REG_FIELD_PREP(MI_SEMW_COMPARE_OP_MASK, COMPARE_OP_##OP) +#define MI_SEMW_TOKEN(token) REG_FIELD_PREP(REG_GENMASK(9, 2), (token)) + +#define MI_STORE_DATA_IMM __MI_INSTR(0x20) +#define MI_SDI_GGTT REG_BIT(22) +#define MI_SDI_LEN_DW GENMASK(9, 0) +#define MI_SDI_NUM_DW(x) REG_FIELD_PREP(MI_SDI_LEN_DW, (x) + 3 - 2) +#define MI_SDI_NUM_QW(x) (REG_FIELD_PREP(MI_SDI_LEN_DW, 2 * (x) + 3 - 2) | \ + REG_BIT(21)) + +#define MI_LOAD_REGISTER_IMM __MI_INSTR(0x22) +#define MI_LRI_LRM_CS_MMIO REG_BIT(19) +#define MI_LRI_MMIO_REMAP_EN REG_BIT(17) +#define MI_LRI_NUM_REGS(x) XE_INSTR_NUM_DW(2 * (x) + 1) +#define MI_LRI_FORCE_POSTED REG_BIT(12) +#define MI_LRI_LEN(x) (((x) & 0xff) + 1) + +#define MI_STORE_REGISTER_MEM (__MI_INSTR(0x24) | XE_INSTR_NUM_DW(4)) +#define MI_SRM_USE_GGTT REG_BIT(22) +#define MI_SRM_ADD_CS_OFFSET REG_BIT(19) + +#define MI_FLUSH_DW __MI_INSTR(0x26) +#define MI_FLUSH_DW_PROTECTED_MEM_EN REG_BIT(22) +#define MI_FLUSH_DW_STORE_INDEX REG_BIT(21) +#define MI_INVALIDATE_TLB REG_BIT(18) +#define MI_FLUSH_DW_CCS REG_BIT(16) +#define MI_FLUSH_DW_OP_STOREDW REG_BIT(14) +#define MI_FLUSH_DW_LEN_DW REG_GENMASK(5, 0) +#define MI_FLUSH_IMM_DW REG_FIELD_PREP(MI_FLUSH_DW_LEN_DW, 4 - 2) +#define MI_FLUSH_IMM_QW REG_FIELD_PREP(MI_FLUSH_DW_LEN_DW, 5 - 2) +#define MI_FLUSH_DW_USE_GTT REG_BIT(2) + +#define MI_LOAD_REGISTER_MEM (__MI_INSTR(0x29) | XE_INSTR_NUM_DW(4)) +#define MI_LRM_USE_GGTT REG_BIT(22) +#define MI_LRM_ASYNC REG_BIT(21) + +#define MI_LOAD_REGISTER_REG (__MI_INSTR(0x2a) | XE_INSTR_NUM_DW(3)) +#define MI_LRR_DST_CS_MMIO REG_BIT(19) +#define MI_LRR_SRC_CS_MMIO REG_BIT(18) + +#define MI_COPY_MEM_MEM (__MI_INSTR(0x2e) | XE_INSTR_NUM_DW(5)) +#define MI_COPY_MEM_MEM_SRC_GGTT REG_BIT(22) +#define MI_COPY_MEM_MEM_DST_GGTT REG_BIT(21) + +#define MI_BATCH_BUFFER_START __MI_INSTR(0x31) + +#define MI_SET_APPID __MI_INSTR(0x0e) +#define MI_SET_APPID_SESSION_ID_MASK REG_GENMASK(6, 0) +#define MI_SET_APPID_SESSION_ID(x) REG_FIELD_PREP(MI_SET_APPID_SESSION_ID_MASK, x) + +#define MI_SEMAPHORE_WAIT_TOKEN (__MI_INSTR(0x1c) | XE_INSTR_NUM_DW(5)) /* XeLP+ */ +#define MI_SEMAPHORE_REGISTER_POLL REG_BIT(16) +#define MI_SEMAPHORE_POLL REG_BIT(15) +#define MI_SEMAPHORE_CMP_OP_MASK REG_GENMASK(14, 12) +#define MI_SEMAPHORE_SAD_EQ_SDD REG_FIELD_PREP(MI_SEMAPHORE_CMP_OP_MASK, 4) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_bars.h b/drivers/gpu/drm/xe/regs/xe_bars.h new file mode 100644 index 000000000..880140d6c --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_bars.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ +#ifndef _XE_BARS_H_ +#define _XE_BARS_H_ + +#define GTTMMADR_BAR 0 /* MMIO + GTT */ +#define LMEM_BAR 2 /* VRAM */ +#define VF_LMEM_BAR 9 /* VF VRAM */ + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h new file mode 100644 index 000000000..94033982e --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -0,0 +1,237 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_ENGINE_REGS_H_ +#define _XE_ENGINE_REGS_H_ + +#include + +#include "regs/xe_reg_defs.h" + +/* + * These *_BASE values represent the MMIO offset where each hardware engine's + * registers start. The other definitions in this header are parameterized + * macros that will take one of these values as a parameter. + */ +#define RENDER_RING_BASE 0x02000 +#define BSD_RING_BASE 0x1c0000 +#define BSD2_RING_BASE 0x1c4000 +#define BSD3_RING_BASE 0x1d0000 +#define BSD4_RING_BASE 0x1d4000 +#define XEHP_BSD5_RING_BASE 0x1e0000 +#define XEHP_BSD6_RING_BASE 0x1e4000 +#define XEHP_BSD7_RING_BASE 0x1f0000 +#define XEHP_BSD8_RING_BASE 0x1f4000 +#define VEBOX_RING_BASE 0x1c8000 +#define VEBOX2_RING_BASE 0x1d8000 +#define XEHP_VEBOX3_RING_BASE 0x1e8000 +#define XEHP_VEBOX4_RING_BASE 0x1f8000 +#define COMPUTE0_RING_BASE 0x1a000 +#define COMPUTE1_RING_BASE 0x1c000 +#define COMPUTE2_RING_BASE 0x1e000 +#define COMPUTE3_RING_BASE 0x26000 +#define BLT_RING_BASE 0x22000 +#define XEHPC_BCS1_RING_BASE 0x3e0000 +#define XEHPC_BCS2_RING_BASE 0x3e2000 +#define XEHPC_BCS3_RING_BASE 0x3e4000 +#define XEHPC_BCS4_RING_BASE 0x3e6000 +#define XEHPC_BCS5_RING_BASE 0x3e8000 +#define XEHPC_BCS6_RING_BASE 0x3ea000 +#define XEHPC_BCS7_RING_BASE 0x3ec000 +#define XEHPC_BCS8_RING_BASE 0x3ee000 +#define GSCCS_RING_BASE 0x11a000 + +#define ENGINE_ID(base) XE_REG((base) + 0x8c) +#define ENGINE_INSTANCE_ID REG_GENMASK(9, 4) +#define ENGINE_CLASS_ID REG_GENMASK(2, 0) + +#define RING_TAIL(base) XE_REG((base) + 0x30) +#define TAIL_ADDR REG_GENMASK(20, 3) + +#define RING_HEAD(base) XE_REG((base) + 0x34) +#define HEAD_ADDR REG_GENMASK(20, 2) + +#define RING_START(base) XE_REG((base) + 0x38) + +#define RING_CTL(base) XE_REG((base) + 0x3c) +#define RING_CTL_SIZE(size) ((size) - PAGE_SIZE) /* in bytes -> pages */ + +#define RING_START_UDW(base) XE_REG((base) + 0x48) + +#define RING_PSMI_CTL(base) XE_REG((base) + 0x50, XE_REG_OPTION_MASKED) +#define RC_SEMA_IDLE_MSG_DISABLE REG_BIT(12) +#define WAIT_FOR_EVENT_POWER_DOWN_DISABLE REG_BIT(7) +#define IDLE_MSG_DISABLE REG_BIT(0) + +#define RING_PWRCTX_MAXCNT(base) XE_REG((base) + 0x54) +#define IDLE_WAIT_TIME REG_GENMASK(19, 0) + +#define RING_ACTHD_UDW(base) XE_REG((base) + 0x5c) +#define RING_DMA_FADD_UDW(base) XE_REG((base) + 0x60) +#define RING_IPEHR(base) XE_REG((base) + 0x68) +#define RING_INSTDONE(base) XE_REG((base) + 0x6c) +#define RING_ACTHD(base) XE_REG((base) + 0x74) +#define RING_DMA_FADD(base) XE_REG((base) + 0x78) +#define RING_HWS_PGA(base) XE_REG((base) + 0x80) +#define RING_HWSTAM(base) XE_REG((base) + 0x98) +#define RING_MI_MODE(base) XE_REG((base) + 0x9c) +#define RING_NOPID(base) XE_REG((base) + 0x94) + +#define FF_THREAD_MODE(base) XE_REG((base) + 0xa0) +#define FF_TESSELATION_DOP_GATE_DISABLE BIT(19) + +#define RING_INT_SRC_RPT_PTR(base) XE_REG((base) + 0xa4) +#define RING_IMR(base) XE_REG((base) + 0xa8) +#define RING_INT_STATUS_RPT_PTR(base) XE_REG((base) + 0xac) + +#define CS_INT_VEC(base) XE_REG((base) + 0x1b8) + +#define RING_EIR(base) XE_REG((base) + 0xb0) +#define RING_EMR(base) XE_REG((base) + 0xb4) +#define RING_ESR(base) XE_REG((base) + 0xb8) + +#define INSTPM(base) XE_REG((base) + 0xc0, XE_REG_OPTION_MASKED) +#define ENABLE_SEMAPHORE_POLL_BIT REG_BIT(13) + +#define RING_CMD_CCTL(base) XE_REG((base) + 0xc4, XE_REG_OPTION_MASKED) + +#define CS_MMIO_GROUP_INSTANCE_SELECT(base) XE_REG((base) + 0xcc) +#define SELECTIVE_READ_ADDRESSING REG_BIT(30) +#define SELECTIVE_READ_GROUP REG_GENMASK(29, 23) +#define SELECTIVE_READ_INSTANCE REG_GENMASK(22, 16) + +/* + * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. + * The lsb of each can be considered a separate enabling bit for encryption. + * 6:0 == default MOCS value for reads => 6:1 == table index for reads. + * 13:7 == default MOCS value for writes => 13:8 == table index for writes. + * 15:14 == Reserved => 31:30 are set to 0. + */ +#define CMD_CCTL_WRITE_OVERRIDE_MASK REG_GENMASK(13, 8) +#define CMD_CCTL_READ_OVERRIDE_MASK REG_GENMASK(6, 1) + +#define CSFE_CHICKEN1(base) XE_REG((base) + 0xd4, XE_REG_OPTION_MASKED) +#define GHWSP_CSB_REPORT_DIS REG_BIT(15) +#define PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS REG_BIT(14) +#define CS_PRIORITY_MEM_READ REG_BIT(7) + +#define CS_DEBUG_MODE2(base) XE_REG((base) + 0xd8, XE_REG_OPTION_MASKED) +#define INSTRUCTION_STATE_CACHE_INVALIDATE REG_BIT(6) + +#define FF_SLICE_CS_CHICKEN1(base) XE_REG((base) + 0xe0, XE_REG_OPTION_MASKED) +#define FFSC_PERCTX_PREEMPT_CTRL REG_BIT(14) + +#define CS_DEBUG_MODE1(base) XE_REG((base) + 0xec, XE_REG_OPTION_MASKED) +#define FF_DOP_CLOCK_GATE_DISABLE REG_BIT(1) +#define REPLAY_MODE_GRANULARITY REG_BIT(0) + +#define INDIRECT_RING_STATE(base) XE_REG((base) + 0x108) + +#define RING_BBADDR(base) XE_REG((base) + 0x140) +#define RING_BBADDR_UDW(base) XE_REG((base) + 0x168) + +#define PR_CTR_CTRL(base) XE_REG((base) + 0x178) +#define CTR_COUNT_SELECT_FF REG_BIT(31) +#define CTR_LOGIC_OP_MASK REG_GENMASK(30, 0) +#define CTR_START 0 +#define CTR_STOP 1 +#define CTR_LOGIC_OP(OP) REG_FIELD_PREP(CTR_LOGIC_OP_MASK, CTR_##OP) +#define PR_CTR_THRSH(base) XE_REG((base) + 0x17c) + +#define BCS_SWCTRL(base) XE_REG((base) + 0x200, XE_REG_OPTION_MASKED) +#define BCS_SWCTRL_DISABLE_256B REG_BIT(2) + +/* Handling MOCS value in BLIT_CCTL like it was done CMD_CCTL */ +#define BLIT_CCTL(base) XE_REG((base) + 0x204) +#define BLIT_CCTL_DST_MOCS_MASK REG_GENMASK(14, 9) +#define BLIT_CCTL_SRC_MOCS_MASK REG_GENMASK(6, 1) + +#define RING_EXECLIST_STATUS_LO(base) XE_REG((base) + 0x234) +#define RING_EXECLIST_STATUS_HI(base) XE_REG((base) + 0x234 + 4) + +#define RING_IDLEDLY(base) XE_REG((base) + 0x23c) +#define INHIBIT_SWITCH_UNTIL_PREEMPTED REG_BIT(31) +#define IDLE_DELAY REG_GENMASK(20, 0) + +#define RING_CURRENT_LRCA(base) XE_REG((base) + 0x240) + +#define RING_CONTEXT_CONTROL(base) XE_REG((base) + 0x244, XE_REG_OPTION_MASKED) +#define CTX_CTRL_PXP_ENABLE REG_BIT(10) +#define CTX_CTRL_OAC_CONTEXT_ENABLE REG_BIT(8) +#define CTX_CTRL_RUN_ALONE REG_BIT(7) +#define CTX_CTRL_INDIRECT_RING_STATE_ENABLE REG_BIT(4) +#define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH REG_BIT(3) +#define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT REG_BIT(0) + +#define GFX_MODE(base) XE_REG((base) + 0x29c, XE_REG_OPTION_MASKED) +#define GFX_MSIX_INTERRUPT_ENABLE REG_BIT(13) +#define GFX_DISABLE_LEGACY_MODE REG_BIT(3) + +#define RING_CSMQDEBUG(base) XE_REG((base) + 0x2b0) +#define CURRENT_ACTIVE_QUEUE_ID_MASK REG_GENMASK(7, 0) + +#define RING_QUEUE_TIMESTAMP(base) XE_REG((base) + 0x4c0) +#define RING_QUEUE_TIMESTAMP_UDW(base) XE_REG((base) + 0x4c0 + 4) + +#define RING_TIMESTAMP(base) XE_REG((base) + 0x358) + +#define RING_TIMESTAMP_UDW(base) XE_REG((base) + 0x358 + 4) +#define RING_VALID_MASK 0x00000001 +#define RING_VALID 0x00000001 + +#define RING_CTX_TIMESTAMP(base) XE_REG((base) + 0x3a8) +#define RING_CTX_TIMESTAMP_UDW(base) XE_REG((base) + 0x3ac) +#define CSBE_DEBUG_STATUS(base) XE_REG((base) + 0x3fc) + +#define RING_FORCE_TO_NONPRIV(base, i) XE_REG(((base) + 0x4d0) + (i) * 4) +#define RING_FORCE_TO_NONPRIV_DENY REG_BIT(30) +#define RING_FORCE_TO_NONPRIV_ACCESS_MASK REG_GENMASK(29, 28) +#define RING_FORCE_TO_NONPRIV_ACCESS_RW REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_ACCESS_MASK, 0) +#define RING_FORCE_TO_NONPRIV_ACCESS_RD REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_ACCESS_MASK, 1) +#define RING_FORCE_TO_NONPRIV_ACCESS_WR REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_ACCESS_MASK, 2) +#define RING_FORCE_TO_NONPRIV_ACCESS_INVALID REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_ACCESS_MASK, 3) +#define RING_FORCE_TO_NONPRIV_ADDRESS_MASK REG_GENMASK(25, 2) +#define RING_FORCE_TO_NONPRIV_RANGE_MASK REG_GENMASK(1, 0) +#define RING_FORCE_TO_NONPRIV_RANGE_1 REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_RANGE_MASK, 0) +#define RING_FORCE_TO_NONPRIV_RANGE_4 REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_RANGE_MASK, 1) +#define RING_FORCE_TO_NONPRIV_RANGE_16 REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_RANGE_MASK, 2) +#define RING_FORCE_TO_NONPRIV_RANGE_64 REG_FIELD_PREP(RING_FORCE_TO_NONPRIV_RANGE_MASK, 3) +#define RING_FORCE_TO_NONPRIV_MASK_VALID (RING_FORCE_TO_NONPRIV_RANGE_MASK | \ + RING_FORCE_TO_NONPRIV_ACCESS_MASK | \ + RING_FORCE_TO_NONPRIV_DENY) +#define RING_MAX_NONPRIV_SLOTS 12 + +#define RING_EXECLIST_SQ_CONTENTS_LO(base) XE_REG((base) + 0x510) +#define RING_EXECLIST_SQ_CONTENTS_HI(base) XE_REG((base) + 0x510 + 4) + +#define RING_EXECLIST_CONTROL(base) XE_REG((base) + 0x550) +#define EL_CTRL_LOAD REG_BIT(0) + +#define CS_CHICKEN1(base) XE_REG((base) + 0x580, XE_REG_OPTION_MASKED) +#define PREEMPT_GPGPU_LEVEL(hi, lo) (((hi) << 2) | ((lo) << 1)) +#define PREEMPT_GPGPU_MID_THREAD_LEVEL PREEMPT_GPGPU_LEVEL(0, 0) +#define PREEMPT_GPGPU_THREAD_GROUP_LEVEL PREEMPT_GPGPU_LEVEL(0, 1) +#define PREEMPT_GPGPU_COMMAND_LEVEL PREEMPT_GPGPU_LEVEL(1, 0) +#define PREEMPT_GPGPU_LEVEL_MASK PREEMPT_GPGPU_LEVEL(1, 1) +#define PREEMPT_3D_OBJECT_LEVEL REG_BIT(0) + +#define CS_GPR_DATA(base, n) XE_REG((base) + 0x600 + (n) * 4) +#define CS_GPR_REG(base, n) CS_GPR_DATA((base), (n) * 2) +#define CS_GPR_REG_UDW(base, n) CS_GPR_DATA((base), (n) * 2 + 1) + +#define VDBOX_CGCTL3F08(base) XE_REG((base) + 0x3f08) +#define CG3DDISHRS_CLKGATE_DIS REG_BIT(5) + +#define VDBOX_CGCTL3F10(base) XE_REG((base) + 0x3f10) +#define IECPUNIT_CLKGATE_DIS REG_BIT(22) +#define RAMDFTUNIT_CLKGATE_DIS REG_BIT(9) + +#define VDBOX_CGCTL3F18(base) XE_REG((base) + 0x3f18) +#define ALNUNIT_CLKGATE_DIS REG_BIT(13) + +#define VDBOX_CGCTL3F1C(base) XE_REG((base) + 0x3f1c) +#define MFXPIPE_CLKGATE_DIS REG_BIT(3) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h b/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h new file mode 100644 index 000000000..c53f57fdd --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_EU_STALL_REGS_H_ +#define _XE_EU_STALL_REGS_H_ + +#include "regs/xe_reg_defs.h" + +#define XEHPC_EUSTALL_BASE XE_REG_MCR(0xe520) +#define XEHPC_EUSTALL_BASE_BUF_ADDR REG_GENMASK(31, 6) +#define XEHPC_EUSTALL_BASE_XECORE_BUF_SZ REG_GENMASK(5, 3) +#define XEHPC_EUSTALL_BASE_ENABLE_SAMPLING REG_BIT(1) + +#define XEHPC_EUSTALL_BASE_UPPER XE_REG_MCR(0xe524) + +#define XEHPC_EUSTALL_REPORT XE_REG_MCR(0xe528, XE_REG_OPTION_MASKED) +#define XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK REG_GENMASK(15, 2) +#define XEHPC_EUSTALL_REPORT_OVERFLOW_DROP REG_BIT(1) + +#define XEHPC_EUSTALL_REPORT1 XE_REG_MCR(0xe52c, XE_REG_OPTION_MASKED) +#define XEHPC_EUSTALL_REPORT1_READ_PTR_MASK REG_GENMASK(15, 2) + +#define XEHPC_EUSTALL_CTRL XE_REG_MCR(0xe53c, XE_REG_OPTION_MASKED) +#define EUSTALL_MOCS REG_GENMASK(9, 3) +#define EUSTALL_SAMPLE_RATE REG_GENMASK(2, 0) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h new file mode 100644 index 000000000..180be8267 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GSC_REGS_H_ +#define _XE_GSC_REGS_H_ + +#include +#include + +#include "regs/xe_reg_defs.h" + +/* Definitions of GSC H/W registers, bits, etc */ + +#define BMG_GSC_HECI1_BASE 0x373000 + +#define MTL_GSC_HECI1_BASE 0x00116000 +#define MTL_GSC_HECI2_BASE 0x00117000 + +#define DG1_GSC_HECI2_BASE 0x00259000 +#define PVC_GSC_HECI2_BASE 0x00285000 +#define DG2_GSC_HECI2_BASE 0x00374000 + +#define HECI_H_CSR(base) XE_REG((base) + 0x4) +#define HECI_H_CSR_IE REG_BIT(0) +#define HECI_H_CSR_IS REG_BIT(1) +#define HECI_H_CSR_IG REG_BIT(2) +#define HECI_H_CSR_RDY REG_BIT(3) +#define HECI_H_CSR_RST REG_BIT(4) + +/* + * The FWSTS register values are FW defined and can be different between + * HECI1 and HECI2 + */ +#define HECI_FWSTS1(base) XE_REG((base) + 0xc40) +#define HECI1_FWSTS1_CURRENT_STATE REG_GENMASK(3, 0) +#define HECI1_FWSTS1_CURRENT_STATE_RESET 0 +#define HECI1_FWSTS1_PROXY_STATE_NORMAL 5 +#define HECI1_FWSTS1_INIT_COMPLETE REG_BIT(9) +#define HECI_FWSTS2(base) XE_REG((base) + 0xc48) +#define HECI_FWSTS3(base) XE_REG((base) + 0xc60) +#define HECI_FWSTS4(base) XE_REG((base) + 0xc64) +#define HECI_FWSTS5(base) XE_REG((base) + 0xc68) +#define HECI1_FWSTS5_HUC_AUTH_DONE REG_BIT(19) +#define HECI_FWSTS6(base) XE_REG((base) + 0xc6c) + +#define HECI_H_GS1(base) XE_REG((base) + 0xc4c) +#define HECI_H_GS1_ER_PREP REG_BIT(0) + +#define GSCI_TIMER_STATUS XE_REG(0x11ca28) +#define GSCI_TIMER_STATUS_VALUE REG_GENMASK(1, 0) +#define GSCI_TIMER_STATUS_RESET_IN_PROGRESS 0 +#define GSCI_TIMER_STATUS_TIMER_EXPIRED 1 +#define GSCI_TIMER_STATUS_RESET_COMPLETE 2 +#define GSCI_TIMER_STATUS_OUT_OF_RESET 3 + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h new file mode 100644 index 000000000..08251c7a1 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -0,0 +1,666 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_REGS_H_ +#define _XE_GT_REGS_H_ + +#include "regs/xe_reg_defs.h" + +/* + * The GSI register range [0x0 - 0x40000) is replicated at a higher offset + * for the media GT. xe_mmio and xe_gt_mcr functions will automatically + * translate offsets by MEDIA_GT_GSI_OFFSET when operating on the media GT. + */ +#define MEDIA_GT_GSI_OFFSET 0x380000 +#define MEDIA_GT_GSI_LENGTH 0x40000 + +/* MTL workpoint reg to get core C state and actual freq of 3D, SAMedia */ +#define MTL_MIRROR_TARGET_WP1 XE_REG(0xc60) +#define MTL_CAGF_MASK REG_GENMASK(8, 0) +#define MTL_CC_MASK REG_GENMASK(12, 9) +#define MTL_CRST 0xf + +/* RPM unit config (Gen8+) */ +#define RPM_CONFIG0 XE_REG(0xd00) +#define RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK REG_GENMASK(5, 3) +#define RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 0 +#define RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 1 +#define RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ 2 +#define RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ 3 +#define RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK REG_GENMASK(2, 1) + +#define FORCEWAKE_ACK_MEDIA_VDBOX(n) XE_REG(0xd50 + (n) * 4) +#define FORCEWAKE_ACK_MEDIA_VEBOX(n) XE_REG(0xd70 + (n) * 4) +#define FORCEWAKE_ACK_RENDER XE_REG(0xd84) + +#define GMD_ID XE_REG(0xd8c) +#define GMD_ID_ARCH_MASK REG_GENMASK(31, 22) +#define GMD_ID_RELEASE_MASK REG_GENMASK(21, 14) +/* + * Spec defines these bits as "Reserved", but then make them assume some + * meaning that depends on the ARCH. To avoid any confusion, call them + * SUBIP_FLAG_MASK. + */ +#define GMD_ID_SUBIP_FLAG_MASK REG_GENMASK(13, 6) +#define GMD_ID_REVID REG_GENMASK(5, 0) + +#define FORCEWAKE_ACK_GSC XE_REG(0xdf8) +#define FORCEWAKE_ACK_GT_MTL XE_REG(0xdfc) + +#define STEER_SEMAPHORE XE_REG(0xfd0) +#define MTL_MCR_SELECTOR XE_REG(0xfd4) +#define SF_MCR_SELECTOR XE_REG(0xfd8) +#define MCR_SELECTOR XE_REG(0xfdc) +#define GAM_MCR_SELECTOR XE_REG(0xfe0) +#define MCR_MULTICAST REG_BIT(31) +#define MCR_SLICE_MASK REG_GENMASK(30, 27) +#define MCR_SLICE(slice) REG_FIELD_PREP(MCR_SLICE_MASK, slice) +#define MCR_SUBSLICE_MASK REG_GENMASK(26, 24) +#define MCR_SUBSLICE(subslice) REG_FIELD_PREP(MCR_SUBSLICE_MASK, subslice) +#define MTL_MCR_GROUPID REG_GENMASK(12, 8) +#define MTL_MCR_INSTANCEID REG_GENMASK(3, 0) + +#define PS_INVOCATION_COUNT XE_REG(0x2348) + +#define XELP_GLOBAL_MOCS(i) XE_REG(0x4000 + (i) * 4) +#define XEHP_GLOBAL_MOCS(i) XE_REG_MCR(0x4000 + (i) * 4) +#define LE_SSE_MASK REG_GENMASK(18, 17) +#define LE_SSE(value) REG_FIELD_PREP(LE_SSE_MASK, value) +#define LE_COS_MASK REG_GENMASK(16, 15) +#define LE_SCF_MASK REG_BIT(14) +#define LE_SCF(value) REG_FIELD_PREP(LE_SCF_MASK, value) +#define LE_PFM_MASK REG_GENMASK(13, 11) +#define LE_PFM(value) REG_FIELD_PREP(LE_PFM_MASK, value) +#define LE_SCC_MASK REG_GENMASK(10, 8) +#define LE_SCC(value) REG_FIELD_PREP(LE_SCC_MASK, value) +#define LE_RSC_MASK REG_BIT(7) +#define LE_RSC(value) REG_FIELD_PREP(LE_RSC_MASK, value) +#define LE_AOM_MASK REG_BIT(6) +#define LE_AOM(value) REG_FIELD_PREP(LE_AOM_MASK, value) +#define LE_LRUM_MASK REG_GENMASK(5, 4) +#define LE_LRUM(value) REG_FIELD_PREP(LE_LRUM_MASK, value) +#define LE_TGT_CACHE_MASK REG_GENMASK(3, 2) +#define LE_TGT_CACHE(value) REG_FIELD_PREP(LE_TGT_CACHE_MASK, value) +#define LE_CACHEABILITY_MASK REG_GENMASK(1, 0) +#define LE_CACHEABILITY(value) REG_FIELD_PREP(LE_CACHEABILITY_MASK, value) + +#define STATELESS_COMPRESSION_CTRL XE_REG_MCR(0x4148) +#define UNIFIED_COMPRESSION_FORMAT REG_GENMASK(3, 0) + +#define XE2_GAMREQSTRM_CTRL XE_REG_MCR(0x4194) +#define EN_CMP_1WCOH REG_BIT(15) +#define CG_DIS_CNTLBUS REG_BIT(6) + +#define CCS_AUX_INV XE_REG(0x4208) + +#define VD0_AUX_INV XE_REG(0x4218) +#define VE0_AUX_INV XE_REG(0x4238) + +#define VE1_AUX_INV XE_REG(0x42b8) +#define AUX_INV REG_BIT(0) + +#define GAMSTLB_CTRL XE_REG_MCR(0x477c) +#define DIS_PEND_GPA_LINK REG_BIT(13) + +#define GAMSTLB_CTRL2 XE_REG_MCR(0x4788) +#define STLB_SINGLE_BANK_MODE REG_BIT(11) + +#define XE2_LMEM_CFG XE_REG(0x48b0) + +#define XE2_GAMWALK_CTRL 0x47e4 +#define XE2_GAMWALK_CTRL_MEDIA XE_REG(XE2_GAMWALK_CTRL + MEDIA_GT_GSI_OFFSET) +#define XE2_GAMWALK_CTRL_3D XE_REG_MCR(XE2_GAMWALK_CTRL) +#define EN_CMP_1WCOH_GW REG_BIT(14) + +#define MMIOATSREQLIMIT_GAM_WALK_3D XE_REG_MCR(0x47f8) +#define DIS_ATS_WRONLY_PG REG_BIT(18) + +#define XEHP_FLAT_CCS_BASE_ADDR XE_REG_MCR(0x4910) +#define XEHP_FLAT_CCS_PTR REG_GENMASK(31, 8) + +#define WM_CHICKEN3 XE_REG_MCR(0x5588, XE_REG_OPTION_MASKED) +#define HIZ_PLANE_COMPRESSION_DIS REG_BIT(10) + +#define CHICKEN_RASTER_1 XE_REG_MCR(0x6204, XE_REG_OPTION_MASKED) +#define DIS_SF_ROUND_NEAREST_EVEN REG_BIT(8) +#define DIS_CLIP_NEGATIVE_BOUNDING_BOX REG_BIT(6) + +#define CHICKEN_RASTER_2 XE_REG_MCR(0x6208, XE_REG_OPTION_MASKED) +#define TBIMR_FAST_CLIP REG_BIT(5) + +#define FF_MODE XE_REG_MCR(0x6210) +#define DIS_TE_AUTOSTRIP REG_BIT(31) +#define VS_HIT_MAX_VALUE_MASK REG_GENMASK(25, 20) +#define DIS_MESH_PARTIAL_AUTOSTRIP REG_BIT(16) +#define DIS_MESH_AUTOSTRIP REG_BIT(15) +#define DIS_TE_PATCH_CTRL REG_BIT(4) + +#define VFLSKPD XE_REG_MCR(0x62a8, XE_REG_OPTION_MASKED) +#define DIS_PARTIAL_AUTOSTRIP REG_BIT(9) +#define DIS_AUTOSTRIP REG_BIT(6) +#define DIS_OVER_FETCH_CACHE REG_BIT(1) +#define DIS_MULT_MISS_RD_SQUASH REG_BIT(0) + +#define FF_MODE2 XE_REG(0x6604) +#define XEHP_FF_MODE2 XE_REG_MCR(0x6604) +#define FF_MODE2_GS_TIMER_MASK REG_GENMASK(31, 24) +#define FF_MODE2_GS_TIMER_224 REG_FIELD_PREP(FF_MODE2_GS_TIMER_MASK, 224) +#define FF_MODE2_TDS_TIMER_MASK REG_GENMASK(23, 16) +#define FF_MODE2_TDS_TIMER_128 REG_FIELD_PREP(FF_MODE2_TDS_TIMER_MASK, 4) + +#define XEHPG_INSTDONE_GEOM_SVGUNIT XE_REG_MCR(0x666c) + +#define CACHE_MODE_1 XE_REG_MCR(0x7004, XE_REG_OPTION_MASKED) +#define MSAA_OPTIMIZATION_REDUC_DISABLE REG_BIT(11) + +#define COMMON_SLICE_CHICKEN1 XE_REG(0x7010, XE_REG_OPTION_MASKED) +#define XEHP_COMMON_SLICE_CHICKEN1 XE_REG_MCR(0x7010, XE_REG_OPTION_MASKED) +#define DISABLE_BOTTOM_CLIP_RECTANGLE_TEST REG_BIT(14) + +#define HIZ_CHICKEN XE_REG(0x7018, XE_REG_OPTION_MASKED) +#define DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE REG_BIT(14) +#define HZ_DEPTH_TEST_LE_GE_OPT_DISABLE REG_BIT(13) + +#define XEHP_PSS_MODE2 XE_REG_MCR(0x703c, XE_REG_OPTION_MASKED) +#define SCOREBOARD_STALL_FLUSH_CONTROL REG_BIT(5) + +#define XEHP_PSS_CHICKEN XE_REG_MCR(0x7044, XE_REG_OPTION_MASKED) +#define FLSH_IGNORES_PSD REG_BIT(10) +#define FD_END_COLLECT REG_BIT(5) + +#define SC_INSTDONE XE_REG(0x7100) +#define SC_INSTDONE_EXTRA XE_REG(0x7104) +#define SC_INSTDONE_EXTRA2 XE_REG(0x7108) + +#define XEHPG_SC_INSTDONE XE_REG_MCR(0x7100) +#define XEHPG_SC_INSTDONE_EXTRA XE_REG_MCR(0x7104) +#define XEHPG_SC_INSTDONE_EXTRA2 XE_REG_MCR(0x7108) + +#define COMMON_SLICE_CHICKEN4 XE_REG(0x7300, XE_REG_OPTION_MASKED) +#define XEHP_COMMON_SLICE_CHICKEN4 XE_REG_MCR(0x7300, XE_REG_OPTION_MASKED) +#define SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE REG_BIT(12) +#define DISABLE_TDC_LOAD_BALANCING_CALC REG_BIT(6) +#define HW_FILTERING REG_BIT(5) + +#define COMMON_SLICE_CHICKEN3 XE_REG(0x7304, XE_REG_OPTION_MASKED) +#define XEHP_COMMON_SLICE_CHICKEN3 XE_REG_MCR(0x7304, XE_REG_OPTION_MASKED) +#define DISABLE_STATE_CACHE_PERF_FIX REG_BIT(13) +#define DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN REG_BIT(12) +#define XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE REG_BIT(12) +#define BLEND_EMB_FIX_DISABLE_IN_RCC REG_BIT(11) +#define DISABLE_CPS_AWARE_COLOR_PIPE REG_BIT(9) + +#define XEHP_SLICE_COMMON_ECO_CHICKEN1 XE_REG_MCR(0x731c, XE_REG_OPTION_MASKED) +#define MSC_MSAA_REODER_BUF_BYPASS_DISABLE REG_BIT(14) +#define FAST_CLEAR_VALIGN_FIX REG_BIT(13) + +#define XE2LPM_CCCHKNREG1 XE_REG(0x82a8) + +#define VF_PREEMPTION XE_REG(0x83a4, XE_REG_OPTION_MASKED) +#define PREEMPTION_VERTEX_COUNT REG_GENMASK(15, 0) + +#define VF_SCRATCHPAD XE_REG(0x83a8, XE_REG_OPTION_MASKED) +#define XE2_VFG_TED_CREDIT_INTERFACE_DISABLE REG_BIT(13) + +#define VFG_PREEMPTION_CHICKEN XE_REG(0x83b4, XE_REG_OPTION_MASKED) +#define POLYGON_TRIFAN_LINELOOP_DISABLE REG_BIT(4) + +#define SQCNT1 XE_REG_MCR(0x8718) +#define XELPMP_SQCNT1 XE_REG(0x8718) +#define SQCNT1_PMON_ENABLE REG_BIT(30) +#define SQCNT1_OABPC REG_BIT(29) +#define ENFORCE_RAR REG_BIT(23) + +#define XEHP_SQCM XE_REG_MCR(0x8724) +#define EN_32B_ACCESS REG_BIT(30) + +#define XE2_FLAT_CCS_BASE_RANGE_LOWER XE_REG_MCR(0x8800) +#define XE2_FLAT_CCS_ENABLE REG_BIT(0) +#define XE2_FLAT_CCS_BASE_LOWER_ADDR_MASK REG_GENMASK(31, 6) + +#define XE2_FLAT_CCS_BASE_RANGE_UPPER XE_REG_MCR(0x8804) +#define XE2_FLAT_CCS_BASE_UPPER_ADDR_MASK REG_GENMASK(7, 0) + +#define GSCPSMI_BASE XE_REG(0x880c) + +#define CCCHKNREG1 XE_REG_MCR(0x8828) +#define L3CMPCTRL REG_BIT(23) +#define ENCOMPPERFFIX REG_BIT(18) + +/* Fuse readout registers for GT */ +#define XEHP_FUSE4 XE_REG(0x9114) +#define CFEG_WMTP_DISABLE REG_BIT(20) +#define CCS_EN_MASK REG_GENMASK(19, 16) +#define GT_L3_EXC_MASK REG_GENMASK(6, 4) + +#define MIRROR_FUSE3 XE_REG(0x9118) +#define XE2_NODE_ENABLE_MASK REG_GENMASK(31, 16) +#define L3BANK_PAIR_COUNT 4 +#define XEHPC_GT_L3_MODE_MASK REG_GENMASK(7, 4) +#define XE2_GT_L3_MODE_MASK REG_GENMASK(7, 4) +#define L3BANK_MASK REG_GENMASK(3, 0) +#define XELP_GT_L3_MODE_MASK REG_GENMASK(7, 0) +/* on Xe_HP the same fuses indicates mslices instead of L3 banks */ +#define MAX_MSLICES 4 +#define MEML3_EN_MASK REG_GENMASK(3, 0) + +#define MIRROR_FUSE1 XE_REG(0x911c) + +#define FUSE2 XE_REG(0x9120) +#define PRODUCTION_HW REG_BIT(2) + +#define MIRROR_L3BANK_ENABLE XE_REG(0x9130) +#define XE3_L3BANK_ENABLE REG_GENMASK(31, 0) + +#define XELP_EU_ENABLE XE_REG(0x9134) /* "_DISABLE" on Xe_LP */ +#define XELP_EU_MASK REG_GENMASK(7, 0) +#define XELP_GT_SLICE_ENABLE XE_REG(0x9138) +#define XELP_GT_GEOMETRY_DSS_ENABLE XE_REG(0x913c) + +#define GT_VEBOX_VDBOX_DISABLE XE_REG(0x9140) +#define GT_VEBOX_DISABLE_MASK REG_GENMASK(19, 16) +#define GT_VDBOX_DISABLE_MASK REG_GENMASK(7, 0) + +#define XEHP_GT_COMPUTE_DSS_ENABLE XE_REG(0x9144) +#define XEHPC_GT_COMPUTE_DSS_ENABLE_EXT XE_REG(0x9148) +#define XE2_GT_COMPUTE_DSS_2 XE_REG(0x914c) +#define XE2_GT_GEOMETRY_DSS_1 XE_REG(0x9150) +#define XE2_GT_GEOMETRY_DSS_2 XE_REG(0x9154) +#define XE3P_XPC_GT_GEOMETRY_DSS_3 XE_REG(0x915c) +#define XE3P_XPC_GT_COMPUTE_DSS_3 XE_REG(0x9160) + +#define SERVICE_COPY_ENABLE XE_REG(0x9170) +#define FUSE_SERVICE_COPY_ENABLE_MASK REG_GENMASK(7, 0) + +#define GDRST XE_REG(0x941c) +#define GRDOM_GUC REG_BIT(3) +#define GRDOM_FULL REG_BIT(0) + +#define MISCCPCTL XE_REG(0x9424) +#define DOP_CLOCK_GATE_RENDER_ENABLE REG_BIT(1) + +#define UNSLCGCTL9430 XE_REG(0x9430) +#define MSQDUNIT_CLKGATE_DIS REG_BIT(3) + +#define UNSLICE_UNIT_LEVEL_CLKGATE XE_REG(0x9434) +#define VFUNIT_CLKGATE_DIS REG_BIT(20) +#define TSGUNIT_CLKGATE_DIS REG_BIT(17) /* XEHPSDV */ +#define CG3DDISCFEG_CLKGATE_DIS REG_BIT(17) /* DG2 */ +#define GAMEDIA_CLKGATE_DIS REG_BIT(11) +#define HSUNIT_CLKGATE_DIS REG_BIT(8) +#define VSUNIT_CLKGATE_DIS REG_BIT(3) + +#define UNSLCGCTL9440 XE_REG(0x9440) +#define GAMTLBOACS_CLKGATE_DIS REG_BIT(28) +#define GAMTLBVDBOX5_CLKGATE_DIS REG_BIT(27) +#define GAMTLBVDBOX6_CLKGATE_DIS REG_BIT(26) +#define GAMTLBVDBOX3_CLKGATE_DIS REG_BIT(24) +#define GAMTLBVDBOX4_CLKGATE_DIS REG_BIT(23) +#define GAMTLBVDBOX7_CLKGATE_DIS REG_BIT(22) +#define GAMTLBVDBOX2_CLKGATE_DIS REG_BIT(21) +#define GAMTLBVDBOX0_CLKGATE_DIS REG_BIT(17) +#define GAMTLBKCR_CLKGATE_DIS REG_BIT(16) +#define GAMTLBGUC_CLKGATE_DIS REG_BIT(15) +#define GAMTLBBLT_CLKGATE_DIS REG_BIT(14) +#define GAMTLBVDBOX1_CLKGATE_DIS REG_BIT(6) + +#define UNSLCGCTL9444 XE_REG(0x9444) +#define GAMTLBGFXA0_CLKGATE_DIS REG_BIT(30) +#define GAMTLBGFXA1_CLKGATE_DIS REG_BIT(29) +#define GAMTLBCOMPA0_CLKGATE_DIS REG_BIT(28) +#define GAMTLBCOMPA1_CLKGATE_DIS REG_BIT(27) +#define GAMTLBCOMPB0_CLKGATE_DIS REG_BIT(26) +#define GAMTLBCOMPB1_CLKGATE_DIS REG_BIT(25) +#define GAMTLBCOMPC0_CLKGATE_DIS REG_BIT(24) +#define GAMTLBCOMPC1_CLKGATE_DIS REG_BIT(23) +#define GAMTLBCOMPD0_CLKGATE_DIS REG_BIT(22) +#define GAMTLBCOMPD1_CLKGATE_DIS REG_BIT(21) +#define GAMTLBMERT_CLKGATE_DIS REG_BIT(20) +#define GAMTLBVEBOX3_CLKGATE_DIS REG_BIT(19) +#define GAMTLBVEBOX2_CLKGATE_DIS REG_BIT(18) +#define GAMTLBVEBOX1_CLKGATE_DIS REG_BIT(17) +#define GAMTLBVEBOX0_CLKGATE_DIS REG_BIT(16) +#define LTCDD_CLKGATE_DIS REG_BIT(10) + +#define UNSLCGCTL9454 XE_REG(0x9454) +#define LSCFE_CLKGATE_DIS REG_BIT(4) + +#define XEHP_SLICE_UNIT_LEVEL_CLKGATE XE_REG_MCR(0x94d4) +#define L3_CR2X_CLKGATE_DIS REG_BIT(17) +#define L3_CLKGATE_DIS REG_BIT(16) +#define NODEDSS_CLKGATE_DIS REG_BIT(12) +#define MSCUNIT_CLKGATE_DIS REG_BIT(10) +#define RCCUNIT_CLKGATE_DIS REG_BIT(7) +#define SARBUNIT_CLKGATE_DIS REG_BIT(5) +#define SBEUNIT_CLKGATE_DIS REG_BIT(4) + +#define UNSLICE_UNIT_LEVEL_CLKGATE2 XE_REG(0x94e4) +#define VSUNIT_CLKGATE2_DIS REG_BIT(19) + +#define SUBSLICE_UNIT_LEVEL_CLKGATE XE_REG_MCR(0x9524) +#define DSS_ROUTER_CLKGATE_DIS REG_BIT(28) +#define GWUNIT_CLKGATE_DIS REG_BIT(16) + +#define SUBSLICE_UNIT_LEVEL_CLKGATE2 XE_REG_MCR(0x9528) +#define CPSSUNIT_CLKGATE_DIS REG_BIT(9) + +#define SSMCGCTL9530 XE_REG_MCR(0x9530) +#define RTFUNIT_CLKGATE_DIS REG_BIT(18) + +#define DFR_RATIO_EN_AND_CHICKEN XE_REG_MCR(0x9550) +#define DFR_DISABLE REG_BIT(9) + +#define RPNSWREQ XE_REG(0xa008) +#define REQ_RATIO_MASK REG_GENMASK(31, 23) + +#define RP_CONTROL XE_REG(0xa024) +#define RPSWCTL_MASK REG_GENMASK(10, 9) +#define RPSWCTL_ENABLE REG_FIELD_PREP(RPSWCTL_MASK, 2) +#define RPSWCTL_DISABLE REG_FIELD_PREP(RPSWCTL_MASK, 0) +#define RC_CONTROL XE_REG(0xa090) +#define RC_CTL_HW_ENABLE REG_BIT(31) +#define RC_CTL_TO_MODE REG_BIT(28) +#define RC_CTL_RC6_ENABLE REG_BIT(18) +#define RC_STATE XE_REG(0xa094) +#define RC_IDLE_HYSTERSIS XE_REG(0xa0ac) +#define MEDIA_POWERGATE_IDLE_HYSTERESIS XE_REG(0xa0c4) +#define RENDER_POWERGATE_IDLE_HYSTERESIS XE_REG(0xa0c8) + +#define PMINTRMSK XE_REG(0xa168) +#define PMINTR_DISABLE_REDIRECT_TO_GUC REG_BIT(31) +#define ARAT_EXPIRED_INTRMSK REG_BIT(9) + +#define FORCEWAKE_GT XE_REG(0xa188) + +#define POWERGATE_ENABLE XE_REG(0xa210) +#define RENDER_POWERGATE_ENABLE REG_BIT(0) +#define MEDIA_POWERGATE_ENABLE REG_BIT(1) +#define MEDIA_SAMPLERS_POWERGATE_ENABLE REG_BIT(2) +#define VDN_HCP_POWERGATE_ENABLE(n) REG_BIT(3 + 2 * (n)) +#define VDN_MFXVDENC_POWERGATE_ENABLE(n) REG_BIT(4 + 2 * (n)) + +#define FORCEWAKE_RENDER XE_REG(0xa278) + +#define POWERGATE_DOMAIN_STATUS XE_REG(0xa2a0) +#define GSC_AWAKE_STATUS REG_BIT(8) +#define MEDIA_SLICE3_AWAKE_STATUS REG_BIT(4) +#define MEDIA_SLICE2_AWAKE_STATUS REG_BIT(3) +#define MEDIA_SLICE1_AWAKE_STATUS REG_BIT(2) +#define RENDER_AWAKE_STATUS REG_BIT(1) +#define MEDIA_SLICE0_AWAKE_STATUS REG_BIT(0) + +#define MISC_STATUS_0 XE_REG(0xa500) + +#define FORCEWAKE_MEDIA_VDBOX(n) XE_REG(0xa540 + (n) * 4) +#define FORCEWAKE_MEDIA_VEBOX(n) XE_REG(0xa560 + (n) * 4) +#define FORCEWAKE_GSC XE_REG(0xa618) + +#define XELP_GARBCNTL XE_REG(0xb004) +#define XELP_BUS_HASH_CTL_BIT_EXC REG_BIT(7) + +#define XEHPC_LNCFMISCCFGREG0 XE_REG_MCR(0xb01c, XE_REG_OPTION_MASKED) +#define XEHPC_OVRLSCCC REG_BIT(0) + +#define LNCFCMOCS_REG_COUNT 32 +#define XELP_LNCFCMOCS(i) XE_REG(0xb020 + (i) * 4) +#define XEHP_LNCFCMOCS(i) XE_REG_MCR(0xb020 + (i) * 4) +#define L3_UPPER_LKUP_MASK REG_BIT(23) +#define L3_UPPER_GLBGO_MASK REG_BIT(22) +#define L3_UPPER_IDX_CACHEABILITY_MASK REG_GENMASK(21, 20) +#define L3_UPPER_IDX_SCC_MASK REG_GENMASK(19, 17) +#define L3_UPPER_IDX_ESC_MASK REG_BIT(16) +#define L3_LKUP_MASK REG_BIT(7) +#define L3_LKUP(value) REG_FIELD_PREP(L3_LKUP_MASK, value) +#define L3_GLBGO_MASK REG_BIT(6) +#define L3_GLBGO(value) REG_FIELD_PREP(L3_GLBGO_MASK, value) +#define L3_CACHEABILITY_MASK REG_GENMASK(5, 4) +#define L3_CACHEABILITY(value) REG_FIELD_PREP(L3_CACHEABILITY_MASK, value) +#define L3_SCC_MASK REG_GENMASK(3, 1) +#define L3_SCC(value) REG_FIELD_PREP(L3_SCC_MASK, value) +#define L3_ESC_MASK REG_BIT(0) +#define L3_ESC(value) REG_FIELD_PREP(L3_ESC_MASK, value) + +#define XEHP_L3NODEARBCFG XE_REG_MCR(0xb0b4) +#define XEHP_LNESPARE REG_BIT(19) + +#define LSN_VC_REG2 XE_REG_MCR(0xb0c8) +#define LSN_LNI_WGT_MASK REG_GENMASK(31, 28) +#define LSN_LNI_WGT(value) REG_FIELD_PREP(LSN_LNI_WGT_MASK, value) +#define LSN_LNE_WGT_MASK REG_GENMASK(27, 24) +#define LSN_LNE_WGT(value) REG_FIELD_PREP(LSN_LNE_WGT_MASK, value) +#define LSN_DIM_X_WGT_MASK REG_GENMASK(23, 20) +#define LSN_DIM_X_WGT(value) REG_FIELD_PREP(LSN_DIM_X_WGT_MASK, value) +#define LSN_DIM_Y_WGT_MASK REG_GENMASK(19, 16) +#define LSN_DIM_Y_WGT(value) REG_FIELD_PREP(LSN_DIM_Y_WGT_MASK, value) +#define LSN_DIM_Z_WGT_MASK REG_GENMASK(15, 12) +#define LSN_DIM_Z_WGT(value) REG_FIELD_PREP(LSN_DIM_Z_WGT_MASK, value) + +#define L3SQCREG2 XE_REG_MCR(0xb104) +#define L3_SQ_DISABLE_COAMA_2WAY_COH REG_BIT(30) +#define L3_SQ_DISABLE_COAMA REG_BIT(22) +#define COMPMEMRD256BOVRFETCHEN REG_BIT(20) + +#define L3SQCREG3 XE_REG_MCR(0xb108) +#define COMPPWOVERFETCHEN REG_BIT(28) + +#define SCRATCH3_LBCF XE_REG_MCR(0xb154) +#define RWFLUSHALLEN REG_BIT(17) + +#define XEHP_L3SQCREG5 XE_REG_MCR(0xb158) +#define L3_PWM_TIMER_INIT_VAL_MASK REG_GENMASK(9, 0) + +#define XEHP_L3SCQREG7 XE_REG_MCR(0xb188) +#define BLEND_FILL_CACHING_OPT_DIS REG_BIT(3) + +#define XEHPC_L3CLOS_MASK(i) XE_REG_MCR(0xb194 + (i) * 8) + +#define L2COMPUTESIDECTRL XE_REG_MCR(0xb1c0) +#define CECTRL REG_GENMASK(2, 1) +#define CECTRL_CENODATA_ALWAYS REG_FIELD_PREP(CECTRL, 0x0) + +#define XE2_GLOBAL_INVAL XE_REG(0xb404) + +#define XE2LPM_L3SQCREG2 XE_REG_MCR(0xb604) + +#define XE2LPM_L3SQCREG3 XE_REG_MCR(0xb608) + +#define XE2LPM_SCRATCH3_LBCF XE_REG_MCR(0xb654) + +#define XE2LPM_L3SQCREG5 XE_REG_MCR(0xb658) + +#define XE2_TDF_CTRL XE_REG(0xb418) +#define TRANSIENT_FLUSH_REQUEST REG_BIT(0) + +#define XEHP_MERT_MOD_CTRL XE_REG_MCR(0xcf28) +#define RENDER_MOD_CTRL XE_REG_MCR(0xcf2c) +#define COMP_MOD_CTRL XE_REG_MCR(0xcf30) +#define XEHP_VDBX_MOD_CTRL XE_REG_MCR(0xcf34) +#define XELPMP_VDBX_MOD_CTRL XE_REG(0xcf34) +#define XEHP_VEBX_MOD_CTRL XE_REG_MCR(0xcf38) +#define XELPMP_VEBX_MOD_CTRL XE_REG(0xcf38) +#define FORCE_MISS_FTLB REG_BIT(3) + +#define XEHP_GAMSTLB_CTRL XE_REG_MCR(0xcf4c) +#define BANK_HASH_MODE REG_GENMASK(27, 26) +#define BANK_HASH_4KB_MODE REG_FIELD_PREP(BANK_HASH_MODE, 0x3) +#define CONTROL_BLOCK_CLKGATE_DIS REG_BIT(12) +#define EGRESS_BLOCK_CLKGATE_DIS REG_BIT(11) +#define TAG_BLOCK_CLKGATE_DIS REG_BIT(7) + +#define XEHP_GAMCNTRL_CTRL XE_REG_MCR(0xcf54) +#define INVALIDATION_BROADCAST_MODE_DIS REG_BIT(12) +#define GLOBAL_INVALIDATION_MODE REG_BIT(2) + +#define LMEM_CFG XE_REG(0xcf58) +#define LMEM_EN REG_BIT(31) +#define LMTT_DIR_PTR REG_GENMASK(30, 0) /* in multiples of 64KB */ + +#define HALF_SLICE_CHICKEN5 XE_REG_MCR(0xe188, XE_REG_OPTION_MASKED) +#define DISABLE_SAMPLE_G_PERFORMANCE REG_BIT(0) + +#define SAMPLER_INSTDONE XE_REG_MCR(0xe160) +#define ROW_INSTDONE XE_REG_MCR(0xe164) + +#define SAMPLER_MODE XE_REG_MCR(0xe18c, XE_REG_OPTION_MASKED) +#define ENABLE_SMALLPL REG_BIT(15) +#define SMP_WAIT_FETCH_MERGING_COUNTER REG_GENMASK(11, 10) +#define SMP_FORCE_128B_OVERFETCH REG_FIELD_PREP(SMP_WAIT_FETCH_MERGING_COUNTER, 1) +#define SC_DISABLE_POWER_OPTIMIZATION_EBB REG_BIT(9) +#define SAMPLER_ENABLE_HEADLESS_MSG REG_BIT(5) +#define INDIRECT_STATE_BASE_ADDR_OVERRIDE REG_BIT(0) + +#define HALF_SLICE_CHICKEN7 XE_REG_MCR(0xe194, XE_REG_OPTION_MASKED) +#define DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA REG_BIT(15) +#define CLEAR_OPTIMIZATION_DISABLE REG_BIT(6) + +#define CACHE_MODE_SS XE_REG_MCR(0xe420, XE_REG_OPTION_MASKED) +#define DISABLE_ECC REG_BIT(5) +#define ENABLE_PREFETCH_INTO_IC REG_BIT(3) + +#define ROW_CHICKEN4 XE_REG_MCR(0xe48c, XE_REG_OPTION_MASKED) +#define DISABLE_GRF_CLEAR REG_BIT(13) +#define XEHP_DIS_BBL_SYSPIPE REG_BIT(11) +#define DISABLE_TDL_PUSH REG_BIT(9) +#define DIS_PICK_2ND_EU REG_BIT(7) +#define DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX REG_BIT(4) +#define THREAD_EX_ARB_MODE REG_GENMASK(3, 2) +#define THREAD_EX_ARB_MODE_RR_AFTER_DEP REG_FIELD_PREP(THREAD_EX_ARB_MODE, 0x2) + +#define ROW_CHICKEN3 XE_REG_MCR(0xe49c, XE_REG_OPTION_MASKED) +#define XE2_EUPEND_CHK_FLUSH_DIS REG_BIT(14) +#define DIS_EU_GRF_POISON_TO_LSC REG_BIT(13) +#define DIS_FIX_EOT1_FLUSH REG_BIT(9) + +#define TDL_TSL_CHICKEN XE_REG_MCR(0xe4c4, XE_REG_OPTION_MASKED) +#define STK_ID_RESTRICT REG_BIT(12) +#define SLM_WMTP_RESTORE REG_BIT(11) +#define RES_CHK_SPR_DIS REG_BIT(6) + +#define TDL_TSL_CHICKEN2 XE_REG_MCR(0xe4cc, XE_REG_OPTION_MASKED) +#define TILEY_LOCALID REG_BIT(2) + +#define ROW_CHICKEN XE_REG_MCR(0xe4f0, XE_REG_OPTION_MASKED) +#define UGM_BACKUP_MODE REG_BIT(13) +#define MDQ_ARBITRATION_MODE REG_BIT(12) +#define STALL_DOP_GATING_DISABLE REG_BIT(5) +#define EARLY_EOT_DIS REG_BIT(1) + +#define ROW_CHICKEN2 XE_REG_MCR(0xe4f4, XE_REG_OPTION_MASKED) +#define DISABLE_READ_SUPPRESSION REG_BIT(15) +#define DISABLE_EARLY_READ REG_BIT(14) +#define ENABLE_LARGE_GRF_MODE REG_BIT(12) +#define PUSH_CONST_DEREF_HOLD_DIS REG_BIT(8) +#define DISABLE_TDL_SVHS_GATING REG_BIT(1) +#define DISABLE_DOP_GATING REG_BIT(0) + +#define RT_CTRL XE_REG_MCR(0xe530) +#define DIS_NULL_QUERY REG_BIT(10) + +#define EU_SYSTOLIC_LIC_THROTTLE_CTL_WITH_LOCK XE_REG_MCR(0xe534) +#define EU_SYSTOLIC_LIC_THROTTLE_CTL_LOCK_BIT REG_BIT(31) + +#define XEHP_HDC_CHICKEN0 XE_REG_MCR(0xe5f0, XE_REG_OPTION_MASKED) +#define LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK REG_GENMASK(13, 11) +#define DIS_ATOMIC_CHAINING_TYPED_WRITES REG_BIT(3) + +#define TDL_CHICKEN XE_REG_MCR(0xe5f4, XE_REG_OPTION_MASKED) +#define BIT_APQ_OPT_DIS REG_BIT(14) +#define QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE REG_BIT(12) +#define EUSTALL_PERF_SAMPLING_DISABLE REG_BIT(5) + +#define LSC_CHICKEN_BIT_0 XE_REG_MCR(0xe7c8) +#define DISABLE_D8_D16_COASLESCE REG_BIT(30) +#define WR_REQ_CHAINING_DIS REG_BIT(26) +#define TGM_WRITE_EOM_FORCE REG_BIT(17) +#define FORCE_1_SUB_MESSAGE_PER_FRAGMENT REG_BIT(15) +#define SEQUENTIAL_ACCESS_UPGRADE_DISABLE REG_BIT(13) + +#define LSC_CHICKEN_BIT_0_UDW XE_REG_MCR(0xe7c8 + 4) +#define UGM_FRAGMENT_THRESHOLD_TO_3 REG_BIT(58 - 32) +#define DIS_CHAIN_2XSIMD8 REG_BIT(55 - 32) +#define XE2_ALLOC_DPA_STARVE_FIX_DIS REG_BIT(47 - 32) +#define SAMPLER_LD_LSC_DISABLE REG_BIT(45 - 32) +#define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32) +#define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32) +#define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32) +#define L3_128B_256B_WRT_DIS REG_BIT(40 - 32) +#define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32) +#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) +#define LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE REG_BIT(35 - 32) + +#define ROW_CHICKEN5 XE_REG_MCR(0xe7f0, XE_REG_OPTION_MASKED) +#define CPSS_AWARE_DIS REG_BIT(3) + +#define SARB_CHICKEN1 XE_REG_MCR(0xe90c) +#define COMP_CKN_IN REG_GENMASK(30, 29) + +#define MAIN_GAMCTRL_MODE XE_REG(0xef00) +#define MAIN_GAMCTRL_QUEUE_SELECT REG_BIT(0) + +#define RCU_MODE XE_REG(0x14800, XE_REG_OPTION_MASKED) +#define RCU_MODE_FIXED_SLICE_CCS_MODE REG_BIT(1) +#define RCU_MODE_CCS_ENABLE REG_BIT(0) + +/* + * Total of 4 cslices, where each cslice is in the form: + * [0-3] CCS ID + * [4-6] RSVD + * [7] Disabled + */ +#define CCS_MODE XE_REG(0x14804, XE_REG_OPTION_MASKED) +#define CCS_MODE_CSLICE_0_3_MASK REG_GENMASK(11, 0) /* 3 bits per cslice */ +#define CCS_MODE_CSLICE_MASK 0x7 /* CCS0-3 + rsvd */ +#define CCS_MODE_CSLICE_WIDTH ilog2(CCS_MODE_CSLICE_MASK + 1) +#define CCS_MODE_CSLICE(cslice, ccs) \ + ((ccs) << ((cslice) * CCS_MODE_CSLICE_WIDTH)) + +#define FORCEWAKE_ACK_GT XE_REG(0x130044) + +/* Applicable for all FORCEWAKE_DOMAIN and FORCEWAKE_ACK_DOMAIN regs */ +#define FORCEWAKE_KERNEL 0 +#define FORCEWAKE_MT(bit) BIT(bit) +#define FORCEWAKE_MT_MASK(bit) BIT((bit) + 16) + +#define MTL_MEDIA_PERF_LIMIT_REASONS XE_REG(0x138030) +#define MTL_MEDIA_MC6 XE_REG(0x138048) + +#define GT_CORE_STATUS XE_REG(0x138060) +#define RCN_MASK REG_GENMASK(2, 0) +#define GT_C0 0 +#define GT_C6 3 + +#define GT_GFX_RC6_LOCKED XE_REG(0x138104) +#define GT_GFX_RC6 XE_REG(0x138108) + +#define GT0_PERF_LIMIT_REASONS XE_REG(0x1381a8) +/* Common performance limit reason bits - available on all platforms */ +#define GT0_PERF_LIMIT_REASONS_MASK 0xde3 +#define PROCHOT_MASK REG_BIT(0) +#define THERMAL_LIMIT_MASK REG_BIT(1) +#define RATL_MASK REG_BIT(5) +#define VR_THERMALERT_MASK REG_BIT(6) +#define VR_TDC_MASK REG_BIT(7) +#define POWER_LIMIT_4_MASK REG_BIT(8) +#define POWER_LIMIT_1_MASK REG_BIT(10) +#define POWER_LIMIT_2_MASK REG_BIT(11) +/* Platform-specific performance limit reason bits - for Crescent Island */ +#define CRI_PERF_LIMIT_REASONS_MASK 0xfdff +#define SOC_THERMAL_LIMIT_MASK REG_BIT(1) +#define MEM_THERMAL_MASK REG_BIT(2) +#define VR_THERMAL_MASK REG_BIT(3) +#define ICCMAX_MASK REG_BIT(4) +#define SOC_AVG_THERMAL_MASK REG_BIT(6) +#define FASTVMODE_MASK REG_BIT(7) +#define PSYS_PL1_MASK REG_BIT(12) +#define PSYS_PL2_MASK REG_BIT(13) +#define P0_FREQ_MASK REG_BIT(14) +#define PSYS_CRIT_MASK REG_BIT(15) + +#define GT_PERF_STATUS XE_REG(0x1381b4) +#define VOLTAGE_MASK REG_GENMASK(10, 0) + +#define SFC_DONE(n) XE_REG(0x1cc000 + (n) * 0x1000) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h new file mode 100644 index 000000000..d6bc19ef2 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GTT_DEFS_H_ +#define _XE_GTT_DEFS_H_ + +#define XELPG_GGTT_PTE_PAT0 BIT_ULL(52) +#define XELPG_GGTT_PTE_PAT1 BIT_ULL(53) + +/* + * Mask for PTE address bits [51:shift]. + * shift is the lower address boundary of page. + */ +#define XE_PAGE_ADDR_MASK(shift) GENMASK_ULL(51, (shift)) +#define GGTT_PTE_VFID GENMASK_ULL(11, 2) + +#define GUC_GGTT_TOP 0xFEE00000 + +#define XELPG_PPGTT_PTE_PAT3 BIT_ULL(62) +#define XE2_PPGTT_PTE_PAT4 BIT_ULL(61) +#define XE_PPGTT_PDE_PDPE_PAT2 BIT_ULL(12) +#define XE_PPGTT_PTE_PAT2 BIT_ULL(7) +#define XE_PPGTT_PTE_PAT1 BIT_ULL(4) +#define XE_PPGTT_PTE_PAT0 BIT_ULL(3) + +#define XE_PDE_PS_2M BIT_ULL(7) +#define XE_PDPE_PS_1G BIT_ULL(7) +#define XE_PDE_IPS_64K BIT_ULL(11) + +#define XE_GGTT_PTE_DM BIT_ULL(1) +#define XE_USM_PPGTT_PTE_AE BIT_ULL(10) +#define XE_PPGTT_PTE_DM BIT_ULL(11) +#define XE_PDE_64K BIT_ULL(6) +#define XE_PTE_PS64 BIT_ULL(8) +#define XE_PTE_NULL BIT_ULL(9) + +#define XE_PAGE_PRESENT BIT_ULL(0) +#define XE_PAGE_RW BIT_ULL(1) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h new file mode 100644 index 000000000..5faac8316 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_REGS_H_ +#define _XE_GUC_REGS_H_ + +#include +#include + +#include "regs/xe_reg_defs.h" + +/* Definitions of GuC H/W registers, bits, etc */ + +#define DIST_DBS_POPULATED XE_REG(0xd08) +#define DOORBELLS_PER_SQIDI_MASK REG_GENMASK(23, 16) +#define SQIDIS_DOORBELL_EXIST_MASK REG_GENMASK(15, 0) + +#define DRBREGL(x) XE_REG(0x1000 + (x) * 8) +#define DRB_VALID REG_BIT(0) +#define DRBREGU(x) XE_REG(0x1000 + (x) * 8 + 4) + +#define GTCR XE_REG(0x4274) +#define GTCR_INVALIDATE REG_BIT(0) + +#define GUC_ARAT_C6DIS XE_REG(0xa178) + +#define GUC_STATUS XE_REG(0xc000) +#define GS_AUTH_STATUS_MASK REG_GENMASK(31, 30) +#define GS_AUTH_STATUS_BAD REG_FIELD_PREP(GS_AUTH_STATUS_MASK, 0x1) +#define GS_AUTH_STATUS_GOOD REG_FIELD_PREP(GS_AUTH_STATUS_MASK, 0x2) +#define GS_MIA_MASK REG_GENMASK(18, 16) +#define GS_MIA_CORE_STATE REG_FIELD_PREP(GS_MIA_MASK, 0x1) +#define GS_MIA_HALT_REQUESTED REG_FIELD_PREP(GS_MIA_MASK, 0x2) +#define GS_MIA_ISR_ENTRY REG_FIELD_PREP(GS_MIA_MASK, 0x4) +#define GS_UKERNEL_MASK REG_GENMASK(15, 8) +#define GS_BOOTROM_MASK REG_GENMASK(7, 1) +#define GS_BOOTROM_RSA_FAILED REG_FIELD_PREP(GS_BOOTROM_MASK, 0x50) +#define GS_BOOTROM_JUMP_PASSED REG_FIELD_PREP(GS_BOOTROM_MASK, 0x76) +#define GS_MIA_IN_RESET REG_BIT(0) + +#define BOOT_HASH_CHK XE_REG(0xc010) +#define GUC_BOOT_UKERNEL_VALID REG_BIT(31) + +#define GUC_HEADER_INFO XE_REG(0xc014) + +#define GUC_WOPCM_SIZE XE_REG(0xc050) +#define GUC_WOPCM_SIZE_MASK REG_GENMASK(31, 12) +#define GUC_WOPCM_SIZE_LOCKED REG_BIT(0) + +#define GUC_SHIM_CONTROL XE_REG(0xc064) +#define GUC_MOCS_INDEX_MASK REG_GENMASK(27, 24) +#define GUC_SHIM_WC_ENABLE REG_BIT(21) +#define GUC_ENABLE_MIA_CLOCK_GATING REG_BIT(15) +#define GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA REG_BIT(10) +#define GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA REG_BIT(9) +#define GUC_MSGCH_ENABLE REG_BIT(4) +#define GUC_ENABLE_MIA_CACHING REG_BIT(2) +#define GUC_ENABLE_READ_CACHE_LOGIC REG_BIT(1) +#define GUC_DISABLE_SRAM_INIT_TO_ZEROES REG_BIT(0) + +#define SOFT_SCRATCH(n) XE_REG(0xc180 + (n) * 4) +#define SOFT_SCRATCH_COUNT 16 + +#define HUC_KERNEL_LOAD_INFO XE_REG(0xc1dc) +#define HUC_LOAD_SUCCESSFUL REG_BIT(0) + +#define UOS_RSA_SCRATCH(i) XE_REG(0xc200 + (i) * 4) +#define UOS_RSA_SCRATCH_COUNT 64 + +#define DMA_ADDR_0_LOW XE_REG(0xc300) +#define DMA_ADDR_0_HIGH XE_REG(0xc304) +#define DMA_ADDR_1_LOW XE_REG(0xc308) +#define DMA_ADDR_1_HIGH XE_REG(0xc30c) +#define DMA_ADDR_SPACE_MASK REG_GENMASK(20, 16) +#define DMA_ADDRESS_SPACE_WOPCM REG_FIELD_PREP(DMA_ADDR_SPACE_MASK, 7) +#define DMA_ADDRESS_SPACE_GGTT REG_FIELD_PREP(DMA_ADDR_SPACE_MASK, 8) +#define DMA_COPY_SIZE XE_REG(0xc310) +#define DMA_CTRL XE_REG(0xc314) +#define HUC_UKERNEL REG_BIT(9) +#define UOS_MOVE REG_BIT(4) +#define START_DMA REG_BIT(0) +#define DMA_GUC_WOPCM_OFFSET XE_REG(0xc340) +#define GUC_WOPCM_OFFSET_SHIFT 14 +#define GUC_WOPCM_OFFSET_MASK REG_GENMASK(31, GUC_WOPCM_OFFSET_SHIFT) +#define HUC_LOADING_AGENT_GUC REG_BIT(1) +#define GUC_WOPCM_OFFSET_VALID REG_BIT(0) + +#define GUC_SRAM_STATUS XE_REG(0xc398) +#define GUC_SRAM_HANDLING_MASK REG_GENMASK(8, 7) + +#define GUC_MAX_IDLE_COUNT XE_REG(0xc3e4) +#define GUC_IDLE_FLOW_DISABLE REG_BIT(31) +#define GUC_PMTIMESTAMP_LO XE_REG(0xc3e8) +#define GUC_PMTIMESTAMP_HI XE_REG(0xc3ec) + +#define GUC_SEND_INTERRUPT XE_REG(0xc4c8) +#define GUC_SEND_TRIGGER REG_BIT(0) + +#define GUC_INTR_CHICKEN XE_REG(0xc50c) +#define DISABLE_SIGNALING_ENGINES REG_BIT(1) + +#define GUC_BCS_RCS_IER XE_REG(0xc550) +#define GUC_VCS2_VCS1_IER XE_REG(0xc554) +#define GUC_WD_VECS_IER XE_REG(0xc558) +#define GUC_PM_P24C_IER XE_REG(0xc55c) + +#define GUC_TLB_INV_CR XE_REG(0xcee8) +#define GUC_TLB_INV_CR_INVALIDATE REG_BIT(0) + +#define HUC_STATUS2 XE_REG(0xd3b0) +#define HUC_FW_VERIFIED REG_BIT(7) + +#define GT_PM_CONFIG XE_REG(0x13816c) +#define GT_DOORBELL_ENABLE REG_BIT(0) + +#define GUC_HOST_INTERRUPT XE_REG(0x1901f0, XE_REG_OPTION_VF) + +#define VF_SW_FLAG(n) XE_REG(0x190240 + (n) * 4, XE_REG_OPTION_VF) +#define VF_SW_FLAG_COUNT 4 + +#define MED_GUC_HOST_INTERRUPT XE_REG(0x190304, XE_REG_OPTION_VF) + +#define MED_VF_SW_FLAG(n) XE_REG(0x190310 + (n) * 4, XE_REG_OPTION_VF) +#define MED_VF_SW_FLAG_COUNT 4 + +#define GUC_TLB_INV_CR XE_REG(0xcee8) +#define GUC_TLB_INV_CR_INVALIDATE REG_BIT(0) +#define PVC_GUC_TLB_INV_DESC0 XE_REG(0xcf7c) +#define PVC_GUC_TLB_INV_DESC0_VALID REG_BIT(0) +#define PVC_GUC_TLB_INV_DESC1 XE_REG(0xcf80) +#define PVC_GUC_TLB_INV_DESC1_INVALIDATE REG_BIT(6) + +/* GuC Interrupt Vector */ +#define GUC_INTR_GUC2HOST REG_BIT(15) +#define GUC_INTR_EXEC_ERROR REG_BIT(14) +#define GUC_INTR_DISPLAY_EVENT REG_BIT(13) +#define GUC_INTR_SEM_SIG REG_BIT(12) +#define GUC_INTR_IOMMU2GUC REG_BIT(11) +#define GUC_INTR_DOORBELL_RANG REG_BIT(10) +#define GUC_INTR_DMA_DONE REG_BIT(9) +#define GUC_INTR_FATAL_ERROR REG_BIT(8) +#define GUC_INTR_NOTIF_ERROR REG_BIT(7) +#define GUC_INTR_SW_INT_6 REG_BIT(6) +#define GUC_INTR_SW_INT_5 REG_BIT(5) +#define GUC_INTR_SW_INT_4 REG_BIT(4) +#define GUC_INTR_SW_INT_3 REG_BIT(3) +#define GUC_INTR_SW_INT_2 REG_BIT(2) +#define GUC_INTR_SW_INT_1 REG_BIT(1) +#define GUC_INTR_SW_INT_0 REG_BIT(0) + +#define GUC_NUM_DOORBELLS 256 + +/* format of the HW-monitored doorbell cacheline */ +struct guc_doorbell_info { + u32 db_status; +#define GUC_DOORBELL_DISABLED 0 +#define GUC_DOORBELL_ENABLED 1 + + u32 cookie; + u32 reserved[14]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h new file mode 100644 index 000000000..046e1756c --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_HW_ERROR_REGS_H_ +#define _XE_HW_ERROR_REGS_H_ + +#define HEC_UNCORR_ERR_STATUS(base) XE_REG((base) + 0x118) +#define UNCORR_FW_REPORTED_ERR REG_BIT(6) + +#define HEC_UNCORR_FW_ERR_DW0(base) XE_REG((base) + 0x124) + +#define ERR_STAT_GT_COR 0x100160 +#define EU_GRF_COR_ERR REG_BIT(15) +#define EU_IC_COR_ERR REG_BIT(14) +#define SLM_COR_ERR REG_BIT(13) +#define GUC_COR_ERR REG_BIT(1) + +#define ERR_STAT_GT_NONFATAL 0x100164 +#define ERR_STAT_GT_FATAL 0x100168 +#define EU_GRF_FAT_ERR REG_BIT(15) +#define SLM_FAT_ERR REG_BIT(13) +#define GUC_FAT_ERR REG_BIT(6) +#define FPU_FAT_ERR REG_BIT(3) + +#define ERR_STAT_GT_REG(x) XE_REG(_PICK_EVEN((x), \ + ERR_STAT_GT_COR, \ + ERR_STAT_GT_NONFATAL)) + +#define PVC_COR_ERR_MASK (GUC_COR_ERR | SLM_COR_ERR | \ + EU_IC_COR_ERR | EU_GRF_COR_ERR) + +#define PVC_FAT_ERR_MASK (FPU_FAT_ERR | GUC_FAT_ERR | \ + EU_GRF_FAT_ERR | SLM_FAT_ERR) + +#define DEV_ERR_STAT_NONFATAL 0x100178 +#define DEV_ERR_STAT_CORRECTABLE 0x10017c +#define DEV_ERR_STAT_REG(x) XE_REG(_PICK_EVEN((x), \ + DEV_ERR_STAT_CORRECTABLE, \ + DEV_ERR_STAT_NONFATAL)) + +#define XE_CSC_ERROR 17 +#define XE_SOC_ERROR 16 +#define XE_GT_ERROR 0 + +#define ERR_STAT_GT_FATAL_VECTOR_0 0x100260 +#define ERR_STAT_GT_FATAL_VECTOR_1 0x100264 + +#define ERR_STAT_GT_FATAL_VECTOR_REG(x) XE_REG(_PICK_EVEN((x), \ + ERR_STAT_GT_FATAL_VECTOR_0, \ + ERR_STAT_GT_FATAL_VECTOR_1)) + +#define ERR_STAT_GT_COR_VECTOR_0 0x1002a0 +#define ERR_STAT_GT_COR_VECTOR_1 0x1002a4 + +#define ERR_STAT_GT_COR_VECTOR_REG(x) XE_REG(_PICK_EVEN((x), \ + ERR_STAT_GT_COR_VECTOR_0, \ + ERR_STAT_GT_COR_VECTOR_1)) + +#define ERR_STAT_GT_VECTOR_REG(hw_err, x) (hw_err == HARDWARE_ERROR_CORRECTABLE ? \ + ERR_STAT_GT_COR_VECTOR_REG(x) : \ + ERR_STAT_GT_FATAL_VECTOR_REG(x)) + +#define SOC_PVC_MASTER_BASE 0x282000 +#define SOC_PVC_SLAVE_BASE 0x283000 + +#define SOC_GCOERRSTS 0x200 +#define SOC_GNFERRSTS 0x210 +#define SOC_GLOBAL_ERR_STAT_REG(base, x) XE_REG(_PICK_EVEN((x), \ + (base) + SOC_GCOERRSTS, \ + (base) + SOC_GNFERRSTS)) +#define SOC_SLAVE_IEH REG_BIT(1) +#define SOC_IEH0_LOCAL_ERR_STATUS REG_BIT(0) +#define SOC_IEH1_LOCAL_ERR_STATUS REG_BIT(0) + +#define SOC_GSYSEVTCTL 0x264 +#define SOC_GSYSEVTCTL_REG(master, slave, x) XE_REG(_PICK_EVEN((x), \ + (master) + SOC_GSYSEVTCTL, \ + (slave) + SOC_GSYSEVTCTL)) + +#define SOC_LERRUNCSTS 0x280 +#define SOC_LERRCORSTS 0x294 +#define SOC_LOCAL_ERR_STAT_REG(base, hw_err) XE_REG(hw_err == HARDWARE_ERROR_CORRECTABLE ? \ + (base) + SOC_LERRCORSTS : \ + (base) + SOC_LERRUNCSTS) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_i2c_regs.h b/drivers/gpu/drm/xe/regs/xe_i2c_regs.h new file mode 100644 index 000000000..37550e4a2 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_i2c_regs.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef _XE_I2C_REGS_H_ +#define _XE_I2C_REGS_H_ + +#include + +#include "xe_reg_defs.h" +#include "xe_regs.h" + +#define I2C_BRIDGE_OFFSET (SOC_BASE + 0xd9000) +#define I2C_CONFIG_SPACE_OFFSET (SOC_BASE + 0xf6000) +#define I2C_MEM_SPACE_OFFSET (SOC_BASE + 0xf7400) + +#define REG_SG_REMAP_ADDR_PREFIX XE_REG(SOC_BASE + 0x0164) +#define REG_SG_REMAP_ADDR_POSTFIX XE_REG(SOC_BASE + 0x0168) + +#define I2C_BRIDGE_PCICFGCTL XE_REG(I2C_BRIDGE_OFFSET + 0x200) +#define ACPI_INTR_EN REG_BIT(1) + +#define I2C_CONFIG_CMD XE_REG(I2C_CONFIG_SPACE_OFFSET + PCI_COMMAND) +#define I2C_CONFIG_PMCSR XE_REG(I2C_CONFIG_SPACE_OFFSET + 0x84) + +#define I2C_REG(reg) XE_REG((reg) + I2C_MEM_SPACE_OFFSET) + +#endif /* _XE_I2C_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h new file mode 100644 index 000000000..1d6b976c4 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ +#ifndef _XE_IRQ_REGS_H_ +#define _XE_IRQ_REGS_H_ + +#include "regs/xe_reg_defs.h" + +#define PCU_IRQ_OFFSET 0x444e0 +#define GU_MISC_IRQ_OFFSET 0x444f0 +#define GU_MISC_GSE REG_BIT(27) + +#define DG1_MSTR_TILE_INTR XE_REG(0x190008) +#define DG1_MSTR_IRQ REG_BIT(31) +#define DG1_MSTR_TILE(t) REG_BIT(t) + +#define GFX_MSTR_IRQ XE_REG(0x190010, XE_REG_OPTION_VF) +#define MASTER_IRQ REG_BIT(31) +#define GU_MISC_IRQ REG_BIT(29) +#define ERROR_IRQ(x) REG_BIT(26 + (x)) +#define DISPLAY_IRQ REG_BIT(16) +#define SOC_H2DMEMINT_IRQ REG_BIT(13) +#define I2C_IRQ REG_BIT(12) +#define SYSCTRL_IRQ REG_BIT(11) +#define GT_DW_IRQ(x) REG_BIT(x) + +/* + * Note: Interrupt registers 1900xx are VF accessible only until version 12.50. + * On newer platforms, VFs are using memory-based interrupts instead. + * However, for simplicity we keep this XE_REG_OPTION_VF tag intact. + */ + +#define GT_INTR_DW(x) XE_REG(0x190018 + ((x) * 4), XE_REG_OPTION_VF) +#define INTR_GSC REG_BIT(31) +#define INTR_GUC REG_BIT(25) +#define INTR_MGUC REG_BIT(24) +#define INTR_BCS8 REG_BIT(23) +#define INTR_BCS(x) REG_BIT(15 - (x)) +#define INTR_CCS(x) REG_BIT(4 + (x)) +#define INTR_RCS0 REG_BIT(0) +#define INTR_VECS(x) REG_BIT(31 - (x)) +#define INTR_VCS(x) REG_BIT(x) + +#define RENDER_COPY_INTR_ENABLE XE_REG(0x190030, XE_REG_OPTION_VF) +#define VCS_VECS_INTR_ENABLE XE_REG(0x190034, XE_REG_OPTION_VF) +#define GUC_SG_INTR_ENABLE XE_REG(0x190038, XE_REG_OPTION_VF) +#define ENGINE1_MASK REG_GENMASK(31, 16) +#define ENGINE0_MASK REG_GENMASK(15, 0) +#define GPM_WGBOXPERF_INTR_ENABLE XE_REG(0x19003c, XE_REG_OPTION_VF) +#define CRYPTO_RSVD_INTR_ENABLE XE_REG(0x190040) +#define GUNIT_GSC_INTR_ENABLE XE_REG(0x190044, XE_REG_OPTION_VF) +#define CCS_RSVD_INTR_ENABLE XE_REG(0x190048, XE_REG_OPTION_VF) + +#define INTR_IDENTITY_REG(x) XE_REG(0x190060 + ((x) * 4), XE_REG_OPTION_VF) +#define INTR_DATA_VALID REG_BIT(31) +#define INTR_ENGINE_INSTANCE(x) REG_FIELD_GET(GENMASK(25, 20), x) +#define INTR_ENGINE_CLASS(x) REG_FIELD_GET(GENMASK(18, 16), x) +#define INTR_ENGINE_INTR(x) REG_FIELD_GET(GENMASK(15, 0), x) +#define OTHER_GUC_INSTANCE 0 +#define OTHER_GSC_HECI2_INSTANCE 3 +#define OTHER_KCR_INSTANCE 4 +#define OTHER_GSC_INSTANCE 6 + +#define IIR_REG_SELECTOR(x) XE_REG(0x190070 + ((x) * 4), XE_REG_OPTION_VF) +#define RCS0_RSVD_INTR_MASK XE_REG(0x190090, XE_REG_OPTION_VF) +#define BCS_RSVD_INTR_MASK XE_REG(0x1900a0, XE_REG_OPTION_VF) +#define VCS0_VCS1_INTR_MASK XE_REG(0x1900a8, XE_REG_OPTION_VF) +#define VCS2_VCS3_INTR_MASK XE_REG(0x1900ac, XE_REG_OPTION_VF) +#define VCS4_VCS5_INTR_MASK XE_REG(0x1900b0, XE_REG_OPTION_VF) +#define VCS6_VCS7_INTR_MASK XE_REG(0x1900b4, XE_REG_OPTION_VF) +#define VECS0_VECS1_INTR_MASK XE_REG(0x1900d0, XE_REG_OPTION_VF) +#define VECS2_VECS3_INTR_MASK XE_REG(0x1900d4, XE_REG_OPTION_VF) +#define HECI2_RSVD_INTR_MASK XE_REG(0x1900e4) +#define GUC_SG_INTR_MASK XE_REG(0x1900e8, XE_REG_OPTION_VF) +#define GPM_WGBOXPERF_INTR_MASK XE_REG(0x1900ec, XE_REG_OPTION_VF) +#define CRYPTO_RSVD_INTR_MASK XE_REG(0x1900f0) +#define GUNIT_GSC_INTR_MASK XE_REG(0x1900f4, XE_REG_OPTION_VF) +#define CCS0_CCS1_INTR_MASK XE_REG(0x190100) +#define CCS2_CCS3_INTR_MASK XE_REG(0x190104) +#define XEHPC_BCS1_BCS2_INTR_MASK XE_REG(0x190110) +#define XEHPC_BCS3_BCS4_INTR_MASK XE_REG(0x190114) +#define XEHPC_BCS5_BCS6_INTR_MASK XE_REG(0x190118) +#define XEHPC_BCS7_BCS8_INTR_MASK XE_REG(0x19011c) +#define GT_WAIT_SEMAPHORE_INTERRUPT REG_BIT(11) +#define GT_CONTEXT_SWITCH_INTERRUPT REG_BIT(8) +#define GSC_ER_COMPLETE REG_BIT(5) +#define GT_FLUSH_COMPLETE_INTERRUPT REG_BIT(4) +#define GT_CS_MASTER_ERROR_INTERRUPT REG_BIT(3) +#define GT_COMPUTE_WALKER_INTERRUPT REG_BIT(2) +#define GT_MI_USER_INTERRUPT REG_BIT(0) + +/* irqs for OTHER_KCR_INSTANCE */ +#define KCR_PXP_STATE_TERMINATED_INTERRUPT REG_BIT(1) +#define KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT REG_BIT(2) +#define KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT REG_BIT(3) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h new file mode 100644 index 000000000..4ab86fc36 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_LRC_LAYOUT_H_ +#define _XE_LRC_LAYOUT_H_ + +#define CTX_CONTEXT_CONTROL (0x02 + 1) +#define CTX_RING_HEAD (0x04 + 1) +#define CTX_RING_TAIL (0x06 + 1) +#define CTX_RING_START (0x08 + 1) +#define CTX_RING_CTL (0x0a + 1) +#define CTX_BB_PER_CTX_PTR (0x12 + 1) +#define CTX_CS_INDIRECT_CTX (0x14 + 1) +#define CTX_CS_INDIRECT_CTX_OFFSET (0x16 + 1) +#define CTX_TIMESTAMP (0x22 + 1) +#define CTX_TIMESTAMP_UDW (0x24 + 1) +#define CTX_INDIRECT_RING_STATE (0x26 + 1) +#define CTX_ACC_CTR_THOLD (0x2a + 1) +#define CTX_ASID (0x2e + 1) +#define CTX_PDP0_UDW (0x30 + 1) +#define CTX_PDP0_LDW (0x32 + 1) + +#define CTX_LRM_INT_MASK_ENABLE 0x50 +#define CTX_INT_MASK_ENABLE_REG (CTX_LRM_INT_MASK_ENABLE + 1) +#define CTX_INT_MASK_ENABLE_PTR (CTX_LRM_INT_MASK_ENABLE + 2) +#define CTX_LRI_INT_REPORT_PTR 0x55 +#define CTX_INT_STATUS_REPORT_REG (CTX_LRI_INT_REPORT_PTR + 1) +#define CTX_INT_STATUS_REPORT_PTR (CTX_LRI_INT_REPORT_PTR + 2) +#define CTX_INT_SRC_REPORT_REG (CTX_LRI_INT_REPORT_PTR + 3) +#define CTX_INT_SRC_REPORT_PTR (CTX_LRI_INT_REPORT_PTR + 4) + +#define CTX_CS_INT_VEC_REG 0x5a +#define CTX_CS_INT_VEC_DATA (CTX_CS_INT_VEC_REG + 1) + +#define CTX_QUEUE_TIMESTAMP (0xd0 + 1) +#define CTX_QUEUE_TIMESTAMP_UDW (0xd2 + 1) + +#define INDIRECT_CTX_RING_HEAD (0x02 + 1) +#define INDIRECT_CTX_RING_TAIL (0x04 + 1) +#define INDIRECT_CTX_RING_START (0x06 + 1) +#define INDIRECT_CTX_RING_START_UDW (0x08 + 1) +#define INDIRECT_CTX_RING_CTL (0x0a + 1) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_mchbar_regs.h b/drivers/gpu/drm/xe/regs/xe_mchbar_regs.h new file mode 100644 index 000000000..ef2bf9847 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_mchbar_regs.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_MCHBAR_REGS_H_ +#define _XE_MCHBAR_REGS_H_ + +#include "regs/xe_reg_defs.h" + +/* + * MCHBAR mirror. + * + * This mirrors the MCHBAR MMIO space whose location is determined by + * device 0 function 0's pci config register 0x44 or 0x48 and matches it in + * every way. + */ + +#define MCHBAR_MIRROR_BASE_SNB 0x140000 + +#define PCU_CR_PACKAGE_POWER_SKU XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5930) +#define PKG_TDP GENMASK_ULL(14, 0) +#define PKG_MIN_PWR GENMASK_ULL(30, 16) +#define PKG_MAX_PWR GENMASK_ULL(46, 32) +#define PKG_MAX_WIN GENMASK_ULL(54, 48) +#define PKG_MAX_WIN_X GENMASK_ULL(54, 53) +#define PKG_MAX_WIN_Y GENMASK_ULL(52, 48) + + +#define PCU_CR_PACKAGE_POWER_SKU_UNIT XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5938) +#define PKG_PWR_UNIT REG_GENMASK(3, 0) +#define PKG_ENERGY_UNIT REG_GENMASK(12, 8) +#define PKG_TIME_UNIT REG_GENMASK(19, 16) + +#define PCU_CR_PACKAGE_ENERGY_STATUS XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x593c) + +#define PCU_CR_PACKAGE_TEMPERATURE XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5978) +#define TEMP_MASK REG_GENMASK(7, 0) + +#define PCU_CR_PACKAGE_RAPL_LIMIT XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x59a0) +#define PWR_LIM_VAL REG_GENMASK(14, 0) +#define PWR_LIM_EN REG_BIT(15) +#define PWR_LIM REG_GENMASK(15, 0) +#define PWR_LIM_TIME REG_GENMASK(23, 17) +#define PWR_LIM_TIME_X REG_GENMASK(23, 22) +#define PWR_LIM_TIME_Y REG_GENMASK(21, 17) + +#endif /* _XE_MCHBAR_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/regs/xe_mert_regs.h b/drivers/gpu/drm/xe/regs/xe_mert_regs.h new file mode 100644 index 000000000..99e5a26da --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_mert_regs.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_MERT_REGS_H_ +#define _XE_MERT_REGS_H_ + +#include "regs/xe_reg_defs.h" + +#define MERT_LMEM_CFG XE_REG(0x1448b0) + +#define MERT_TLB_CT_INTR_ERR_ID_PORT XE_REG(0x145190) +#define CATERR_VFID REG_GENMASK(16, 9) +#define CATERR_CODES REG_GENMASK(5, 0) +#define CATERR_NO_ERROR 0x00 +#define CATERR_UNMAPPED_GGTT 0x01 +#define CATERR_LMTT_FAULT 0x05 + +#define MERT_TLB_INV_DESC_A XE_REG(0x14cf7c) +#define MERT_TLB_INV_DESC_A_VALID REG_BIT(0) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h new file mode 100644 index 000000000..aa66af7e9 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef __XE_OA_REGS__ +#define __XE_OA_REGS__ + +#define SYS_MEM_LAT_MEASURE XE_REG(0x145194) +#define SYS_MEM_LAT_MEASURE_EN REG_BIT(31) + +#define RPM_CONFIG1 XE_REG(0xd04) +#define GT_NOA_ENABLE REG_BIT(9) + +#define EU_PERF_CNTL0 XE_REG(0xe458) +#define EU_PERF_CNTL4 XE_REG(0xe45c) +#define EU_PERF_CNTL1 XE_REG(0xe558) +#define EU_PERF_CNTL5 XE_REG(0xe55c) +#define EU_PERF_CNTL2 XE_REG(0xe658) +#define EU_PERF_CNTL6 XE_REG(0xe65c) +#define EU_PERF_CNTL3 XE_REG(0xe758) + +#define OA_TLB_INV_CR XE_REG(0xceec) + +/* OAR unit */ +#define OAR_OACONTROL XE_REG(0x2960) +#define OAR_OACONTROL_COUNTER_SEL_MASK REG_GENMASK(3, 1) +#define OAR_OACONTROL_COUNTER_ENABLE REG_BIT(0) + +#define OACTXCONTROL(base) XE_REG((base) + 0x360) +#define OAR_OASTATUS XE_REG(0x2968) +#define OA_COUNTER_RESUME REG_BIT(0) + +/* OAG unit */ +#define OAG_OAGLBCTXCTRL XE_REG(0x2b28) +#define OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK REG_GENMASK(7, 2) +#define OAG_OAGLBCTXCTRL_TIMER_ENABLE REG_BIT(1) +#define OAG_OAGLBCTXCTRL_COUNTER_RESUME REG_BIT(0) + +#define OAG_OAHEADPTR XE_REG(0xdb00) +#define OAG_OAHEADPTR_MASK REG_GENMASK(31, 6) +#define OAG_OATAILPTR XE_REG(0xdb04) +#define OAG_OATAILPTR_MASK REG_GENMASK(31, 6) + +#define OAG_OABUFFER XE_REG(0xdb08) +#define OABUFFER_SIZE_MASK REG_GENMASK(5, 3) +#define OAG_OABUFFER_MEMORY_SELECT REG_BIT(0) /* 0: PPGTT, 1: GGTT */ + +#define OAG_OACONTROL XE_REG(0xdaf4) +#define OAG_OACONTROL_OA_PES_DISAG_EN REG_GENMASK(27, 22) +#define OAG_OACONTROL_OA_CCS_SELECT_MASK REG_GENMASK(18, 16) +#define OAG_OACONTROL_OA_COUNTER_SEL_MASK REG_GENMASK(4, 2) +#define OAG_OACONTROL_OA_COUNTER_ENABLE REG_BIT(0) +/* Common to all OA units */ +#define OA_OACONTROL_REPORT_BC_MASK REG_GENMASK(9, 9) +#define OA_OACONTROL_COUNTER_SIZE_MASK REG_GENMASK(8, 8) +#define OAG_OACONTROL_USED_BITS \ + (OAG_OACONTROL_OA_PES_DISAG_EN | OAG_OACONTROL_OA_CCS_SELECT_MASK | \ + OAG_OACONTROL_OA_COUNTER_SEL_MASK | OAG_OACONTROL_OA_COUNTER_ENABLE | \ + OA_OACONTROL_REPORT_BC_MASK | OA_OACONTROL_COUNTER_SIZE_MASK) + +#define OAG_OA_DEBUG XE_REG(0xdaf8, XE_REG_OPTION_MASKED) +#define OAG_OA_DEBUG_DISABLE_MMIO_TRG REG_BIT(14) +#define OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL REG_BIT(13) +#define OAG_OA_DEBUG_BUF_SIZE_SELECT REG_BIT(12) +#define OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL REG_BIT(8) +#define OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL REG_BIT(7) +#define OAG_OA_DEBUG_INCLUDE_CLK_RATIO REG_BIT(6) +#define OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS REG_BIT(5) +#define OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS REG_BIT(1) + +#define OAG_OASTATUS XE_REG(0xdafc) +#define OASTATUS_MMIO_TRG_Q_FULL REG_BIT(6) +#define OASTATUS_COUNTER_OVERFLOW REG_BIT(2) +#define OASTATUS_BUFFER_OVERFLOW REG_BIT(1) +#define OASTATUS_REPORT_LOST REG_BIT(0) +#define OAG_MMIOTRIGGER XE_REG(0xdb1c) +/* OAC unit */ +#define OAC_OACONTROL XE_REG(0x15114) + +/* OAM unit */ +#define OAM_HEAD_POINTER_OFFSET (0x1a0) +#define OAM_TAIL_POINTER_OFFSET (0x1a4) +#define OAM_BUFFER_OFFSET (0x1a8) +#define OAM_CONTEXT_CONTROL_OFFSET (0x1bc) +#define OAM_CONTROL_OFFSET (0x194) +#define OAM_CONTROL_COUNTER_SEL_MASK REG_GENMASK(3, 1) +#define OAM_OACONTROL_USED_BITS \ + (OAM_CONTROL_COUNTER_SEL_MASK | OAG_OACONTROL_OA_COUNTER_ENABLE) +#define OAM_DEBUG_OFFSET (0x198) +#define OAM_STATUS_OFFSET (0x19c) +#define OAM_MMIO_TRG_OFFSET (0x1d0) + +#define OAM_HEAD_POINTER(base) XE_REG((base) + OAM_HEAD_POINTER_OFFSET) +#define OAM_TAIL_POINTER(base) XE_REG((base) + OAM_TAIL_POINTER_OFFSET) +#define OAM_BUFFER(base) XE_REG((base) + OAM_BUFFER_OFFSET) +#define OAM_CONTEXT_CONTROL(base) XE_REG((base) + OAM_CONTEXT_CONTROL_OFFSET) +#define OAM_CONTROL(base) XE_REG((base) + OAM_CONTROL_OFFSET) +#define OAM_DEBUG(base) XE_REG((base) + OAM_DEBUG_OFFSET) +#define OAM_STATUS(base) XE_REG((base) + OAM_STATUS_OFFSET) +#define OAM_MMIO_TRG(base) XE_REG((base) + OAM_MMIO_TRG_OFFSET) + +#define OAM_COMPRESSION_T3_CONTROL XE_REG(0x1c2e00) +#define OAM_LAT_MEASURE_ENABLE REG_BIT(4) + +/* Actual address is MEDIA_GT_GSI_OFFSET + the base addr below */ +#define XE_OAM_SAG_BASE 0x13000 +#define XE_OAM_SCMI_0_BASE 0x14000 +#define XE_OAM_SCMI_1_BASE 0x14800 +#define XE_OAM_SAG_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SAG_BASE) +#define XE_OAM_SCMI_0_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_0_BASE) +#define XE_OAM_SCMI_1_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_1_BASE) + +#define OAMERT_CONTROL XE_REG(0x1453a0) +#define OAMERT_DEBUG XE_REG(0x1453a4) +#define OAMERT_STATUS XE_REG(0x1453a8) +#define OAMERT_HEAD_POINTER XE_REG(0x1453ac) +#define OAMERT_TAIL_POINTER XE_REG(0x1453b0) +#define OAMERT_BUFFER XE_REG(0x1453b4) +#define OAMERT_CONTEXT_CONTROL XE_REG(0x1453c8) +#define OAMERT_MMIO_TRG XE_REG(0x1453cc) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h new file mode 100644 index 000000000..c63b409d7 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_PCODE_REGS_H_ +#define _XE_PCODE_REGS_H_ + +#include "regs/xe_reg_defs.h" + +/* + * This file contains addresses of PCODE registers visible through GT MMIO space. + */ + +#define PVC_GT0_PACKAGE_ENERGY_STATUS XE_REG(0x281004) +#define PVC_GT0_PACKAGE_RAPL_LIMIT XE_REG(0x281008) +#define PVC_GT0_PACKAGE_POWER_SKU_UNIT XE_REG(0x281068) +#define PVC_GT0_PLATFORM_ENERGY_STATUS XE_REG(0x28106c) +#define PVC_GT0_PACKAGE_POWER_SKU XE_REG(0x281080) + +#define BMG_FAN_1_SPEED XE_REG(0x138140) +#define BMG_FAN_2_SPEED XE_REG(0x138170) +#define BMG_FAN_3_SPEED XE_REG(0x1381a0) +#define BMG_VRAM_TEMPERATURE_N(n) XE_REG(0x138260 + ((n) * (sizeof(u32)))) +#define BMG_VRAM_TEMPERATURE XE_REG(0x1382c0) +#define TEMP_MASK_VRAM_N REG_GENMASK(30, 8) +#define TEMP_SIGN_MASK REG_BIT(31) +#define BMG_PACKAGE_TEMPERATURE XE_REG(0x138434) + +#define CRI_PACKAGE_ENERGY_STATUS XE_REG(0x138120) +#define CRI_PLATFORM_ENERGY_STATUS XE_REG(0x138458) + +#endif /* _XE_PCODE_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/regs/xe_pmt.h b/drivers/gpu/drm/xe/regs/xe_pmt.h new file mode 100644 index 000000000..a62ab05c6 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_pmt.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ +#ifndef _XE_PMT_H_ +#define _XE_PMT_H_ + +#include "xe_regs.h" + +#define BMG_PMT_BASE_OFFSET 0xDB000 +#define BMG_DISCOVERY_OFFSET (SOC_BASE + BMG_PMT_BASE_OFFSET) + +#define PUNIT_TELEMETRY_GUID XE_REG(BMG_DISCOVERY_OFFSET + 0x4) +#define BMG_ENERGY_STATUS_PMT_OFFSET (0x30) +#define ENERGY_PKG REG_GENMASK64(31, 0) +#define ENERGY_CARD REG_GENMASK64(63, 32) + +#define PUNIT_VERSION_OFFSET 0xA0 + +#define BMG_TELEMETRY_BASE_OFFSET 0xE0000 +#define BMG_TELEMETRY_OFFSET (SOC_BASE + BMG_TELEMETRY_BASE_OFFSET) + +#define BMG_MODS_RESIDENCY_OFFSET (0x4D0) +#define BMG_G2_RESIDENCY_OFFSET (0x530) +#define BMG_G6_RESIDENCY_OFFSET (0x538) +#define BMG_G7_RESIDENCY_OFFSET (0x4B0) +#define BMG_G8_RESIDENCY_OFFSET (0x540) +#define BMG_G10_RESIDENCY_OFFSET (0x548) + +#define BMG_PCIE_LINK_L0_RESIDENCY_OFFSET (0x570) +#define BMG_PCIE_LINK_L1_RESIDENCY_OFFSET (0x578) +#define BMG_PCIE_LINK_L1_2_RESIDENCY_OFFSET (0x580) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_pxp_regs.h b/drivers/gpu/drm/xe/regs/xe_pxp_regs.h new file mode 100644 index 000000000..aa158938b --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_pxp_regs.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2024, Intel Corporation. All rights reserved. + */ + +#ifndef __XE_PXP_REGS_H__ +#define __XE_PXP_REGS_H__ + +#include "regs/xe_regs.h" + +/* The following registers are only valid on platforms with a media GT */ + +/* KCR enable/disable control */ +#define KCR_INIT XE_REG(0x3860f0) +#define KCR_INIT_ALLOW_DISPLAY_ME_WRITES REG_BIT(14) + +/* KCR hwdrm session in play status 0-31 */ +#define KCR_SIP XE_REG(0x386260) + +/* PXP global terminate register for session termination */ +#define KCR_GLOBAL_TERMINATE XE_REG(0x3860f8) + +#endif /* __XE_PXP_REGS_H__ */ diff --git a/drivers/gpu/drm/xe/regs/xe_reg_defs.h b/drivers/gpu/drm/xe/regs/xe_reg_defs.h new file mode 100644 index 000000000..27ac0bf1f --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_reg_defs.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_REG_DEFS_H_ +#define _XE_REG_DEFS_H_ + +#include +#include + +#include +#include +#include + +/** + * XE_REG_ADDR_MAX - The upper limit on MMIO register address + * + * This macro specifies the upper limit (not inclusive) on MMIO register offset + * supported by struct xe_reg and functions based on struct xe_mmio. + * + * Currently this is defined as 4 MiB. + */ +#define XE_REG_ADDR_MAX SZ_4M + +/** + * struct xe_reg - Register definition + * + * Register definition to be used by the individual register. Although the same + * definition is used for xe_reg and xe_reg_mcr, they use different internal + * APIs for accesses. + */ +struct xe_reg { + union { + struct { + /** @addr: address */ + u32 addr:const_ilog2(XE_REG_ADDR_MAX); + /** + * @masked: register is "masked", with upper 16bits used + * to identify the bits that are updated on the lower + * bits + */ + u32 masked:1; + /** + * @mcr: register is multicast/replicated in the + * hardware and needs special handling. Any register + * with this set should also use a type of xe_reg_mcr_t. + * It's only here so the few places that deal with MCR + * registers specially (xe_sr.c) and tests using the raw + * value can inspect it. + */ + u32 mcr:1; + /** + * @vf: register is accessible from the Virtual Function. + */ + u32 vf:1; + }; + /** @raw: Raw value with both address and options */ + u32 raw; + }; +}; +static_assert(sizeof(struct xe_reg) == sizeof(u32)); + +/** + * struct xe_reg_mcr - MCR register definition + * + * MCR register is the same as a regular register, but uses another type since + * the internal API used for accessing them is different: it's never correct to + * use regular MMIO access. + */ +struct xe_reg_mcr { + /** @__reg: The register */ + struct xe_reg __reg; +}; + + +/** + * XE_REG_OPTION_MASKED - Register is "masked", with upper 16 bits marking the + * written bits on the lower 16 bits. + * + * It only applies to registers explicitly marked in bspec with + * "Access: Masked". Registers with this option can have write operations to + * specific lower bits by setting the corresponding upper bits. Other bits will + * not be affected. This allows register writes without needing a RMW cycle and + * without caching in software the register value. + * + * Example: a write with value 0x00010001 will set bit 0 and all other bits + * retain their previous values. + * + * To be used with XE_REG(). XE_REG_MCR() and XE_REG_INITIALIZER() + */ +#define XE_REG_OPTION_MASKED .masked = 1 + +/** + * XE_REG_OPTION_VF - Register is "VF" accessible. + * + * To be used with XE_REG() and XE_REG_INITIALIZER(). + */ +#define XE_REG_OPTION_VF .vf = 1 + +/** + * XE_REG_INITIALIZER - Initializer for xe_reg_t. + * @r_: Register offset + * @...: Additional options like access mode. See struct xe_reg for available + * options. + * + * Register field is mandatory, and additional options may be passed as + * arguments. Usually ``XE_REG()`` should be preferred since it creates an + * object of the right type. However when initializing static const storage, + * where a compound statement is not allowed, this can be used instead. + */ +#define XE_REG_INITIALIZER(r_, ...) { .addr = r_, __VA_ARGS__ } + + +/** + * XE_REG - Create a struct xe_reg from offset and additional flags + * @r_: Register offset + * @...: Additional options like access mode. See struct xe_reg for available + * options. + */ +#define XE_REG(r_, ...) ((const struct xe_reg)XE_REG_INITIALIZER(r_, ##__VA_ARGS__)) + +/** + * XE_REG_MCR - Create a struct xe_reg_mcr from offset and additional flags + * @r_: Register offset + * @...: Additional options like access mode. See struct xe_reg for available + * options. + */ +#define XE_REG_MCR(r_, ...) ((const struct xe_reg_mcr){ \ + .__reg = XE_REG_INITIALIZER(r_, ##__VA_ARGS__, .mcr = 1) \ + }) + +static inline bool xe_reg_is_valid(struct xe_reg r) +{ + return r.addr; +} + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_regs.h b/drivers/gpu/drm/xe/regs/xe_regs.h new file mode 100644 index 000000000..ef4746b7b --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_regs.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ +#ifndef _XE_REGS_H_ +#define _XE_REGS_H_ + +#include "regs/xe_reg_defs.h" + +#define SWF_SCRATCHPAD(_idx) XE_REG(0x4f000 + (_idx) * 4) + +#define SOC_BASE 0x280000 + +#define GU_CNTL_PROTECTED XE_REG(0x10100C) +#define DRIVERINT_FLR_DIS REG_BIT(31) + +#define GU_CNTL XE_REG(0x101010) +#define LMEM_INIT REG_BIT(7) +#define DRIVERFLR REG_BIT(31) + +#define XEHP_CLOCK_GATE_DIS XE_REG(0x101014) +#define SGSI_SIDECLK_DIS REG_BIT(17) + +#define GU_DEBUG XE_REG(0x101018) +#define DRIVERFLR_STATUS REG_BIT(31) + +#define VIRTUAL_CTRL_REG XE_REG(0x10108c) +#define GUEST_GTT_UPDATE_EN REG_BIT(8) + +#define XEHP_MTCFG_ADDR XE_REG(0x101800) +#define TILE_COUNT REG_GENMASK(15, 8) + +#define GGC XE_REG(0x108040) +#define GMS_MASK REG_GENMASK(15, 8) +#define GGMS_MASK REG_GENMASK(7, 6) + +#define DSMBASE XE_REG(0x1080C0) +#define BDSM_MASK REG_GENMASK64(63, 20) + +#define GSMBASE XE_REG(0x108100) + +#define STOLEN_RESERVED XE_REG(0x1082c0) +#define WOPCM_SIZE_MASK REG_GENMASK64(9, 7) + +#define SG_TILE_ADDR_RANGE(_idx) XE_REG(0x1083a0 + (_idx) * 4) + +#define MTL_RP_STATE_CAP XE_REG(0x138000) + +#define MTL_GT_RPA_FREQUENCY XE_REG(0x138008) +#define MTL_GT_RPE_FREQUENCY XE_REG(0x13800c) + +#define MTL_MEDIAP_STATE_CAP XE_REG(0x138020) +#define MTL_RPN_CAP_MASK REG_GENMASK(24, 16) +#define MTL_RP0_CAP_MASK REG_GENMASK(8, 0) + +#define MTL_MPA_FREQUENCY XE_REG(0x138028) +#define MTL_RPA_MASK REG_GENMASK(8, 0) + +#define MTL_MPE_FREQUENCY XE_REG(0x13802c) +#define MTL_RPE_MASK REG_GENMASK(8, 0) + +#define VF_CAP_REG XE_REG(0x1901f8, XE_REG_OPTION_VF) +#define VF_CAP REG_BIT(0) + +#define PVC_RP_STATE_CAP XE_REG(0x281014) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_soc_remapper_regs.h b/drivers/gpu/drm/xe/regs/xe_soc_remapper_regs.h new file mode 100644 index 000000000..be0eb37e7 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_soc_remapper_regs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ +#ifndef _XE_SOC_REMAPPER_REGS_H_ +#define _XE_SOC_REMAPPER_REGS_H_ + +#include "xe_regs.h" + +#define SG_REMAP_INDEX1 XE_REG(SOC_BASE + 0x08) +#define SG_REMAP_TELEM_MASK REG_GENMASK(31, 24) +#define SG_REMAP_SYSCTRL_MASK REG_GENMASK(23, 16) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h new file mode 100644 index 000000000..59f3f3ec5 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_REGS_H_ +#define _XE_SYSCTRL_REGS_H_ + +#include "xe_regs.h" + +#define SYSCTRL_BASE_OFFSET 0xdb000 +#define SYSCTRL_BASE (SOC_BASE + SYSCTRL_BASE_OFFSET) +#define SYSCTRL_MAILBOX_INDEX 0x03 +#define SYSCTRL_BAR_LENGTH 0x1000 + +#define SYSCTRL_MB_CTRL XE_REG(0x10) +#define SYSCTRL_MB_CTRL_RUN_BUSY REG_BIT(31) +#define SYSCTRL_MB_CTRL_IRQ REG_BIT(30) +#define SYSCTRL_MB_CTRL_RUN_BUSY_OUT REG_BIT(29) +#define SYSCTRL_MB_CTRL_PARAM3_MASK REG_GENMASK(28, 24) +#define SYSCTRL_MB_CTRL_PARAM2_MASK REG_GENMASK(23, 16) +#define SYSCTRL_MB_CTRL_PARAM1_MASK REG_GENMASK(15, 8) +#define SYSCTRL_MB_CTRL_COMMAND_MASK REG_GENMASK(7, 0) +#define SYSCTRL_MB_CTRL_CMD REG_FIELD_PREP(SYSCTRL_MB_CTRL_COMMAND_MASK, 5) + +#define SYSCTRL_MB_DATA0 XE_REG(0x14) +#define SYSCTRL_MB_DATA1 XE_REG(0x18) +#define SYSCTRL_MB_DATA2 XE_REG(0x1c) +#define SYSCTRL_MB_DATA3 XE_REG(0x20) + +#define SYSCTRL_FRAME_PHASE REG_BIT(24) +#define SYSCTRL_FRAME_CURRENT_MASK REG_GENMASK(21, 16) +#define SYSCTRL_FRAME_TOTAL_MASK REG_GENMASK(13, 8) +#define SYSCTRL_FRAME_COMMAND_MASK REG_GENMASK(7, 0) + +#endif diff --git a/drivers/gpu/drm/xe/tests/Makefile b/drivers/gpu/drm/xe/tests/Makefile new file mode 100644 index 000000000..f7aa47f11 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 + +# "live" kunit tests +obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_live_test.o +xe_live_test-y = xe_live_test_mod.o + +# Normal kunit tests +obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_test.o +xe_test-y = xe_test_mod.o \ + xe_args_test.o \ + xe_pci_test.o \ + xe_rtp_tables_test.o \ + xe_rtp_test.o \ + xe_wa_test.o diff --git a/drivers/gpu/drm/xe/tests/xe_args_test.c b/drivers/gpu/drm/xe/tests/xe_args_test.c new file mode 100644 index 000000000..2687a1b05 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_args_test.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include "xe_args.h" + +static void call_args_example(struct kunit *test) +{ +#define foo X, Y, Z, Q +#define bar COUNT_ARGS(foo) +#define buz CALL_ARGS(COUNT_ARGS, foo) + + KUNIT_EXPECT_EQ(test, bar, 1); + KUNIT_EXPECT_EQ(test, buz, 4); + +#undef foo +#undef bar +#undef buz +} + +static void drop_first_arg_example(struct kunit *test) +{ +#define foo X, Y, Z, Q +#define bar CALL_ARGS(COUNT_ARGS, DROP_FIRST_ARG(foo)) + + KUNIT_EXPECT_EQ(test, bar, 3); + +#undef foo +#undef bar +} + +static void first_arg_example(struct kunit *test) +{ + int X = 1; + +#define foo X, Y, Z, Q +#define bar FIRST_ARG(foo) + + KUNIT_EXPECT_EQ(test, bar, X); + KUNIT_EXPECT_STREQ(test, __stringify(bar), "X"); + +#undef foo +#undef bar +} + +static void last_arg_example(struct kunit *test) +{ + int Q = 1; + +#define foo X, Y, Z, Q +#define bar LAST_ARG(foo) + + KUNIT_EXPECT_EQ(test, bar, Q); + KUNIT_EXPECT_STREQ(test, __stringify(bar), "Q"); + +#undef foo +#undef bar +} + +static void pick_arg_example(struct kunit *test) +{ + int Y = 1, Z = 2; + +#define foo X, Y, Z, Q +#define bar PICK_ARG(2, foo) +#define buz PICK_ARG3(foo) + + KUNIT_EXPECT_EQ(test, bar, Y); + KUNIT_EXPECT_STREQ(test, __stringify(bar), "Y"); + KUNIT_EXPECT_EQ(test, buz, Z); + KUNIT_EXPECT_STREQ(test, __stringify(buz), "Z"); + +#undef foo +#undef bar +#undef buz +} + +static void if_args_example(struct kunit *test) +{ + enum { Z = 1, Q }; + +#define foo X, Y +#define bar IF_ARGS(Z, Q, foo) +#define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo))) + + KUNIT_EXPECT_EQ(test, bar, Z); + KUNIT_EXPECT_EQ(test, buz, Q); + KUNIT_EXPECT_STREQ(test, __stringify(bar), "Z"); + KUNIT_EXPECT_STREQ(test, __stringify(buz), "Q"); + +#undef foo +#undef bar +#undef buz +} + +static void sep_comma_example(struct kunit *test) +{ +#define foo(f) f(X) f(Y) f(Z) f(Q) +#define bar DROP_FIRST_ARG(foo(ARGS_SEP_COMMA __stringify)) +#define buz CALL_ARGS(COUNT_ARGS, DROP_FIRST_ARG(foo(ARGS_SEP_COMMA))) + + static const char * const a[] = { bar }; + + KUNIT_EXPECT_STREQ(test, a[0], "X"); + KUNIT_EXPECT_STREQ(test, a[1], "Y"); + KUNIT_EXPECT_STREQ(test, a[2], "Z"); + KUNIT_EXPECT_STREQ(test, a[3], "Q"); + + KUNIT_EXPECT_EQ(test, buz, 4); + +#undef foo +#undef bar +#undef buz +} + +#define NO_ARGS +#define FOO_ARGS X, Y, Z, Q +#define MAX_ARGS -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12 + +static void count_args_test(struct kunit *test) +{ + int count; + + /* COUNT_ARGS() counts to 12 */ + + count = COUNT_ARGS(); + KUNIT_EXPECT_EQ(test, count, 0); + + count = COUNT_ARGS(1); + KUNIT_EXPECT_EQ(test, count, 1); + + count = COUNT_ARGS(a, b, c, d, e); + KUNIT_EXPECT_EQ(test, count, 5); + + count = COUNT_ARGS(a, b, c, d, e, f, g, h, i, j, k, l); + KUNIT_EXPECT_EQ(test, count, 12); + + /* COUNT_ARGS() does not expand params */ + + count = COUNT_ARGS(NO_ARGS); + KUNIT_EXPECT_EQ(test, count, 1); + + count = COUNT_ARGS(FOO_ARGS); + KUNIT_EXPECT_EQ(test, count, 1); +} + +static void call_args_test(struct kunit *test) +{ + int count; + + count = CALL_ARGS(COUNT_ARGS, NO_ARGS); + KUNIT_EXPECT_EQ(test, count, 0); + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, NO_ARGS), 0); + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, FOO_ARGS), 4); + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, FOO_ARGS, FOO_ARGS), 8); + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, MAX_ARGS), 12); +} + +static void drop_first_arg_test(struct kunit *test) +{ + int Y = -2, Z = -3, Q = -4; + int a[] = { DROP_FIRST_ARG(FOO_ARGS) }; + + KUNIT_EXPECT_EQ(test, DROP_FIRST_ARG(0, -1), -1); + KUNIT_EXPECT_EQ(test, DROP_FIRST_ARG(DROP_FIRST_ARG(0, -1, -2)), -2); + + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, DROP_FIRST_ARG(FOO_ARGS)), 3); + KUNIT_EXPECT_EQ(test, DROP_FIRST_ARG(DROP_FIRST_ARG(DROP_FIRST_ARG(FOO_ARGS))), -4); + KUNIT_EXPECT_EQ(test, a[0], -2); + KUNIT_EXPECT_EQ(test, a[1], -3); + KUNIT_EXPECT_EQ(test, a[2], -4); + +#define foo DROP_FIRST_ARG(FOO_ARGS) +#define bar DROP_FIRST_ARG(DROP_FIRST_ARG(FOO_ARGS)) +#define buz DROP_FIRST_ARG(DROP_FIRST_ARG(DROP_FIRST_ARG(FOO_ARGS))) + + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, foo), 3); + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, bar), 2); + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, buz), 1); + KUNIT_EXPECT_STREQ(test, __stringify(buz), "Q"); + +#undef foo +#undef bar +#undef buz +} + +static void first_arg_test(struct kunit *test) +{ + int X = -1; + int a[] = { FIRST_ARG(FOO_ARGS) }; + + KUNIT_EXPECT_EQ(test, FIRST_ARG(-1, -2), -1); + + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, FIRST_ARG(FOO_ARGS)), 1); + KUNIT_EXPECT_EQ(test, FIRST_ARG(FOO_ARGS), -1); + KUNIT_EXPECT_EQ(test, a[0], -1); + KUNIT_EXPECT_STREQ(test, __stringify(FIRST_ARG(FOO_ARGS)), "X"); +} + +static void last_arg_test(struct kunit *test) +{ + int Q = -4; + int a[] = { LAST_ARG(FOO_ARGS) }; + + KUNIT_EXPECT_EQ(test, LAST_ARG(-1, -2), -2); + + KUNIT_EXPECT_EQ(test, CALL_ARGS(COUNT_ARGS, LAST_ARG(FOO_ARGS)), 1); + KUNIT_EXPECT_EQ(test, LAST_ARG(FOO_ARGS), -4); + KUNIT_EXPECT_EQ(test, a[0], -4); + KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(FOO_ARGS)), "Q"); + + KUNIT_EXPECT_EQ(test, LAST_ARG(MAX_ARGS), -12); + KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(MAX_ARGS)), "-12"); +} + +static void if_args_test(struct kunit *test) +{ + bool with_args = true; + bool no_args = false; + enum { X = 100 }; + + KUNIT_EXPECT_TRUE(test, IF_ARGS(true, false, FOO_ARGS)); + KUNIT_EXPECT_FALSE(test, IF_ARGS(true, false, NO_ARGS)); + + KUNIT_EXPECT_TRUE(test, CONCATENATE(IF_ARGS(with, no, FOO_ARGS), _args)); + KUNIT_EXPECT_FALSE(test, CONCATENATE(IF_ARGS(with, no, NO_ARGS), _args)); + + KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, FOO_ARGS)), "yes"); + KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, NO_ARGS)), "no"); + + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, FOO_ARGS), 4); + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, NO_ARGS), -1); + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, FOO_ARGS), 0); + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, NO_ARGS), -1); + + KUNIT_EXPECT_EQ(test, + CALL_ARGS(FIRST_ARG, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), X); + KUNIT_EXPECT_EQ(test, + CALL_ARGS(FIRST_ARG, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), -1); + KUNIT_EXPECT_EQ(test, + CALL_ARGS(COUNT_ARGS, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), 4); + KUNIT_EXPECT_EQ(test, + CALL_ARGS(COUNT_ARGS, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), 12); +} + +static struct kunit_case args_tests[] = { + KUNIT_CASE(count_args_test), + KUNIT_CASE(call_args_example), + KUNIT_CASE(call_args_test), + KUNIT_CASE(drop_first_arg_example), + KUNIT_CASE(drop_first_arg_test), + KUNIT_CASE(first_arg_example), + KUNIT_CASE(first_arg_test), + KUNIT_CASE(last_arg_example), + KUNIT_CASE(last_arg_test), + KUNIT_CASE(pick_arg_example), + KUNIT_CASE(if_args_example), + KUNIT_CASE(if_args_test), + KUNIT_CASE(sep_comma_example), + {} +}; + +static struct kunit_suite args_test_suite = { + .name = "args", + .test_cases = args_tests, +}; + +kunit_test_suite(args_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c new file mode 100644 index 000000000..6a17e13d5 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_bo.c @@ -0,0 +1,874 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include +#include +#include +#include + +#include + +#include "tests/xe_kunit_helpers.h" +#include "tests/xe_pci_test.h" +#include "tests/xe_test.h" + +#include "xe_bo_evict.h" +#include "xe_gt.h" +#include "xe_pci.h" +#include "xe_pm.h" + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +struct page_size_alloc_saved { + enum xe_page_size_alloc_ctrl_mode mode; + u32 cur_index; +}; + +/* Caller must hold xe->page_size_alloc_ctrl.lock. */ +static void page_size_alloc_save(struct xe_device *xe, + struct page_size_alloc_saved *s) +{ + s->mode = xe->page_size_alloc_ctrl.mode; + s->cur_index = xe->page_size_alloc_ctrl.cur_index; +} + +static void page_size_alloc_restore(struct xe_device *xe, + const struct page_size_alloc_saved *s) +{ + mutex_lock(&xe->page_size_alloc_ctrl.lock); + xe->page_size_alloc_ctrl.mode = s->mode; + xe->page_size_alloc_ctrl.cur_index = s->cur_index; + mutex_unlock(&xe->page_size_alloc_ctrl.lock); +} + +/* Expected properties for a forced page-size allocation mode. */ +struct leaf_info { + u64 leaf; + u64 alloc_size; + u32 flag; + const char *name; +}; + +static const struct leaf_info leaf_2m = { + .leaf = SZ_2M, + .alloc_size = SZ_2M - PAGE_SIZE, + .flag = XE_BO_FLAG_NEEDS_2M, + .name = "2M", +}; + +static const struct leaf_info leaf_1g = { + .leaf = SZ_1G, + .alloc_size = SZ_1G - PAGE_SIZE, + .flag = XE_BO_FLAG_NEEDS_1G, + .name = "1G", +}; + +static void run_only_leaf(struct kunit *test, + enum xe_page_size_alloc_ctrl_mode mode, + const struct leaf_info *li) +{ + struct xe_device *xe = test->priv; + struct page_size_alloc_saved saved; + struct xe_bo *bo; + struct ttm_buffer_object *ttm_bo; + u32 other_flags; + + if (!IS_DGFX(xe)) { + kunit_skip(test, "requires dGFX VRAM"); + return; + } + + mutex_lock(&xe->page_size_alloc_ctrl.lock); + page_size_alloc_save(xe, &saved); + xe->page_size_alloc_ctrl.mode = mode; + mutex_unlock(&xe->page_size_alloc_ctrl.lock); + + bo = xe_bo_create_user(xe, NULL, li->alloc_size, + DRM_XE_GEM_CPU_CACHING_WC, + XE_BO_FLAG_VRAM0, NULL); + if (IS_ERR(bo)) { + page_size_alloc_restore(xe, &saved); + if (PTR_ERR(bo) == -ENOSPC) { + kunit_skip(test, + "no contiguous %s VRAM available right now", + li->name); + return; + } + + KUNIT_FAIL(test, "%s BO alloc failed: %pe", li->name, bo); + return; + } + + ttm_bo = &bo->ttm; + + /* 1) The mode added the right NEEDS_* flag. */ + KUNIT_EXPECT_TRUE_MSG(test, bo->flags & li->flag, + "%s: flag missing, flags=0x%x", + li->name, bo->flags); + + /* 2) No other NEEDS_* flags accidentally tagged on. */ + other_flags = (XE_BO_FLAG_NEEDS_64K | + XE_BO_FLAG_NEEDS_2M | + XE_BO_FLAG_NEEDS_1G) & ~li->flag; + KUNIT_EXPECT_FALSE_MSG(test, bo->flags & other_flags, + "%s: stray flags=0x%x", + li->name, bo->flags); + /* 3) BO size was rounded up to the expected leaf size. */ + KUNIT_EXPECT_EQ_MSG(test, xe_bo_size(bo), li->leaf, + "%s: bo size=%llu expected=%llu", + li->name, + (u64)xe_bo_size(bo), + (u64)li->leaf); + /* + * 4) Allocator honored the requested alignment. + * ttm_bo->page_alignment is stored in PAGE_SIZE units, so compare against + * the expected leaf size converted with >> PAGE_SHIFT. + */ + KUNIT_EXPECT_EQ_MSG(test, ttm_bo->page_alignment, + li->leaf >> PAGE_SHIFT, + "%s: page_alignment=%u pages expected=%llu pages", + li->name, ttm_bo->page_alignment, + (u64)(li->leaf >> PAGE_SHIFT)); + + xe_bo_put(bo); + page_size_alloc_restore(xe, &saved); +} + +static void xe_bo_page_size_alloc_only_2m(struct kunit *test) +{ + run_only_leaf(test, XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M, &leaf_2m); +} + +static void xe_bo_page_size_alloc_only_1g(struct kunit *test) +{ + run_only_leaf(test, XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G, &leaf_1g); +} + +static void xe_bo_page_size_alloc_mixed_bos(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct page_size_alloc_saved saved; + struct xe_bo *bo; + struct ttm_buffer_object *ttm_bo; + u32 all_flags = XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M | + XE_BO_FLAG_NEEDS_1G; + u32 flags; + u64 expected_align; + int i; + const int n = 4; + + if (!IS_DGFX(xe)) { + kunit_skip(test, "requires dGFX VRAM"); + return; + } + + mutex_lock(&xe->page_size_alloc_ctrl.lock); + page_size_alloc_save(xe, &saved); + mutex_unlock(&xe->page_size_alloc_ctrl.lock); + + for (i = 0; i < n; i++) { + mutex_lock(&xe->page_size_alloc_ctrl.lock); + xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED; + xe->page_size_alloc_ctrl.cur_index = i; + mutex_unlock(&xe->page_size_alloc_ctrl.lock); + /* + * Request a size valid for any mixed-mode slot. Since cur_index is + * device-global and may be perturbed by concurrent allocations on + * a live system, do not assume this iteration will see a specific + * slot. + */ + bo = xe_bo_create_user(xe, NULL, SZ_1G, + DRM_XE_GEM_CPU_CACHING_WC, + XE_BO_FLAG_VRAM0, NULL); + if (IS_ERR(bo)) { + int err = PTR_ERR(bo); + + page_size_alloc_restore(xe, &saved); + if (err == -ENOSPC) { + kunit_skip(test, + "mixed mode BO allocation unavailable: %d", + err); + return; + } + KUNIT_FAIL(test, "iter=%d alloc failed: %pe", i, bo); + return; + } + + ttm_bo = &bo->ttm; + flags = bo->flags & all_flags; + /* + * Mixed mode may result in: + * 0-> default platform VRAM alignment + * XE_BO_FLAG_NEEDS_64K + * XE_BO_FLAG_NEEDS_2M + * XE_BO_FLAG_NEEDS_1G + * Any other combination is invalid. + */ + if (flags == 0) { + expected_align = SZ_4K; + if (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + expected_align = SZ_64K; + } else if (flags == XE_BO_FLAG_NEEDS_64K) { + expected_align = SZ_64K; + } else if (flags == XE_BO_FLAG_NEEDS_2M) { + expected_align = SZ_2M; + } else if (flags == XE_BO_FLAG_NEEDS_1G) { + expected_align = SZ_1G; + } else { + KUNIT_FAIL(test, + "iter=%d invalid mixed-mode flags: 0x%x", + i, flags); + xe_bo_put(bo); + page_size_alloc_restore(xe, &saved); + return; + } + /* + * BO size should remain valid for the selected mode. Since the + * request is SZ_1G, it should remain unchanged regardless of the + * selected page-size policy. + */ + KUNIT_EXPECT_EQ_MSG(test, xe_bo_size(bo), (u64)SZ_1G, + "iter=%d size=%llu expected=%llu", + i, + (u64)xe_bo_size(bo), + (u64)SZ_1G); + KUNIT_EXPECT_EQ_MSG(test, ttm_bo->page_alignment, + expected_align >> PAGE_SHIFT, + "iter=%d flags=0x%x page_alignment=%u pages expected=%llu pages", + i, flags, ttm_bo->page_alignment, + (u64)(expected_align >> PAGE_SHIFT)); + xe_bo_put(bo); + } + page_size_alloc_restore(xe, &saved); +} +#endif + +static int ccs_test_migrate(struct xe_tile *tile, struct xe_bo *bo, + bool clear, u64 get_val, u64 assign_val, + struct kunit *test, struct drm_exec *exec) +{ + struct dma_fence *fence; + struct ttm_tt *ttm; + struct page *page; + pgoff_t ccs_page; + long timeout; + u64 *cpu_map; + int ret; + u32 offset; + + /* Move bo to VRAM if not already there. */ + ret = xe_bo_validate(bo, NULL, false, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to validate bo.\n"); + return ret; + } + + /* Optionally clear bo *and* CCS data in VRAM. */ + if (clear) { + fence = xe_migrate_clear(tile->migrate, bo, bo->ttm.resource, + XE_MIGRATE_CLEAR_FLAG_FULL); + if (IS_ERR(fence)) { + KUNIT_FAIL(test, "Failed to submit bo clear.\n"); + return PTR_ERR(fence); + } + + if (dma_fence_wait_timeout(fence, false, 5 * HZ) <= 0) { + dma_fence_put(fence); + KUNIT_FAIL(test, "Timeout while clearing bo.\n"); + return -ETIME; + } + + dma_fence_put(fence); + } + + /* Evict to system. CCS data should be copied. */ + ret = xe_bo_evict(bo, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to evict bo.\n"); + return ret; + } + + /* Sync all migration blits */ + timeout = dma_resv_wait_timeout(bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL, + true, + 5 * HZ); + if (timeout <= 0) { + KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); + return -ETIME; + } + + /* + * Bo with CCS data is now in system memory. Verify backing store + * and data integrity. Then assign for the next testing round while + * we still have a CPU map. + */ + ttm = bo->ttm.ttm; + if (!ttm || !ttm_tt_is_populated(ttm)) { + KUNIT_FAIL(test, "Bo was not in expected placement.\n"); + return -EINVAL; + } + + ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT; + if (ccs_page >= ttm->num_pages) { + KUNIT_FAIL(test, "No TTM CCS pages present.\n"); + return -EINVAL; + } + + page = ttm->pages[ccs_page]; + cpu_map = kmap_local_page(page); + + /* Check first CCS value */ + if (cpu_map[0] != get_val) { + KUNIT_FAIL(test, + "Expected CCS readout 0x%016llx, got 0x%016llx.\n", + (unsigned long long)get_val, + (unsigned long long)cpu_map[0]); + ret = -EINVAL; + } + + /* Check last CCS value, or at least last value in page. */ + offset = xe_device_ccs_bytes(tile_to_xe(tile), xe_bo_size(bo)); + offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1; + if (cpu_map[offset] != get_val) { + KUNIT_FAIL(test, + "Expected CCS readout 0x%016llx, got 0x%016llx.\n", + (unsigned long long)get_val, + (unsigned long long)cpu_map[offset]); + ret = -EINVAL; + } + + cpu_map[0] = assign_val; + cpu_map[offset] = assign_val; + kunmap_local(cpu_map); + + return ret; +} + +static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile, + struct kunit *test) +{ + struct xe_bo *bo; + + int ret; + + /* TODO: Sanity check */ + unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); + struct drm_exec *exec = XE_VALIDATION_OPT_OUT; + + if (IS_DGFX(xe)) + kunit_info(test, "Testing vram id %u\n", tile->id); + else + kunit_info(test, "Testing system memory\n"); + + bo = xe_bo_create_user(xe, NULL, SZ_1M, DRM_XE_GEM_CPU_CACHING_WC, + bo_flags, exec); + if (IS_ERR(bo)) { + KUNIT_FAIL(test, "Failed to create bo.\n"); + return; + } + + xe_bo_lock(bo, false); + + kunit_info(test, "Verifying that CCS data is cleared on creation.\n"); + ret = ccs_test_migrate(tile, bo, false, 0ULL, 0xdeadbeefdeadbeefULL, + test, exec); + if (ret) + goto out_unlock; + + kunit_info(test, "Verifying that CCS data survives migration.\n"); + ret = ccs_test_migrate(tile, bo, false, 0xdeadbeefdeadbeefULL, + 0xdeadbeefdeadbeefULL, test, exec); + if (ret) + goto out_unlock; + + kunit_info(test, "Verifying that CCS data can be properly cleared.\n"); + ret = ccs_test_migrate(tile, bo, true, 0ULL, 0ULL, test, exec); + +out_unlock: + xe_bo_unlock(bo); + xe_bo_put(bo); +} + +static int ccs_test_run_device(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_tile *tile; + int id; + + if (!xe_device_has_flat_ccs(xe)) { + kunit_skip(test, "non-flat-ccs device\n"); + return 0; + } + + /* For xe2+ dgfx, we don't handle ccs metadata */ + if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) { + kunit_skip(test, "xe2+ dgfx device\n"); + return 0; + } + + guard(xe_pm_runtime)(xe); + for_each_tile(tile, xe, id) { + /* For igfx run only for primary tile */ + if (!IS_DGFX(xe) && id > 0) + continue; + ccs_test_run_tile(xe, tile, test); + } + + return 0; +} + +static void xe_ccs_migrate_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + ccs_test_run_device(xe); +} + +static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test) +{ + struct xe_bo *bo, *external; + unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); + struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate); + struct drm_exec *exec = XE_VALIDATION_OPT_OUT; + struct xe_gt *__gt; + int err, i, id; + + kunit_info(test, "Testing device %s vram id %u\n", + dev_name(xe->drm.dev), tile->id); + + for (i = 0; i < 2; ++i) { + xe_vm_lock(vm, false); + bo = xe_bo_create_user(xe, vm, 0x10000, + DRM_XE_GEM_CPU_CACHING_WC, + bo_flags, exec); + xe_vm_unlock(vm); + if (IS_ERR(bo)) { + KUNIT_FAIL(test, "bo create err=%pe\n", bo); + break; + } + + external = xe_bo_create_user(xe, NULL, 0x10000, + DRM_XE_GEM_CPU_CACHING_WC, + bo_flags, NULL); + if (IS_ERR(external)) { + KUNIT_FAIL(test, "external bo create err=%pe\n", external); + goto cleanup_bo; + } + + xe_bo_lock(external, false); + err = xe_bo_pin_external(external, false, exec); + xe_bo_unlock(external); + if (err) { + KUNIT_FAIL(test, "external bo pin err=%pe\n", + ERR_PTR(err)); + goto cleanup_external; + } + + err = xe_bo_evict_all(xe); + if (err) { + KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err)); + goto cleanup_all; + } + + for_each_gt(__gt, xe, id) + xe_gt_sanitize(__gt); + err = xe_bo_restore_early(xe); + /* + * Snapshotting the CTB and copying back a potentially old + * version seems risky, depending on what might have been + * inflight. Also it seems snapshotting the ADS object and + * copying back results in serious breakage. Normally when + * calling xe_bo_restore_kernel() we always fully restart the + * GT, which re-intializes such things. We could potentially + * skip saving and restoring such objects in xe_bo_evict_all() + * however seems quite fragile not to also restart the GT. Try + * to do that here by triggering a GT reset. + */ + for_each_gt(__gt, xe, id) + xe_gt_reset(__gt); + + if (err) { + KUNIT_FAIL(test, "restore kernel err=%pe\n", + ERR_PTR(err)); + goto cleanup_all; + } + + err = xe_bo_restore_late(xe); + if (err) { + KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); + goto cleanup_all; + } + + if (!xe_bo_is_vram(external)) { + KUNIT_FAIL(test, "external bo is not vram\n"); + err = -EPROTO; + goto cleanup_all; + } + + if (xe_bo_is_vram(bo)) { + KUNIT_FAIL(test, "bo is vram\n"); + err = -EPROTO; + goto cleanup_all; + } + + if (i) { + down_read(&vm->lock); + xe_vm_lock(vm, false); + err = xe_bo_validate(bo, bo->vm, false, exec); + xe_vm_unlock(vm); + up_read(&vm->lock); + if (err) { + KUNIT_FAIL(test, "bo valid err=%pe\n", + ERR_PTR(err)); + goto cleanup_all; + } + xe_bo_lock(external, false); + err = xe_bo_validate(external, NULL, false, exec); + xe_bo_unlock(external); + if (err) { + KUNIT_FAIL(test, "external bo valid err=%pe\n", + ERR_PTR(err)); + goto cleanup_all; + } + } + + xe_bo_lock(external, false); + xe_bo_unpin_external(external); + xe_bo_unlock(external); + + xe_bo_put(external); + + xe_bo_lock(bo, false); + __xe_bo_unset_bulk_move(bo); + xe_bo_unlock(bo); + xe_bo_put(bo); + continue; + +cleanup_all: + xe_bo_lock(external, false); + xe_bo_unpin_external(external); + xe_bo_unlock(external); +cleanup_external: + xe_bo_put(external); +cleanup_bo: + xe_bo_lock(bo, false); + __xe_bo_unset_bulk_move(bo); + xe_bo_unlock(bo); + xe_bo_put(bo); + break; + } + + xe_vm_put(vm); + + return 0; +} + +static int evict_test_run_device(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_tile *tile; + int id; + + if (!IS_DGFX(xe)) { + kunit_skip(test, "non-discrete device\n"); + return 0; + } + + guard(xe_pm_runtime)(xe); + for_each_tile(tile, xe, id) + evict_test_run_tile(xe, tile, test); + + return 0; +} + +static void xe_bo_evict_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + evict_test_run_device(xe); +} + +struct xe_bo_link { + struct list_head link; + struct xe_bo *bo; + u32 val; +}; + +#define XE_BO_SHRINK_SIZE ((unsigned long)SZ_64M) + +static int shrink_test_fill_random(struct xe_bo *bo, struct rnd_state *state, + struct xe_bo_link *link) +{ + struct iosys_map map; + int ret = ttm_bo_vmap(&bo->ttm, &map); + size_t __maybe_unused i; + + if (ret) + return ret; + + for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) { + u32 val = prandom_u32_state(state); + + iosys_map_wr(&map, i, u32, val); + if (i == 0) + link->val = val; + } + + ttm_bo_vunmap(&bo->ttm, &map); + return 0; +} + +static bool shrink_test_verify(struct kunit *test, struct xe_bo *bo, + unsigned int bo_nr, struct rnd_state *state, + struct xe_bo_link *link) +{ + struct iosys_map map; + int ret = ttm_bo_vmap(&bo->ttm, &map); + size_t i; + bool failed = false; + + if (ret) { + KUNIT_FAIL(test, "Error mapping bo %u for content check.\n", bo_nr); + return true; + } + + for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) { + u32 val = prandom_u32_state(state); + + if (iosys_map_rd(&map, i, u32) != val) { + KUNIT_FAIL(test, "Content not preserved, bo %u offset 0x%016llx", + bo_nr, (unsigned long long)i); + kunit_info(test, "Failed value is 0x%08x, recorded 0x%08x\n", + (unsigned int)iosys_map_rd(&map, i, u32), val); + if (i == 0 && val != link->val) + kunit_info(test, "Looks like PRNG is out of sync.\n"); + failed = true; + break; + } + } + + ttm_bo_vunmap(&bo->ttm, &map); + + return failed; +} + +/* + * Try to create system bos corresponding to twice the amount + * of available system memory to test shrinker functionality. + * If no swap space is available to accommodate the + * memory overcommit, mark bos purgeable. + */ +static int shrink_test_run_device(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + LIST_HEAD(bos); + struct xe_bo_link *link, *next; + struct sysinfo si; + u64 ram, ram_and_swap, purgeable = 0, alloced, to_alloc, limit; + unsigned int interrupted = 0, successful = 0, count = 0; + struct rnd_state prng; + u64 rand_seed; + bool failed = false; + + rand_seed = get_random_u64(); + prandom_seed_state(&prng, rand_seed); + kunit_info(test, "Random seed is 0x%016llx.\n", + (unsigned long long)rand_seed); + + /* Skip if execution time is expected to be too long. */ + + limit = SZ_32G; + /* IGFX with flat CCS needs to copy when swapping / shrinking */ + if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) + limit = SZ_16G; + + si_meminfo(&si); + ram = (size_t)si.freeram * si.mem_unit; + if (ram > limit) { + kunit_skip(test, "Too long expected execution time.\n"); + return 0; + } + to_alloc = ram * 2; + + ram_and_swap = ram + get_nr_swap_pages() * PAGE_SIZE; + if (to_alloc > ram_and_swap) + purgeable = to_alloc - ram_and_swap; + purgeable += div64_u64(purgeable, 5); + + kunit_info(test, "Free ram is %lu bytes. Will allocate twice of that.\n", + (unsigned long)ram); + for (alloced = 0; alloced < to_alloc; alloced += XE_BO_SHRINK_SIZE) { + struct xe_bo *bo; + unsigned int mem_type; + struct xe_ttm_tt *xe_tt; + + link = kzalloc_obj(*link); + if (!link) { + KUNIT_FAIL(test, "Unexpected link allocation failure\n"); + failed = true; + break; + } + + INIT_LIST_HEAD(&link->link); + + /* We can create bos using WC caching here. But it is slower. */ + bo = xe_bo_create_user(xe, NULL, XE_BO_SHRINK_SIZE, + DRM_XE_GEM_CPU_CACHING_WB, + XE_BO_FLAG_SYSTEM, NULL); + if (IS_ERR(bo)) { + if (bo != ERR_PTR(-ENOMEM) && bo != ERR_PTR(-ENOSPC) && + bo != ERR_PTR(-EINTR) && bo != ERR_PTR(-ERESTARTSYS)) + KUNIT_FAIL(test, "Error creating bo: %pe\n", bo); + kfree(link); + failed = true; + break; + } + xe_bo_lock(bo, false); + xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm); + + /* + * Allocate purgeable bos first, because if we do it the + * other way around, they may not be subject to swapping... + */ + if (alloced < purgeable) { + xe_ttm_tt_account_subtract(xe, &xe_tt->ttm); + xe_tt->purgeable = true; + xe_ttm_tt_account_add(xe, &xe_tt->ttm); + bo->ttm.priority = 0; + spin_lock(&bo->ttm.bdev->lru_lock); + ttm_bo_move_to_lru_tail(&bo->ttm); + spin_unlock(&bo->ttm.bdev->lru_lock); + } else { + int ret = shrink_test_fill_random(bo, &prng, link); + + if (ret) { + xe_bo_unlock(bo); + xe_bo_put(bo); + KUNIT_FAIL(test, "Error filling bo with random data: %pe\n", + ERR_PTR(ret)); + kfree(link); + failed = true; + break; + } + } + + mem_type = bo->ttm.resource->mem_type; + xe_bo_unlock(bo); + link->bo = bo; + list_add_tail(&link->link, &bos); + + if (mem_type != XE_PL_TT) { + KUNIT_FAIL(test, "Bo in incorrect memory type: %u\n", + bo->ttm.resource->mem_type); + failed = true; + } + cond_resched(); + if (signal_pending(current)) + break; + } + + /* + * Read back and destroy bos. Reset the pseudo-random seed to get an + * identical pseudo-random number sequence for readback. + */ + prandom_seed_state(&prng, rand_seed); + list_for_each_entry_safe(link, next, &bos, link) { + static struct ttm_operation_ctx ctx = {.interruptible = true}; + struct xe_bo *bo = link->bo; + struct xe_ttm_tt *xe_tt; + int ret; + + count++; + if (!signal_pending(current) && !failed) { + bool purgeable, intr = false; + + xe_bo_lock(bo, NULL); + + /* xe_tt->purgeable is cleared on validate. */ + xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm); + purgeable = xe_tt->purgeable; + do { + ret = ttm_bo_validate(&bo->ttm, &tt_placement, &ctx); + if (ret == -EINTR) + intr = true; + } while (ret == -EINTR && !signal_pending(current)); + if (!ret && !purgeable) + failed = shrink_test_verify(test, bo, count, &prng, link); + + xe_bo_unlock(bo); + if (ret) { + KUNIT_FAIL(test, "Validation failed: %pe\n", + ERR_PTR(ret)); + failed = true; + } else if (intr) { + interrupted++; + } else { + successful++; + } + } + xe_bo_put(link->bo); + list_del(&link->link); + kfree(link); + } + kunit_info(test, "Readbacks interrupted: %u successful: %u\n", + interrupted, successful); + + return 0; +} + +static void xe_bo_shrink_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + shrink_test_run_device(xe); +} + +static struct kunit_case xe_bo_tests[] = { + KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param), + KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param), + {} +}; + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +static struct kunit_case xe_bo_page_size_alloc_cases[] = { + KUNIT_CASE_PARAM(xe_bo_page_size_alloc_only_2m, xe_pci_live_device_gen_param), + KUNIT_CASE_PARAM(xe_bo_page_size_alloc_only_1g, xe_pci_live_device_gen_param), + KUNIT_CASE_PARAM(xe_bo_page_size_alloc_mixed_bos, xe_pci_live_device_gen_param), + {} +}; + +VISIBLE_IF_KUNIT +struct kunit_suite xe_bo_page_size_alloc_suite = { + .name = "xe_bo_page_size_alloc", + .test_cases = xe_bo_page_size_alloc_cases, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_bo_page_size_alloc_suite); +#endif + +VISIBLE_IF_KUNIT +struct kunit_suite xe_bo_test_suite = { + .name = "xe_bo", + .test_cases = xe_bo_tests, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite); + +static struct kunit_case xe_bo_shrink_test[] = { + KUNIT_CASE_PARAM_ATTR(xe_bo_shrink_kunit, xe_pci_live_device_gen_param, + {.speed = KUNIT_SPEED_SLOW}), + {} +}; + +VISIBLE_IF_KUNIT +struct kunit_suite xe_bo_shrink_test_suite = { + .name = "xe_bo_shrink", + .test_cases = xe_bo_shrink_test, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_bo_shrink_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c new file mode 100644 index 000000000..0be8440b3 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include +#include + +#include "tests/xe_kunit_helpers.h" +#include "tests/xe_pci_test.h" + +#include "xe_pci.h" +#include "xe_pm.h" + +static bool p2p_enabled(struct dma_buf_test_params *params) +{ + return IS_ENABLED(CONFIG_PCI_P2PDMA) && params->attach_ops && + params->attach_ops->allow_peer2peer; +} + +static bool is_dynamic(struct dma_buf_test_params *params) +{ + return params->attach_ops && params->attach_ops->invalidate_mappings; +} + +static void check_residency(struct kunit *test, struct xe_bo *exported, + struct xe_bo *imported, struct dma_buf *dmabuf, + struct drm_exec *exec) +{ + struct dma_buf_test_params *params = to_dma_buf_test_params(test->priv); + struct dma_buf_attachment *attach; + u32 mem_type; + int ret; + + xe_bo_assert_held(exported); + xe_bo_assert_held(imported); + + mem_type = XE_PL_VRAM0; + if (!(params->mem_mask & XE_BO_FLAG_VRAM0)) + /* No VRAM allowed */ + mem_type = XE_PL_TT; + else if (params->force_different_devices && !p2p_enabled(params)) + /* No P2P */ + mem_type = XE_PL_TT; + else if (params->force_different_devices && !is_dynamic(params) && + (params->mem_mask & XE_BO_FLAG_SYSTEM)) + /* Pin migrated to TT on non-dynamic attachments. */ + mem_type = XE_PL_TT; + + if (!xe_bo_is_mem_type(exported, mem_type)) { + KUNIT_FAIL(test, "Exported bo was not in expected memory type.\n"); + return; + } + + if (xe_bo_is_pinned(exported)) + return; + + /* + * Evict exporter. Evicting the exported bo will + * evict also the imported bo through the invalidate_mappings() functionality if + * importer is on a different device. If they're on the same device, + * the exporter and the importer should be the same bo. + */ + ret = xe_bo_evict(exported, exec); + if (ret) { + if (ret != -EINTR && ret != -ERESTARTSYS) + KUNIT_FAIL(test, "Evicting exporter failed with err=%d.\n", + ret); + return; + } + + /* Verify that also importer has been evicted to SYSTEM */ + if (exported != imported && !xe_bo_is_mem_type(imported, XE_PL_SYSTEM)) { + KUNIT_FAIL(test, "Importer wasn't properly evicted.\n"); + return; + } + + /* Re-validate the importer. This should move also exporter in. */ + ret = xe_bo_validate(imported, NULL, false, exec); + if (ret) { + if (ret != -EINTR && ret != -ERESTARTSYS) + KUNIT_FAIL(test, "Validating importer failed with err=%d.\n", + ret); + return; + } + + KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(exported, mem_type)); + + /* Check that we can pin without migrating. */ + attach = list_first_entry_or_null(&dmabuf->attachments, typeof(*attach), node); + if (attach) { + int err = dma_buf_pin(attach); + + if (!err) { + KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(exported, mem_type)); + dma_buf_unpin(attach); + } + KUNIT_EXPECT_EQ(test, err, 0); + } + + if (params->force_different_devices) + KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(imported, XE_PL_TT)); + else + KUNIT_EXPECT_TRUE(test, exported == imported); +} + +static void xe_test_dmabuf_import_same_driver(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + struct dma_buf_test_params *params = to_dma_buf_test_params(test->priv); + struct drm_gem_object *import; + struct dma_buf *dmabuf; + struct xe_bo *bo; + size_t size; + + /* No VRAM on this device? */ + if (!ttm_manager_type(&xe->ttm, XE_PL_VRAM0) && + (params->mem_mask & XE_BO_FLAG_VRAM0)) + return; + + size = PAGE_SIZE; + if ((params->mem_mask & XE_BO_FLAG_VRAM0) && + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + size = SZ_64K; + + kunit_info(test, "running %s\n", __func__); + bo = xe_bo_create_user(xe, NULL, size, DRM_XE_GEM_CPU_CACHING_WC, + params->mem_mask, NULL); + if (IS_ERR(bo)) { + KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", + PTR_ERR(bo)); + return; + } + + dmabuf = xe_gem_prime_export(&bo->ttm.base, 0); + if (IS_ERR(dmabuf)) { + KUNIT_FAIL(test, "xe_gem_prime_export() failed with err=%ld\n", + PTR_ERR(dmabuf)); + goto out; + } + bo->ttm.base.dma_buf = dmabuf; + + import = xe_gem_prime_import(&xe->drm, dmabuf); + if (!IS_ERR(import)) { + struct xe_bo *import_bo = gem_to_xe_bo(import); + + /* + * Did import succeed when it shouldn't due to lack of p2p support? + */ + if (params->force_different_devices && + !p2p_enabled(params) && + !(params->mem_mask & XE_BO_FLAG_SYSTEM)) { + KUNIT_FAIL(test, + "xe_gem_prime_import() succeeded when it shouldn't have\n"); + } else { + struct drm_exec *exec = XE_VALIDATION_OPT_OUT; + int err; + + /* Is everything where we expect it to be? */ + xe_bo_lock(import_bo, false); + err = xe_bo_validate(import_bo, NULL, false, exec); + + /* Pinning in VRAM is not allowed for non-dynamic attachments */ + if (!is_dynamic(params) && + params->force_different_devices && + !(params->mem_mask & XE_BO_FLAG_SYSTEM)) + KUNIT_EXPECT_EQ(test, err, -EINVAL); + /* Otherwise only expect interrupts or success. */ + else if (err && err != -EINTR && err != -ERESTARTSYS) + KUNIT_EXPECT_TRUE(test, !err || err == -EINTR || + err == -ERESTARTSYS); + + if (!err) + check_residency(test, bo, import_bo, dmabuf, exec); + xe_bo_unlock(import_bo); + } + drm_gem_object_put(import); + } else if (PTR_ERR(import) != -EOPNOTSUPP) { + /* Unexpected error code. */ + KUNIT_FAIL(test, + "xe_gem_prime_import failed with the wrong err=%ld\n", + PTR_ERR(import)); + } else if (!params->force_different_devices || + p2p_enabled(params) || + (params->mem_mask & XE_BO_FLAG_SYSTEM)) { + /* Shouldn't fail if we can reuse same bo, use p2p or use system */ + KUNIT_FAIL(test, "dynamic p2p attachment failed with err=%ld\n", + PTR_ERR(import)); + } + bo->ttm.base.dma_buf = NULL; + dma_buf_put(dmabuf); +out: + drm_gem_object_put(&bo->ttm.base); +} + +static const struct dma_buf_attach_ops nop2p_attach_ops = { + .allow_peer2peer = false, + .invalidate_mappings = xe_dma_buf_move_notify +}; + +/* + * We test the implementation with bos of different residency and with + * importers with different capabilities; some lacking p2p support and some + * lacking dynamic capabilities (attach_ops == NULL). We also fake + * different devices avoiding the import shortcut that just reuses the same + * gem object. + */ +static const struct dma_buf_test_params test_params[] = { + {.mem_mask = XE_BO_FLAG_VRAM0, + .attach_ops = &xe_dma_buf_attach_ops}, + {.mem_mask = XE_BO_FLAG_VRAM0 | XE_BO_FLAG_NEEDS_CPU_ACCESS, + .attach_ops = &xe_dma_buf_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_VRAM0, + .attach_ops = &nop2p_attach_ops}, + {.mem_mask = XE_BO_FLAG_VRAM0, + .attach_ops = &nop2p_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_VRAM0}, + {.mem_mask = XE_BO_FLAG_VRAM0, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_SYSTEM, + .attach_ops = &xe_dma_buf_attach_ops}, + {.mem_mask = XE_BO_FLAG_SYSTEM, + .attach_ops = &xe_dma_buf_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_SYSTEM, + .attach_ops = &nop2p_attach_ops}, + {.mem_mask = XE_BO_FLAG_SYSTEM, + .attach_ops = &nop2p_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_SYSTEM}, + {.mem_mask = XE_BO_FLAG_SYSTEM, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0, + .attach_ops = &xe_dma_buf_attach_ops}, + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0 | + XE_BO_FLAG_NEEDS_CPU_ACCESS, + .attach_ops = &xe_dma_buf_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0, + .attach_ops = &nop2p_attach_ops}, + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0, + .attach_ops = &nop2p_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0}, + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0, + .force_different_devices = true}, + + {} +}; + +static int dma_buf_run_device(struct xe_device *xe) +{ + const struct dma_buf_test_params *params; + struct kunit *test = kunit_get_current_test(); + + guard(xe_pm_runtime)(xe); + for (params = test_params; params->mem_mask; ++params) { + struct dma_buf_test_params p = *params; + + p.base.id = XE_TEST_LIVE_DMA_BUF; + test->priv = &p; + xe_test_dmabuf_import_same_driver(xe); + } + + /* A non-zero return would halt iteration over driver devices */ + return 0; +} + +static void xe_dma_buf_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + dma_buf_run_device(xe); +} + +static struct kunit_case xe_dma_buf_tests[] = { + KUNIT_CASE_PARAM(xe_dma_buf_kunit, xe_pci_live_device_gen_param), + {} +}; + +VISIBLE_IF_KUNIT +struct kunit_suite xe_dma_buf_test_suite = { + .name = "xe_dma_buf", + .test_cases = xe_dma_buf_tests, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_dma_buf_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c new file mode 100644 index 000000000..e6eaa94d4 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include + +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" + +#define TEST_MAX_VFS 63 +#define TEST_VRAM 0x7a800000ull /* random size that works on 32-bit */ + +static bool xe_device_is_admin_only_stub_enable(const struct xe_device *xe) +{ + return true; +} + +static bool xe_device_is_admin_only_stub_disable(const struct xe_device *xe) +{ + return false; +} + +static void pf_set_admin_mode(struct xe_device *xe, bool enable) +{ + typeof(xe_device_is_admin_only) *stub = enable ? + xe_device_is_admin_only_stub_enable : + xe_device_is_admin_only_stub_disable; + + kunit_activate_static_stub(kunit_get_current_test(), + xe_device_is_admin_only, + *stub); + + KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_device_is_admin_only(xe)); +} + +static void pf_set_usable_vram(struct xe_device *xe, u64 usable) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct kunit *test = kunit_get_current_test(); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tile); + xe->mem.vram->usable_size = usable; + tile->mem.vram->usable_size = usable; + KUNIT_ASSERT_EQ(test, usable, xe_vram_region_usable_size(tile->mem.vram)); +} + +static const void *num_vfs_gen_param(struct kunit *test, const void *prev, char *desc) +{ + unsigned long next = 1 + (unsigned long)prev; + + if (next > TEST_MAX_VFS) + return NULL; + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%lu VF%s", + next, str_plural(next)); + return (void *)next; +} + +static int pf_gt_config_test_init(struct kunit *test) +{ + struct xe_pci_fake_data fake = { + .sriov_mode = XE_SRIOV_MODE_PF, + .platform = XE_BATTLEMAGE, /* any random DGFX platform with SR-IOV */ + .subplatform = XE_SUBPLATFORM_NONE, + .graphics_verx100 = 2001, + }; + struct xe_vram_region *vram; + struct xe_device *xe; + struct xe_gt *gt; + + test->priv = &fake; + xe_kunit_helper_xe_device_test_init(test); + + xe = test->priv; + KUNIT_ASSERT_TRUE(test, IS_SRIOV_PF(xe)); + + gt = xe_root_mmio_gt(xe); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gt); + test->priv = gt; + + /* pretend it has some VRAM */ + KUNIT_ASSERT_TRUE(test, IS_DGFX(xe)); + vram = kunit_kzalloc(test, sizeof(*vram), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vram); + vram->usable_size = TEST_VRAM; + xe->mem.vram = vram; + xe->tiles[0].mem.vram = vram; + + /* pretend we have a valid LMTT */ + KUNIT_ASSERT_TRUE(test, xe_device_has_lmtt(xe)); + KUNIT_ASSERT_GE(test, GRAPHICS_VERx100(xe), 1260); + xe->tiles[0].sriov.pf.lmtt.ops = &lmtt_ml_ops; + + /* pretend it can support up to 63 VFs */ + xe->sriov.pf.device_total_vfs = TEST_MAX_VFS; + xe->sriov.pf.driver_max_vfs = TEST_MAX_VFS; + KUNIT_ASSERT_EQ(test, xe_sriov_pf_get_totalvfs(xe), 63); + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_EQ(test, xe_sriov_init(xe), 0); + + /* more sanity checks */ + KUNIT_EXPECT_EQ(test, GUC_ID_MAX + 1, SZ_64K); + KUNIT_EXPECT_EQ(test, GUC_NUM_DOORBELLS, SZ_256); + + return 0; +} + +static void fair_contexts_1vf(struct kunit *test) +{ + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_FALSE(test, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(test, SZ_32K, pf_profile_fair_ctxs(gt, 1)); + + pf_set_admin_mode(xe, true); + KUNIT_ASSERT_TRUE(test, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(test, SZ_64K - SZ_1K, pf_profile_fair_ctxs(gt, 1)); +} + +static void fair_contexts(struct kunit *test) +{ + unsigned int num_vfs = (unsigned long)test->param_value; + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_FALSE(test, xe_sriov_pf_admin_only(xe)); + + KUNIT_EXPECT_TRUE(test, is_power_of_2(pf_profile_fair_ctxs(gt, num_vfs))); + KUNIT_EXPECT_GT(test, GUC_ID_MAX, num_vfs * pf_profile_fair_ctxs(gt, num_vfs)); + + if (num_vfs > 31) + KUNIT_ASSERT_EQ(test, SZ_1K, pf_profile_fair_ctxs(gt, num_vfs)); + else if (num_vfs > 15) + KUNIT_ASSERT_EQ(test, SZ_2K, pf_profile_fair_ctxs(gt, num_vfs)); + else if (num_vfs > 7) + KUNIT_ASSERT_EQ(test, SZ_4K, pf_profile_fair_ctxs(gt, num_vfs)); + else if (num_vfs > 3) + KUNIT_ASSERT_EQ(test, SZ_8K, pf_profile_fair_ctxs(gt, num_vfs)); + else if (num_vfs > 1) + KUNIT_ASSERT_EQ(test, SZ_16K, pf_profile_fair_ctxs(gt, num_vfs)); + else + KUNIT_ASSERT_EQ(test, SZ_32K, pf_profile_fair_ctxs(gt, num_vfs)); +} + +static void fair_doorbells_1vf(struct kunit *test) +{ + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_FALSE(test, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(test, 128, pf_profile_fair_dbs(gt, 1)); + + pf_set_admin_mode(xe, true); + KUNIT_ASSERT_TRUE(test, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(test, 240, pf_profile_fair_dbs(gt, 1)); +} + +static void fair_doorbells(struct kunit *test) +{ + unsigned int num_vfs = (unsigned long)test->param_value; + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_FALSE(test, xe_sriov_pf_admin_only(xe)); + + KUNIT_EXPECT_TRUE(test, is_power_of_2(pf_profile_fair_dbs(gt, num_vfs))); + KUNIT_EXPECT_GE(test, GUC_NUM_DOORBELLS, (num_vfs + 1) * pf_profile_fair_dbs(gt, num_vfs)); + + if (num_vfs > 31) + KUNIT_ASSERT_EQ(test, SZ_4, pf_profile_fair_dbs(gt, num_vfs)); + else if (num_vfs > 15) + KUNIT_ASSERT_EQ(test, SZ_8, pf_profile_fair_dbs(gt, num_vfs)); + else if (num_vfs > 7) + KUNIT_ASSERT_EQ(test, SZ_16, pf_profile_fair_dbs(gt, num_vfs)); + else if (num_vfs > 3) + KUNIT_ASSERT_EQ(test, SZ_32, pf_profile_fair_dbs(gt, num_vfs)); + else if (num_vfs > 1) + KUNIT_ASSERT_EQ(test, SZ_64, pf_profile_fair_dbs(gt, num_vfs)); + else + KUNIT_ASSERT_EQ(test, SZ_128, pf_profile_fair_dbs(gt, num_vfs)); +} + +static void fair_ggtt_1vf(struct kunit *test) +{ + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_FALSE(test, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(test, SZ_2G, pf_profile_fair_ggtt(gt, 1)); + + pf_set_admin_mode(xe, true); + KUNIT_ASSERT_TRUE(test, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(test, SZ_2G + SZ_1G + SZ_512M, pf_profile_fair_ggtt(gt, 1)); +} + +static void fair_ggtt(struct kunit *test) +{ + unsigned int num_vfs = (unsigned long)test->param_value; + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + u64 alignment = pf_get_ggtt_alignment(gt); + u64 shareable = SZ_2G + SZ_1G + SZ_512M; + + pf_set_admin_mode(xe, false); + KUNIT_ASSERT_FALSE(test, xe_sriov_pf_admin_only(xe)); + + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(pf_profile_fair_ggtt(gt, num_vfs), alignment)); + KUNIT_EXPECT_GE(test, shareable, num_vfs * pf_profile_fair_ggtt(gt, num_vfs)); + + if (num_vfs > 56) + KUNIT_ASSERT_EQ(test, SZ_64M - SZ_8M, pf_profile_fair_ggtt(gt, num_vfs)); + else if (num_vfs > 28) + KUNIT_ASSERT_EQ(test, SZ_64M, pf_profile_fair_ggtt(gt, num_vfs)); + else if (num_vfs > 14) + KUNIT_ASSERT_EQ(test, SZ_128M, pf_profile_fair_ggtt(gt, num_vfs)); + else if (num_vfs > 7) + KUNIT_ASSERT_EQ(test, SZ_256M, pf_profile_fair_ggtt(gt, num_vfs)); + else if (num_vfs > 3) + KUNIT_ASSERT_EQ(test, SZ_512M, pf_profile_fair_ggtt(gt, num_vfs)); + else if (num_vfs > 1) + KUNIT_ASSERT_EQ(test, SZ_1G, pf_profile_fair_ggtt(gt, num_vfs)); + else + KUNIT_ASSERT_EQ(test, SZ_2G, pf_profile_fair_ggtt(gt, num_vfs)); +} + +static const u64 vram_sizes[] = { + SZ_4G - SZ_512M, + SZ_8G + SZ_4G - SZ_512M, + SZ_16G - SZ_512M, + SZ_32G - SZ_512M, + SZ_64G - SZ_512M, + TEST_VRAM, +}; + +static void u64_param_get_desc(const u64 *p, char *desc) +{ + string_get_size(*p, 1, STRING_UNITS_2, desc, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(vram_size, vram_sizes, u64_param_get_desc); + +static void fair_vram_1vf(struct kunit *test) +{ + const u64 usable = *(const u64 *)test->param_value; + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, false); + pf_set_usable_vram(xe, usable); + + KUNIT_EXPECT_NE(test, 0, pf_profile_fair_lmem(gt, 1)); + KUNIT_EXPECT_GE(test, usable, pf_profile_fair_lmem(gt, 1)); + KUNIT_EXPECT_TRUE(test, is_power_of_2(pf_profile_fair_lmem(gt, 1))); + KUNIT_EXPECT_GE(test, usable - pf_profile_fair_lmem(gt, 1), pf_profile_fair_lmem(gt, 1)); +} + +static void fair_vram_1vf_admin_only(struct kunit *test) +{ + const u64 usable = *(const u64 *)test->param_value; + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + + pf_set_admin_mode(xe, true); + pf_set_usable_vram(xe, usable); + + KUNIT_EXPECT_NE(test, 0, pf_profile_fair_lmem(gt, 1)); + KUNIT_EXPECT_GE(test, usable, pf_profile_fair_lmem(gt, 1)); + KUNIT_EXPECT_LT(test, usable - pf_profile_fair_lmem(gt, 1), pf_profile_fair_lmem(gt, 1)); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(pf_profile_fair_lmem(gt, 1), SZ_1G)); +} + +static void fair_vram(struct kunit *test) +{ + unsigned int num_vfs = (unsigned long)test->param_value; + struct xe_gt *gt = test->priv; + struct xe_device *xe = gt_to_xe(gt); + u64 alignment = pf_get_lmem_alignment(gt); + char size[10]; + + pf_set_admin_mode(xe, false); + + string_get_size(pf_profile_fair_lmem(gt, num_vfs), 1, STRING_UNITS_2, size, sizeof(size)); + kunit_info(test, "fair %s %llx\n", size, pf_profile_fair_lmem(gt, num_vfs)); + + KUNIT_EXPECT_TRUE(test, is_power_of_2(pf_profile_fair_lmem(gt, num_vfs))); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(pf_profile_fair_lmem(gt, num_vfs), alignment)); + KUNIT_EXPECT_GE(test, TEST_VRAM, num_vfs * pf_profile_fair_lmem(gt, num_vfs)); +} + +static struct kunit_case pf_gt_config_test_cases[] = { + KUNIT_CASE(fair_contexts_1vf), + KUNIT_CASE(fair_doorbells_1vf), + KUNIT_CASE(fair_ggtt_1vf), + KUNIT_CASE_PARAM(fair_vram_1vf, vram_size_gen_params), + KUNIT_CASE_PARAM(fair_vram_1vf_admin_only, vram_size_gen_params), + KUNIT_CASE_PARAM(fair_contexts, num_vfs_gen_param), + KUNIT_CASE_PARAM(fair_doorbells, num_vfs_gen_param), + KUNIT_CASE_PARAM(fair_ggtt, num_vfs_gen_param), + KUNIT_CASE_PARAM(fair_vram, num_vfs_gen_param), + {} +}; + +static struct kunit_suite pf_gt_config_suite = { + .name = "pf_gt_config", + .test_cases = pf_gt_config_test_cases, + .init = pf_gt_config_test_init, +}; + +kunit_test_suite(pf_gt_config_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c b/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c new file mode 100644 index 000000000..51e1e0400 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include +#include +#include + +#include "xe_device.h" +#include "xe_ggtt.h" +#include "xe_guc_ct.h" +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" + +#define DUT_GGTT_START SZ_1M +#define DUT_GGTT_SIZE SZ_2M + +static struct xe_bo *replacement_xe_managed_bo_create_pin_map(struct xe_device *xe, + struct xe_tile *tile, + size_t size, u32 flags) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_bo *bo; + void *buf; + + bo = drmm_kzalloc(&xe->drm, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bo); + + buf = drmm_kzalloc(&xe->drm, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf); + + bo->tile = tile; + bo->ttm.bdev = &xe->ttm; + bo->ttm.base.size = size; + iosys_map_set_vaddr(&bo->vmap, buf); + + if (flags & XE_BO_FLAG_GGTT) { + struct xe_ggtt *ggtt = tile->mem.ggtt; + + bo->ggtt_node[tile->id] = xe_ggtt_insert_node(ggtt, xe_bo_size(bo), SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bo->ggtt_node[tile->id]); + } + + return bo; +} + +static int guc_buf_test_init(struct kunit *test) +{ + struct xe_pci_fake_data fake = { + .sriov_mode = XE_SRIOV_MODE_PF, + .platform = XE_TIGERLAKE, /* some random platform */ + .subplatform = XE_SUBPLATFORM_NONE, + }; + struct xe_ggtt *ggtt; + struct xe_guc *guc; + + test->priv = &fake; + xe_kunit_helper_xe_device_test_init(test); + + ggtt = xe_device_get_root_tile(test->priv)->mem.ggtt; + guc = &xe_device_get_gt(test->priv, 0)->uc.guc; + + KUNIT_ASSERT_EQ(test, 0, + xe_ggtt_init_kunit(ggtt, DUT_GGTT_START, + DUT_GGTT_SIZE)); + + kunit_activate_static_stub(test, xe_managed_bo_create_pin_map, + replacement_xe_managed_bo_create_pin_map); + + KUNIT_ASSERT_EQ(test, 0, xe_guc_buf_cache_init(&guc->buf)); + + test->priv = &guc->buf; + return 0; +} + +static void test_smallest(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf buf; + + buf = xe_guc_buf_reserve(cache, 1); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_EXPECT_NOT_NULL(test, xe_guc_buf_cpu_ptr(buf)); + KUNIT_EXPECT_NE(test, 0, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_LE(test, DUT_GGTT_START, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_GT(test, DUT_GGTT_START + DUT_GGTT_SIZE, xe_guc_buf_gpu_addr(buf)); + xe_guc_buf_release(buf); +} + +static void test_largest(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf buf; + + buf = xe_guc_buf_reserve(cache, xe_guc_buf_cache_dwords(cache)); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_EXPECT_NOT_NULL(test, xe_guc_buf_cpu_ptr(buf)); + KUNIT_EXPECT_NE(test, 0, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_LE(test, DUT_GGTT_START, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_GT(test, DUT_GGTT_START + DUT_GGTT_SIZE, xe_guc_buf_gpu_addr(buf)); + xe_guc_buf_release(buf); +} + +static void test_granular(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf *bufs; + int n, dwords; + + dwords = xe_guc_buf_cache_dwords(cache); + bufs = kunit_kcalloc(test, dwords, sizeof(*bufs), GFP_KERNEL); + KUNIT_EXPECT_NOT_NULL(test, bufs); + + for (n = 0; n < dwords; n++) + bufs[n] = xe_guc_buf_reserve(cache, 1); + + for (n = 0; n < dwords; n++) + KUNIT_EXPECT_TRUE_MSG(test, xe_guc_buf_is_valid(bufs[n]), "n=%d", n); + + for (n = 0; n < dwords; n++) + xe_guc_buf_release(bufs[n]); +} + +static void test_unique(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf *bufs; + int n, m, dwords; + + dwords = xe_guc_buf_cache_dwords(cache); + bufs = kunit_kcalloc(test, dwords, sizeof(*bufs), GFP_KERNEL); + KUNIT_EXPECT_NOT_NULL(test, bufs); + + for (n = 0; n < dwords; n++) + bufs[n] = xe_guc_buf_reserve(cache, 1); + + for (n = 0; n < dwords; n++) { + for (m = n + 1; m < dwords; m++) { + KUNIT_EXPECT_PTR_NE_MSG(test, xe_guc_buf_cpu_ptr(bufs[n]), + xe_guc_buf_cpu_ptr(bufs[m]), "n=%d, m=%d", n, m); + KUNIT_ASSERT_NE_MSG(test, xe_guc_buf_gpu_addr(bufs[n]), + xe_guc_buf_gpu_addr(bufs[m]), "n=%d, m=%d", n, m); + } + } + + for (n = 0; n < dwords; n++) + xe_guc_buf_release(bufs[n]); +} + +static void test_overlap(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf b1, b2; + u32 dwords = xe_guc_buf_cache_dwords(cache) / 2; + u32 bytes = dwords * sizeof(u32); + void *p1, *p2; + u64 a1, a2; + + b1 = xe_guc_buf_reserve(cache, dwords); + b2 = xe_guc_buf_reserve(cache, dwords); + + p1 = xe_guc_buf_cpu_ptr(b1); + p2 = xe_guc_buf_cpu_ptr(b2); + + a1 = xe_guc_buf_gpu_addr(b1); + a2 = xe_guc_buf_gpu_addr(b2); + + KUNIT_EXPECT_PTR_NE(test, p1, p2); + if (p1 < p2) + KUNIT_EXPECT_LT(test, (uintptr_t)(p1 + bytes - 1), (uintptr_t)p2); + else + KUNIT_EXPECT_LT(test, (uintptr_t)(p2 + bytes - 1), (uintptr_t)p1); + + KUNIT_EXPECT_NE(test, a1, a2); + if (a1 < a2) + KUNIT_EXPECT_LT(test, a1 + bytes - 1, a2); + else + KUNIT_EXPECT_LT(test, a2 + bytes - 1, a1); + + xe_guc_buf_release(b1); + xe_guc_buf_release(b2); +} + +static void test_reusable(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf b1, b2; + void *p1; + u64 a1; + + b1 = xe_guc_buf_reserve(cache, xe_guc_buf_cache_dwords(cache)); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(b1)); + KUNIT_EXPECT_NOT_NULL(test, p1 = xe_guc_buf_cpu_ptr(b1)); + KUNIT_EXPECT_NE(test, 0, a1 = xe_guc_buf_gpu_addr(b1)); + xe_guc_buf_release(b1); + + b2 = xe_guc_buf_reserve(cache, xe_guc_buf_cache_dwords(cache)); + KUNIT_EXPECT_PTR_EQ(test, p1, xe_guc_buf_cpu_ptr(b2)); + KUNIT_EXPECT_EQ(test, a1, xe_guc_buf_gpu_addr(b2)); + xe_guc_buf_release(b2); +} + +static void test_too_big(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf buf; + + buf = xe_guc_buf_reserve(cache, xe_guc_buf_cache_dwords(cache) + 1); + KUNIT_EXPECT_FALSE(test, xe_guc_buf_is_valid(buf)); + xe_guc_buf_release(buf); /* shouldn't crash */ +} + +static void test_flush(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf buf; + const u32 dwords = xe_guc_buf_cache_dwords(cache); + const u32 bytes = dwords * sizeof(u32); + u32 *s, *p, *d; + int n; + + KUNIT_ASSERT_NOT_NULL(test, s = kunit_kcalloc(test, dwords, sizeof(u32), GFP_KERNEL)); + KUNIT_ASSERT_NOT_NULL(test, d = kunit_kcalloc(test, dwords, sizeof(u32), GFP_KERNEL)); + + for (n = 0; n < dwords; n++) + s[n] = n; + + buf = xe_guc_buf_reserve(cache, dwords); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_ASSERT_NOT_NULL(test, p = xe_guc_buf_cpu_ptr(buf)); + KUNIT_EXPECT_PTR_NE(test, p, s); + KUNIT_EXPECT_PTR_NE(test, p, d); + + memcpy(p, s, bytes); + KUNIT_EXPECT_NE(test, 0, xe_guc_buf_flush(buf)); + + iosys_map_memcpy_from(d, &cache->sam->bo->vmap, 0, bytes); + KUNIT_EXPECT_MEMEQ(test, s, d, bytes); + + xe_guc_buf_release(buf); +} + +static void test_lookup(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf buf; + u32 dwords; + u64 addr; + u32 *p; + int n; + + dwords = xe_guc_buf_cache_dwords(cache); + buf = xe_guc_buf_reserve(cache, dwords); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_ASSERT_NOT_NULL(test, p = xe_guc_buf_cpu_ptr(buf)); + KUNIT_ASSERT_NE(test, 0, addr = xe_guc_buf_gpu_addr(buf)); + + KUNIT_EXPECT_EQ(test, 0, xe_guc_cache_gpu_addr_from_ptr(cache, p - 1, sizeof(u32))); + KUNIT_EXPECT_EQ(test, 0, xe_guc_cache_gpu_addr_from_ptr(cache, p + dwords, sizeof(u32))); + + for (n = 0; n < dwords; n++) + KUNIT_EXPECT_EQ_MSG(test, xe_guc_cache_gpu_addr_from_ptr(cache, p + n, sizeof(u32)), + addr + n * sizeof(u32), "n=%d", n); + + xe_guc_buf_release(buf); +} + +static void test_data(struct kunit *test) +{ + static const u32 data[] = { 1, 2, 3, 4, 5, 6 }; + struct xe_guc_buf_cache *cache = test->priv; + struct xe_guc_buf buf; + void *p; + + buf = xe_guc_buf_from_data(cache, data, sizeof(data)); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_ASSERT_NOT_NULL(test, p = xe_guc_buf_cpu_ptr(buf)); + KUNIT_EXPECT_MEMEQ(test, p, data, sizeof(data)); + + xe_guc_buf_release(buf); +} + +static void test_class(struct kunit *test) +{ + struct xe_guc_buf_cache *cache = test->priv; + u32 dwords = xe_guc_buf_cache_dwords(cache); + + { + CLASS(xe_guc_buf, buf)(cache, dwords); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_EXPECT_NOT_NULL(test, xe_guc_buf_cpu_ptr(buf)); + KUNIT_EXPECT_NE(test, 0, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_LE(test, DUT_GGTT_START, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_GT(test, DUT_GGTT_START + DUT_GGTT_SIZE, xe_guc_buf_gpu_addr(buf)); + } + + { + CLASS(xe_guc_buf, buf)(cache, dwords); + KUNIT_ASSERT_TRUE(test, xe_guc_buf_is_valid(buf)); + KUNIT_EXPECT_NOT_NULL(test, xe_guc_buf_cpu_ptr(buf)); + KUNIT_EXPECT_NE(test, 0, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_LE(test, DUT_GGTT_START, xe_guc_buf_gpu_addr(buf)); + KUNIT_EXPECT_GT(test, DUT_GGTT_START + DUT_GGTT_SIZE, xe_guc_buf_gpu_addr(buf)); + } +} + +static struct kunit_case guc_buf_test_cases[] = { + KUNIT_CASE(test_smallest), + KUNIT_CASE(test_largest), + KUNIT_CASE(test_granular), + KUNIT_CASE(test_unique), + KUNIT_CASE(test_overlap), + KUNIT_CASE(test_reusable), + KUNIT_CASE(test_too_big), + KUNIT_CASE(test_flush), + KUNIT_CASE(test_lookup), + KUNIT_CASE(test_data), + KUNIT_CASE(test_class), + {} +}; + +static struct kunit_suite guc_buf_suite = { + .name = "guc_buf", + .test_cases = guc_buf_test_cases, + .init = guc_buf_test_init, +}; + +kunit_test_suites(&guc_buf_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_guc_db_mgr_test.c b/drivers/gpu/drm/xe/tests/xe_guc_db_mgr_test.c new file mode 100644 index 000000000..a87a7b4b0 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_guc_db_mgr_test.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include "xe_device.h" +#include "xe_kunit_helpers.h" + +static int guc_dbm_test_init(struct kunit *test) +{ + struct xe_guc_db_mgr *dbm; + + xe_kunit_helper_xe_device_test_init(test); + dbm = &xe_device_get_gt(test->priv, 0)->uc.guc.dbm; + + mutex_init(dbm_mutex(dbm)); + test->priv = dbm; + return 0; +} + +static void test_empty(struct kunit *test) +{ + struct xe_guc_db_mgr *dbm = test->priv; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, 0), 0); + KUNIT_ASSERT_EQ(test, dbm->count, 0); + + mutex_lock(dbm_mutex(dbm)); + KUNIT_EXPECT_LT(test, xe_guc_db_mgr_reserve_id_locked(dbm), 0); + mutex_unlock(dbm_mutex(dbm)); + + KUNIT_EXPECT_LT(test, xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); +} + +static void test_default(struct kunit *test) +{ + struct xe_guc_db_mgr *dbm = test->priv; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, ~0), 0); + KUNIT_ASSERT_EQ(test, dbm->count, GUC_NUM_DOORBELLS); +} + +static const unsigned int guc_dbm_params[] = { + GUC_NUM_DOORBELLS / 64, + GUC_NUM_DOORBELLS / 32, + GUC_NUM_DOORBELLS / 8, + GUC_NUM_DOORBELLS, +}; + +static void uint_param_get_desc(const unsigned int *p, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%u", *p); +} + +KUNIT_ARRAY_PARAM(guc_dbm, guc_dbm_params, uint_param_get_desc); + +static void test_size(struct kunit *test) +{ + const unsigned int *p = test->param_value; + struct xe_guc_db_mgr *dbm = test->priv; + unsigned int n; + int id; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, *p), 0); + KUNIT_ASSERT_EQ(test, dbm->count, *p); + + mutex_lock(dbm_mutex(dbm)); + for (n = 0; n < *p; n++) { + KUNIT_EXPECT_GE(test, id = xe_guc_db_mgr_reserve_id_locked(dbm), 0); + KUNIT_EXPECT_LT(test, id, dbm->count); + } + KUNIT_EXPECT_LT(test, xe_guc_db_mgr_reserve_id_locked(dbm), 0); + mutex_unlock(dbm_mutex(dbm)); + + mutex_lock(dbm_mutex(dbm)); + for (n = 0; n < *p; n++) + xe_guc_db_mgr_release_id_locked(dbm, n); + mutex_unlock(dbm_mutex(dbm)); +} + +static void test_reuse(struct kunit *test) +{ + const unsigned int *p = test->param_value; + struct xe_guc_db_mgr *dbm = test->priv; + unsigned int n; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, *p), 0); + + mutex_lock(dbm_mutex(dbm)); + for (n = 0; n < *p; n++) + KUNIT_EXPECT_GE(test, xe_guc_db_mgr_reserve_id_locked(dbm), 0); + KUNIT_EXPECT_LT(test, xe_guc_db_mgr_reserve_id_locked(dbm), 0); + mutex_unlock(dbm_mutex(dbm)); + + mutex_lock(dbm_mutex(dbm)); + for (n = 0; n < *p; n++) { + xe_guc_db_mgr_release_id_locked(dbm, n); + KUNIT_EXPECT_EQ(test, xe_guc_db_mgr_reserve_id_locked(dbm), n); + } + KUNIT_EXPECT_LT(test, xe_guc_db_mgr_reserve_id_locked(dbm), 0); + mutex_unlock(dbm_mutex(dbm)); + + mutex_lock(dbm_mutex(dbm)); + for (n = 0; n < *p; n++) + xe_guc_db_mgr_release_id_locked(dbm, n); + mutex_unlock(dbm_mutex(dbm)); +} + +static void test_range_overlap(struct kunit *test) +{ + const unsigned int *p = test->param_value; + struct xe_guc_db_mgr *dbm = test->priv; + int id1, id2, id3; + unsigned int n; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, ~0), 0); + KUNIT_ASSERT_LE(test, *p, dbm->count); + + KUNIT_ASSERT_GE(test, id1 = xe_guc_db_mgr_reserve_range(dbm, *p, 0), 0); + for (n = 0; n < dbm->count - *p; n++) { + KUNIT_ASSERT_GE(test, id2 = xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); + KUNIT_ASSERT_NE(test, id2, id1); + KUNIT_ASSERT_NE_MSG(test, id2 < id1, id2 > id1 + *p - 1, + "id1=%d id2=%d", id1, id2); + } + KUNIT_ASSERT_LT(test, xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); + xe_guc_db_mgr_release_range(dbm, 0, dbm->count); + + if (*p >= 1) { + KUNIT_ASSERT_GE(test, id1 = xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); + KUNIT_ASSERT_GE(test, id2 = xe_guc_db_mgr_reserve_range(dbm, *p - 1, 0), 0); + KUNIT_ASSERT_NE(test, id2, id1); + KUNIT_ASSERT_NE_MSG(test, id1 < id2, id1 > id2 + *p - 2, + "id1=%d id2=%d", id1, id2); + for (n = 0; n < dbm->count - *p; n++) { + KUNIT_ASSERT_GE(test, id3 = xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); + KUNIT_ASSERT_NE(test, id3, id1); + KUNIT_ASSERT_NE(test, id3, id2); + KUNIT_ASSERT_NE_MSG(test, id3 < id2, id3 > id2 + *p - 2, + "id3=%d id2=%d", id3, id2); + } + KUNIT_ASSERT_LT(test, xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); + xe_guc_db_mgr_release_range(dbm, 0, dbm->count); + } +} + +static void test_range_compact(struct kunit *test) +{ + const unsigned int *p = test->param_value; + struct xe_guc_db_mgr *dbm = test->priv; + unsigned int n; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, ~0), 0); + KUNIT_ASSERT_NE(test, *p, 0); + KUNIT_ASSERT_LE(test, *p, dbm->count); + if (dbm->count % *p) + kunit_skip(test, "must be divisible"); + + KUNIT_ASSERT_GE(test, xe_guc_db_mgr_reserve_range(dbm, *p, 0), 0); + for (n = 1; n < dbm->count / *p; n++) + KUNIT_ASSERT_GE(test, xe_guc_db_mgr_reserve_range(dbm, *p, 0), 0); + KUNIT_ASSERT_LT(test, xe_guc_db_mgr_reserve_range(dbm, 1, 0), 0); + xe_guc_db_mgr_release_range(dbm, 0, dbm->count); +} + +static void test_range_spare(struct kunit *test) +{ + const unsigned int *p = test->param_value; + struct xe_guc_db_mgr *dbm = test->priv; + int id; + + KUNIT_ASSERT_EQ(test, xe_guc_db_mgr_init(dbm, ~0), 0); + KUNIT_ASSERT_LE(test, *p, dbm->count); + + KUNIT_ASSERT_LT(test, xe_guc_db_mgr_reserve_range(dbm, *p, dbm->count), 0); + KUNIT_ASSERT_LT(test, xe_guc_db_mgr_reserve_range(dbm, *p, dbm->count - *p + 1), 0); + KUNIT_ASSERT_EQ(test, id = xe_guc_db_mgr_reserve_range(dbm, *p, dbm->count - *p), 0); + KUNIT_ASSERT_LT(test, xe_guc_db_mgr_reserve_range(dbm, 1, dbm->count - *p), 0); + xe_guc_db_mgr_release_range(dbm, id, *p); +} + +static struct kunit_case guc_dbm_test_cases[] = { + KUNIT_CASE(test_empty), + KUNIT_CASE(test_default), + KUNIT_CASE_PARAM(test_size, guc_dbm_gen_params), + KUNIT_CASE_PARAM(test_reuse, guc_dbm_gen_params), + KUNIT_CASE_PARAM(test_range_overlap, guc_dbm_gen_params), + KUNIT_CASE_PARAM(test_range_compact, guc_dbm_gen_params), + KUNIT_CASE_PARAM(test_range_spare, guc_dbm_gen_params), + {} +}; + +static struct kunit_suite guc_dbm_suite = { + .name = "guc_dbm", + .test_cases = guc_dbm_test_cases, + .init = guc_dbm_test_init, +}; + +kunit_test_suites(&guc_dbm_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c b/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c new file mode 100644 index 000000000..ea8ca03ef --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c @@ -0,0 +1,833 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include +#include + +#include "tests/xe_kunit_helpers.h" +#include "tests/xe_pci_test.h" +#include "tests/xe_test.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_pm.h" + +/* + * There are different ways to allocate the G2G buffers. The plan for this test + * is to make sure that all the possible options work. The particular option + * chosen by the driver may vary from one platform to another, it may also change + * with time. So to ensure consistency of testing, the relevant driver code is + * replicated here to guarantee it won't change without the test being updated + * to keep testing the other options. + * + * In order to test the actual code being used by the driver, there is also the + * 'default' scheme. That will use the official driver routines to test whatever + * method the driver is using on the current platform at the current time. + */ +enum { + /* Driver defined allocation scheme */ + G2G_CTB_TYPE_DEFAULT, + /* Single buffer in host memory */ + G2G_CTB_TYPE_HOST, + /* Single buffer in a specific tile, loops across all tiles */ + G2G_CTB_TYPE_TILE, +}; + +/* + * Payload is opaque to GuC. So KMD can define any structure or size it wants. + */ +struct g2g_test_payload { + u32 tx_dev; + u32 tx_tile; + u32 rx_dev; + u32 rx_tile; + u32 seqno; +}; + +static int slot_index_from_gts(struct xe_gt *tx_gt, struct xe_gt *rx_gt) +{ + struct xe_device *xe = gt_to_xe(tx_gt); + int idx = 0, found = 0, id, tx_idx, rx_idx; + struct xe_gt *gt; + struct kunit *test = kunit_get_current_test(); + + for (id = 0; id < xe->info.tile_count * xe->info.max_gt_per_tile; id++) { + gt = xe_device_get_gt(xe, id); + if (!gt) + continue; + if (gt == tx_gt) { + tx_idx = idx; + found++; + } + if (gt == rx_gt) { + rx_idx = idx; + found++; + } + + if (found == 2) + break; + + idx++; + } + + if (found != 2) + KUNIT_FAIL(test, "GT index not found"); + + return (tx_idx * xe->info.gt_count) + rx_idx; +} + +static void g2g_test_send(struct kunit *test, struct xe_guc *guc, + u32 far_tile, u32 far_dev, + struct g2g_test_payload *payload) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + u32 *action, total; + size_t payload_len; + int ret; + + static_assert(IS_ALIGNED(sizeof(*payload), sizeof(u32))); + payload_len = sizeof(*payload) / sizeof(u32); + + total = 4 + payload_len; + action = kunit_kmalloc_array(test, total, sizeof(*action), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, action); + + action[0] = XE_GUC_ACTION_TEST_G2G_SEND; + action[1] = far_tile; + action[2] = far_dev; + action[3] = payload_len; + memcpy(action + 4, payload, payload_len * sizeof(u32)); + + atomic_inc(&xe->g2g_test_count); + + /* + * Should specify the expected response notification here. Problem is that + * the response will be coming from a different GuC. By the end, it should + * all add up as long as an equal number of messages are sent from each GuC + * and to each GuC. However, in the middle negative reservation space errors + * and such like can occur. Rather than add intrusive changes to the CT layer + * it is simpler to just not bother counting it at all. The system should be + * idle when running the selftest, and the selftest's notification total size + * is well within the G2H allocation size. So there should be no issues with + * needing to block for space, which is all the tracking code is really for. + */ + ret = xe_guc_ct_send(&guc->ct, action, total, 0, 0); + kunit_kfree(test, action); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "G2G send failed: %d [%d:%d -> %d:%d]\n", ret, + gt_to_tile(gt)->id, G2G_DEV(gt), far_tile, far_dev); +} + +/* + * NB: Can't use KUNIT_ASSERT and friends in here as this is called asynchronously + * from the G2H notification handler. Need that to actually complete rather than + * thread-abort in order to keep the rest of the driver alive! + */ +int xe_guc_g2g_test_notification(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *rx_gt = guc_to_gt(guc), *test_gt, *tx_gt = NULL; + u32 tx_tile, tx_dev, rx_tile, rx_dev, idx, got_len; + struct g2g_test_payload *payload; + size_t payload_len; + int ret = 0, i; + + payload_len = sizeof(*payload) / sizeof(u32); + + if (unlikely(len != (G2H_LEN_DW_G2G_NOTIFY_MIN + payload_len))) { + xe_gt_err(rx_gt, "G2G test notification invalid length %u", len); + ret = -EPROTO; + goto done; + } + + tx_tile = msg[0]; + tx_dev = msg[1]; + got_len = msg[2]; + payload = (struct g2g_test_payload *)(msg + 3); + + rx_tile = gt_to_tile(rx_gt)->id; + rx_dev = G2G_DEV(rx_gt); + + if (got_len != payload_len) { + xe_gt_err(rx_gt, "G2G: Invalid payload length: %u vs %zu\n", got_len, payload_len); + ret = -EPROTO; + goto done; + } + + if (payload->tx_dev != tx_dev || payload->tx_tile != tx_tile || + payload->rx_dev != rx_dev || payload->rx_tile != rx_tile) { + xe_gt_err(rx_gt, "G2G: Invalid payload: %d:%d -> %d:%d vs %d:%d -> %d:%d! [%d]\n", + payload->tx_tile, payload->tx_dev, payload->rx_tile, payload->rx_dev, + tx_tile, tx_dev, rx_tile, rx_dev, payload->seqno); + ret = -EPROTO; + goto done; + } + + if (!xe->g2g_test_array) { + xe_gt_err(rx_gt, "G2G: Missing test array!\n"); + ret = -ENOMEM; + goto done; + } + + for_each_gt(test_gt, xe, i) { + if (gt_to_tile(test_gt)->id != tx_tile) + continue; + + if (G2G_DEV(test_gt) != tx_dev) + continue; + + if (tx_gt) { + xe_gt_err(rx_gt, "G2G: Got duplicate TX GTs: %d vs %d for %d:%d!\n", + tx_gt->info.id, test_gt->info.id, tx_tile, tx_dev); + ret = -EINVAL; + goto done; + } + + tx_gt = test_gt; + } + if (!tx_gt) { + xe_gt_err(rx_gt, "G2G: Failed to find a TX GT for %d:%d!\n", tx_tile, tx_dev); + ret = -EINVAL; + goto done; + } + + idx = slot_index_from_gts(tx_gt, rx_gt); + + if (xe->g2g_test_array[idx] != payload->seqno - 1) { + xe_gt_err(rx_gt, "G2G: Seqno mismatch %d vs %d for %d:%d -> %d:%d!\n", + xe->g2g_test_array[idx], payload->seqno - 1, + tx_tile, tx_dev, rx_tile, rx_dev); + ret = -EINVAL; + goto done; + } + + xe->g2g_test_array[idx] = payload->seqno; + +done: + atomic_dec(&xe->g2g_test_count); + return ret; +} + +#define G2G_WAIT_TIMEOUT_MS 100 +#define G2G_WAIT_POLL_MS 1 + +/* + * Send the given seqno from all GuCs to all other GuCs in tile/GT order + */ +static void g2g_test_in_order(struct kunit *test, struct xe_device *xe, u32 seqno) +{ + struct xe_gt *near_gt, *far_gt; + int i, j, waited; + u32 idx; + + for_each_gt(near_gt, xe, i) { + u32 near_tile = gt_to_tile(near_gt)->id; + u32 near_dev = G2G_DEV(near_gt); + + for_each_gt(far_gt, xe, j) { + u32 far_tile = gt_to_tile(far_gt)->id; + u32 far_dev = G2G_DEV(far_gt); + struct g2g_test_payload payload; + + if (far_gt->info.id == near_gt->info.id) + continue; + + payload.tx_dev = near_dev; + payload.tx_tile = near_tile; + payload.rx_dev = far_dev; + payload.rx_tile = far_tile; + payload.seqno = seqno; + + /* Calculate idx for event-based wait */ + idx = slot_index_from_gts(near_gt, far_gt); + waited = 0; + + /* + * Wait for previous seqno to be acknowledged before sending, + * to avoid queuing too many back-to-back messages and + * causing a test timeout. Actual correctness of message + * will be checked later in xe_guc_g2g_test_notification() + */ + while (xe->g2g_test_array[idx] != (seqno - 1)) { + msleep(G2G_WAIT_POLL_MS); + waited += G2G_WAIT_POLL_MS; + if (waited >= G2G_WAIT_TIMEOUT_MS) { + kunit_info(test, "Timeout waiting! tx gt: %d, rx gt: %d\n", + near_gt->info.id, far_gt->info.id); + break; + } + } + + g2g_test_send(test, &near_gt->uc.guc, far_tile, far_dev, &payload); + } + } +} + +#define WAIT_TIME_MS 100 +#define WAIT_COUNT (1000 / WAIT_TIME_MS) + +static void g2g_wait_for_complete(void *_xe) +{ + struct xe_device *xe = (struct xe_device *)_xe; + struct kunit *test = kunit_get_current_test(); + int wait = 0; + + /* Wait for all G2H messages to be received */ + while (atomic_read(&xe->g2g_test_count)) { + if (++wait > WAIT_COUNT) + break; + + msleep(WAIT_TIME_MS); + } + + KUNIT_ASSERT_EQ_MSG(test, 0, atomic_read(&xe->g2g_test_count), + "Timed out waiting for notifications\n"); + kunit_info(test, "Got all notifications back\n"); +} + +#undef WAIT_TIME_MS +#undef WAIT_COUNT + +static void g2g_clean_array(void *_xe) +{ + struct xe_device *xe = (struct xe_device *)_xe; + + xe->g2g_test_array = NULL; +} + +#define NUM_LOOPS 16 + +static void g2g_run_test(struct kunit *test, struct xe_device *xe) +{ + u32 seqno, max_array; + int ret, i, j; + + max_array = xe->info.gt_count * xe->info.gt_count; + xe->g2g_test_array = kunit_kcalloc(test, max_array, sizeof(u32), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe->g2g_test_array); + + ret = kunit_add_action_or_reset(test, g2g_clean_array, xe); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "Failed to register clean up action\n"); + + /* + * Send incrementing seqnos from all GuCs to all other GuCs in tile/GT order. + * Tile/GT order doesn't really mean anything to the hardware but it is going + * to be a fixed sequence every time. + * + * Verify that each one comes back having taken the correct route. + */ + ret = kunit_add_action(test, g2g_wait_for_complete, xe); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "Failed to register clean up action\n"); + for (seqno = 1; seqno < NUM_LOOPS; seqno++) + g2g_test_in_order(test, xe, seqno); + seqno--; + + kunit_release_action(test, &g2g_wait_for_complete, xe); + + /* Check for the final seqno in each slot */ + for (i = 0; i < xe->info.gt_count; i++) { + for (j = 0; j < xe->info.gt_count; j++) { + u32 idx = (j * xe->info.gt_count) + i; + + if (i == j) + KUNIT_ASSERT_EQ_MSG(test, 0, xe->g2g_test_array[idx], + "identity seqno modified: %d for %dx%d!\n", + xe->g2g_test_array[idx], i, j); + else + KUNIT_ASSERT_EQ_MSG(test, seqno, xe->g2g_test_array[idx], + "invalid seqno: %d vs %d for %dx%d!\n", + xe->g2g_test_array[idx], seqno, i, j); + } + } + + kunit_kfree(test, xe->g2g_test_array); + kunit_release_action(test, &g2g_clean_array, xe); + + kunit_info(test, "Test passed\n"); +} + +#undef NUM_LOOPS + +static void g2g_ct_stop(struct xe_guc *guc) +{ + struct xe_gt *remote_gt, *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + int i, t; + + for_each_gt(remote_gt, xe, i) { + u32 tile, dev; + + if (remote_gt->info.id == gt->info.id) + continue; + + tile = gt_to_tile(remote_gt)->id; + dev = G2G_DEV(remote_gt); + + for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) + guc_g2g_deregister(guc, tile, dev, t); + } +} + +/* Size of a single allocation that contains all G2G CTBs across all GTs */ +static u32 g2g_ctb_size(struct kunit *test, struct xe_device *xe) +{ + unsigned int count = xe->info.gt_count; + u32 num_channels = (count * (count - 1)) / 2; + + kunit_info(test, "Size: (%d * %d / 2) * %d * 0x%08X + 0x%08X => 0x%08X [%d]\n", + count, count - 1, XE_G2G_TYPE_LIMIT, G2G_BUFFER_SIZE, G2G_DESC_AREA_SIZE, + num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE, + num_channels * XE_G2G_TYPE_LIMIT); + + return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE; +} + +/* + * Use the driver's regular CTB allocation scheme. + */ +static void g2g_alloc_default(struct kunit *test, struct xe_device *xe) +{ + struct xe_gt *gt; + int i; + + kunit_info(test, "Default [tiles = %d, GTs = %d]\n", + xe->info.tile_count, xe->info.gt_count); + + for_each_gt(gt, xe, i) { + struct xe_guc *guc = >->uc.guc; + int ret; + + ret = guc_g2g_alloc(guc); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "G2G alloc failed: %pe", ERR_PTR(ret)); + continue; + } +} + +static void g2g_distribute(struct kunit *test, struct xe_device *xe, struct xe_bo *bo) +{ + struct xe_gt *root_gt, *gt; + int i; + + root_gt = xe_device_get_gt(xe, 0); + root_gt->uc.guc.g2g.bo = bo; + root_gt->uc.guc.g2g.owned = true; + kunit_info(test, "[%d.%d] Assigned 0x%p\n", gt_to_tile(root_gt)->id, root_gt->info.id, bo); + + for_each_gt(gt, xe, i) { + if (gt->info.id != 0) { + gt->uc.guc.g2g.owned = false; + gt->uc.guc.g2g.bo = xe_bo_get(bo); + kunit_info(test, "[%d.%d] Pinned 0x%p\n", + gt_to_tile(gt)->id, gt->info.id, gt->uc.guc.g2g.bo); + } + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gt->uc.guc.g2g.bo); + } +} + +/* + * Allocate a single blob on the host and split between all G2G CTBs. + */ +static void g2g_alloc_host(struct kunit *test, struct xe_device *xe) +{ + struct xe_bo *bo; + u32 g2g_size; + + kunit_info(test, "Host [tiles = %d, GTs = %d]\n", xe->info.tile_count, xe->info.gt_count); + + g2g_size = g2g_ctb_size(test, xe); + bo = xe_managed_bo_create_pin_map(xe, xe_device_get_root_tile(xe), g2g_size, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_ALL | + XE_BO_FLAG_GGTT_INVALIDATE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bo); + kunit_info(test, "[HST] G2G buffer create: 0x%p\n", bo); + + xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size); + + g2g_distribute(test, xe, bo); +} + +/* + * Allocate a single blob on the given tile and split between all G2G CTBs. + */ +static void g2g_alloc_tile(struct kunit *test, struct xe_device *xe, struct xe_tile *tile) +{ + struct xe_bo *bo; + u32 g2g_size; + + KUNIT_ASSERT_TRUE(test, IS_DGFX(xe)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tile); + + kunit_info(test, "Tile %d [tiles = %d, GTs = %d]\n", + tile->id, xe->info.tile_count, xe->info.gt_count); + + g2g_size = g2g_ctb_size(test, xe); + bo = xe_managed_bo_create_pin_map(xe, tile, g2g_size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_ALL | + XE_BO_FLAG_GGTT_INVALIDATE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bo); + kunit_info(test, "[%d.*] G2G buffer create: 0x%p\n", tile->id, bo); + + xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size); + + g2g_distribute(test, xe, bo); +} + +static void g2g_free(struct kunit *test, struct xe_device *xe) +{ + struct xe_gt *gt; + struct xe_bo *bo; + int i; + + for_each_gt(gt, xe, i) { + bo = gt->uc.guc.g2g.bo; + if (!bo) + continue; + + if (gt->uc.guc.g2g.owned) { + xe_managed_bo_unpin_map_no_vm(bo); + kunit_info(test, "[%d.%d] Unmapped 0x%p\n", + gt_to_tile(gt)->id, gt->info.id, bo); + } else { + xe_bo_put(bo); + kunit_info(test, "[%d.%d] Unpinned 0x%p\n", + gt_to_tile(gt)->id, gt->info.id, bo); + } + + gt->uc.guc.g2g.bo = NULL; + } +} + +static void g2g_stop(struct kunit *test, struct xe_device *xe) +{ + struct xe_gt *gt; + int i; + + for_each_gt(gt, xe, i) { + struct xe_guc *guc = >->uc.guc; + + if (!guc->g2g.bo) + continue; + + g2g_ct_stop(guc); + } + + g2g_free(test, xe); +} + +/* + * Generate a unique id for each bi-directional CTB for each pair of + * near and far tiles/devices. The id can then be used as an index into + * a single allocation that is sub-divided into multiple CTBs. + * + * For example, with two devices per tile and two tiles, the table should + * look like: + * Far . + * 0.0 0.1 1.0 1.1 + * N 0.0 --/-- 00/01 02/03 04/05 + * e 0.1 01/00 --/-- 06/07 08/09 + * a 1.0 03/02 07/06 --/-- 10/11 + * r 1.1 05/04 09/08 11/10 --/-- + * + * Where each entry is Rx/Tx channel id. + * + * So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would + * be reading from channel #11 and writing to channel #10. Whereas, + * GuC #2 talking to GuC #3 would be read on #10 and write to #11. + */ +static int g2g_slot_flat(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev, + u32 type, u32 max_inst, bool have_dev) +{ + u32 near = near_tile, far = far_tile; + u32 idx = 0, x, y, direction; + int i; + + if (have_dev) { + near = (near << 1) | near_dev; + far = (far << 1) | far_dev; + } + + /* No need to send to one's self */ + if (far == near) + return -1; + + if (far > near) { + /* Top right table half */ + x = far; + y = near; + + /* T/R is 'forwards' direction */ + direction = type; + } else { + /* Bottom left table half */ + x = near; + y = far; + + /* B/L is 'backwards' direction */ + direction = (1 - type); + } + + /* Count the rows prior to the target */ + for (i = y; i > 0; i--) + idx += max_inst - i; + + /* Count this row up to the target */ + idx += (x - 1 - y); + + /* Slots are in Rx/Tx pairs */ + idx *= 2; + + /* Pick Rx/Tx direction */ + idx += direction; + + return idx; +} + +static int g2g_register_flat(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type, bool have_dev) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + u32 near_tile = gt_to_tile(gt)->id; + u32 near_dev = G2G_DEV(gt); + u32 max = xe->info.gt_count; + int idx; + u32 base, desc, buf; + + if (!guc->g2g.bo) + return -ENODEV; + + idx = g2g_slot_flat(near_tile, near_dev, far_tile, far_dev, type, max, have_dev); + xe_assert(xe, idx >= 0); + + base = guc_bo_ggtt_addr(guc, guc->g2g.bo); + desc = base + idx * G2G_DESC_SIZE; + buf = base + idx * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE; + + xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE); + xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= xe_bo_size(guc->g2g.bo)); + + return guc_action_register_g2g_buffer(guc, type, far_tile, far_dev, + desc, buf, G2G_BUFFER_SIZE); +} + +static void g2g_start(struct kunit *test, struct xe_guc *guc) +{ + struct xe_gt *remote_gt, *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + unsigned int i; + int t, ret; + bool have_dev; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, guc->g2g.bo); + + /* GuC interface will need extending if more GT device types are ever created. */ + KUNIT_ASSERT_TRUE(test, + (gt->info.type == XE_GT_TYPE_MAIN) || + (gt->info.type == XE_GT_TYPE_MEDIA)); + + /* Channel numbering depends on whether there are multiple GTs per tile */ + have_dev = xe->info.gt_count > xe->info.tile_count; + + for_each_gt(remote_gt, xe, i) { + u32 tile, dev; + + if (remote_gt->info.id == gt->info.id) + continue; + + tile = gt_to_tile(remote_gt)->id; + dev = G2G_DEV(remote_gt); + + for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) { + ret = g2g_register_flat(guc, tile, dev, t, have_dev); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "G2G register failed: %pe", ERR_PTR(ret)); + } + } +} + +static void g2g_reinit(struct kunit *test, struct xe_device *xe, int ctb_type, struct xe_tile *tile) +{ + struct xe_gt *gt; + int i, found = 0; + + g2g_stop(test, xe); + + for_each_gt(gt, xe, i) { + struct xe_guc *guc = >->uc.guc; + + KUNIT_ASSERT_NULL(test, guc->g2g.bo); + } + + switch (ctb_type) { + case G2G_CTB_TYPE_DEFAULT: + g2g_alloc_default(test, xe); + break; + + case G2G_CTB_TYPE_HOST: + g2g_alloc_host(test, xe); + break; + + case G2G_CTB_TYPE_TILE: + g2g_alloc_tile(test, xe, tile); + break; + + default: + KUNIT_ASSERT_TRUE(test, false); + } + + for_each_gt(gt, xe, i) { + struct xe_guc *guc = >->uc.guc; + + if (!guc->g2g.bo) + continue; + + if (ctb_type == G2G_CTB_TYPE_DEFAULT) + guc_g2g_start(guc); + else + g2g_start(test, guc); + found++; + } + + KUNIT_ASSERT_GT_MSG(test, found, 1, "insufficient G2G channels running: %d", found); + + kunit_info(test, "Testing across %d GTs\n", found); +} + +static void g2g_recreate_ctb(void *_xe) +{ + struct xe_device *xe = (struct xe_device *)_xe; + struct kunit *test = kunit_get_current_test(); + + g2g_stop(test, xe); + + if (xe_guc_g2g_wanted(xe)) + g2g_reinit(test, xe, G2G_CTB_TYPE_DEFAULT, NULL); +} + +static void g2g_pm_runtime_put(void *_xe) +{ + struct xe_device *xe = (struct xe_device *)_xe; + + xe_pm_runtime_put(xe); +} + +static void g2g_pm_runtime_get(struct kunit *test) +{ + struct xe_device *xe = test->priv; + int ret; + + xe_pm_runtime_get(xe); + ret = kunit_add_action_or_reset(test, g2g_pm_runtime_put, xe); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "Failed to register runtime PM action\n"); +} + +static void g2g_check_skip(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct xe_gt *gt; + int i; + + if (IS_SRIOV_VF(xe)) + kunit_skip(test, "not supported from a VF"); + + if (xe->info.gt_count <= 1) + kunit_skip(test, "not enough GTs"); + + for_each_gt(gt, xe, i) { + struct xe_guc *guc = >->uc.guc; + + if (guc->fw.build_type == CSS_UKERNEL_INFO_BUILDTYPE_PROD) + kunit_skip(test, + "G2G test interface not available in production firmware builds\n"); + } +} + +/* + * Simple test that does not try to recreate the CTBs. + * Requires that the platform already enables G2G comms + * but has no risk of leaving the system in a broken state + * afterwards. + */ +static void xe_live_guc_g2g_kunit_default(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + if (!xe_guc_g2g_wanted(xe)) + kunit_skip(test, "G2G not enabled"); + + g2g_check_skip(test); + + g2g_pm_runtime_get(test); + + kunit_info(test, "Testing default CTBs\n"); + g2g_run_test(test, xe); + + kunit_release_action(test, &g2g_pm_runtime_put, xe); +} + +/* + * More complex test that re-creates the CTBs in various location to + * test access to each location from each GuC. Can be run even on + * systems that do not enable G2G by default. On the other hand, + * because it recreates the CTBs, if something goes wrong it could + * leave the system with broken G2G comms. + */ +static void xe_live_guc_g2g_kunit_allmem(struct kunit *test) +{ + struct xe_device *xe = test->priv; + int ret; + + g2g_check_skip(test); + + g2g_pm_runtime_get(test); + + /* Make sure to leave the system as we found it */ + ret = kunit_add_action_or_reset(test, g2g_recreate_ctb, xe); + KUNIT_ASSERT_EQ_MSG(test, 0, ret, "Failed to register CTB re-creation action\n"); + + kunit_info(test, "Testing CTB type 'default'...\n"); + g2g_reinit(test, xe, G2G_CTB_TYPE_DEFAULT, NULL); + g2g_run_test(test, xe); + + kunit_info(test, "Testing CTB type 'host'...\n"); + g2g_reinit(test, xe, G2G_CTB_TYPE_HOST, NULL); + g2g_run_test(test, xe); + + if (IS_DGFX(xe)) { + struct xe_tile *tile; + int id; + + for_each_tile(tile, xe, id) { + kunit_info(test, "Testing CTB type 'tile: #%d'...\n", id); + + g2g_reinit(test, xe, G2G_CTB_TYPE_TILE, tile); + g2g_run_test(test, xe); + } + } else { + kunit_info(test, "Skipping local memory on integrated platform\n"); + } + + kunit_release_action(test, g2g_recreate_ctb, xe); + kunit_release_action(test, g2g_pm_runtime_put, xe); +} + +static struct kunit_case xe_guc_g2g_tests[] = { + KUNIT_CASE_PARAM(xe_live_guc_g2g_kunit_default, xe_pci_live_device_gen_param), + KUNIT_CASE_PARAM(xe_live_guc_g2g_kunit_allmem, xe_pci_live_device_gen_param), + {} +}; + +VISIBLE_IF_KUNIT +struct kunit_suite xe_guc_g2g_test_suite = { + .name = "xe_guc_g2g", + .test_cases = xe_guc_g2g_tests, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_guc_g2g_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c b/drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c new file mode 100644 index 000000000..ee30a1939 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include "xe_device.h" +#include "xe_kunit_helpers.h" + +static int guc_id_mgr_test_init(struct kunit *test) +{ + struct xe_guc_id_mgr *idm; + + xe_kunit_helper_xe_device_test_init(test); + idm = &xe_device_get_gt(test->priv, 0)->uc.guc.submission_state.idm; + + mutex_init(idm_mutex(idm)); + test->priv = idm; + return 0; +} + +static void bad_init(struct kunit *test) +{ + struct xe_guc_id_mgr *idm = test->priv; + + KUNIT_EXPECT_EQ(test, -EINVAL, xe_guc_id_mgr_init(idm, 0)); + KUNIT_EXPECT_EQ(test, -ERANGE, xe_guc_id_mgr_init(idm, GUC_ID_MAX + 1)); +} + +static void no_init(struct kunit *test) +{ + struct xe_guc_id_mgr *idm = test->priv; + + mutex_lock(idm_mutex(idm)); + KUNIT_EXPECT_EQ(test, -ENODATA, xe_guc_id_mgr_reserve_locked(idm, 0)); + mutex_unlock(idm_mutex(idm)); + + KUNIT_EXPECT_EQ(test, -ENODATA, xe_guc_id_mgr_reserve(idm, 1, 1)); +} + +static void init_fini(struct kunit *test) +{ + struct xe_guc_id_mgr *idm = test->priv; + + KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, -1)); + KUNIT_EXPECT_NOT_NULL(test, idm->bitmap); + KUNIT_EXPECT_EQ(test, idm->total, GUC_ID_MAX); + __fini_idm(NULL, idm); + KUNIT_EXPECT_NULL(test, idm->bitmap); + KUNIT_EXPECT_EQ(test, idm->total, 0); +} + +static void check_used(struct kunit *test) +{ + struct xe_guc_id_mgr *idm = test->priv; + unsigned int n; + + KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, 2)); + + mutex_lock(idm_mutex(idm)); + + for (n = 0; n < idm->total; n++) { + kunit_info(test, "n=%u", n); + KUNIT_EXPECT_EQ(test, idm->used, n); + KUNIT_EXPECT_GE(test, idm_reserve_chunk_locked(idm, 1, 0), 0); + KUNIT_EXPECT_EQ(test, idm->used, n + 1); + } + KUNIT_EXPECT_EQ(test, idm->used, idm->total); + idm_release_chunk_locked(idm, 0, idm->used); + KUNIT_EXPECT_EQ(test, idm->used, 0); + + mutex_unlock(idm_mutex(idm)); +} + +static void check_quota(struct kunit *test) +{ + struct xe_guc_id_mgr *idm = test->priv; + unsigned int n; + + KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, 2)); + + mutex_lock(idm_mutex(idm)); + + for (n = 0; n < idm->total - 1; n++) { + kunit_info(test, "n=%u", n); + KUNIT_EXPECT_EQ(test, idm_reserve_chunk_locked(idm, 1, idm->total), -EDQUOT); + KUNIT_EXPECT_EQ(test, idm_reserve_chunk_locked(idm, 1, idm->total - n), -EDQUOT); + KUNIT_EXPECT_EQ(test, idm_reserve_chunk_locked(idm, idm->total - n, 1), -EDQUOT); + KUNIT_EXPECT_GE(test, idm_reserve_chunk_locked(idm, 1, 1), 0); + } + KUNIT_EXPECT_LE(test, 0, idm_reserve_chunk_locked(idm, 1, 0)); + KUNIT_EXPECT_EQ(test, idm->used, idm->total); + idm_release_chunk_locked(idm, 0, idm->total); + KUNIT_EXPECT_EQ(test, idm->used, 0); + + mutex_unlock(idm_mutex(idm)); +} + +static void check_all(struct kunit *test) +{ + struct xe_guc_id_mgr *idm = test->priv; + unsigned int n; + + KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, -1)); + + mutex_lock(idm_mutex(idm)); + + for (n = 0; n < idm->total; n++) + KUNIT_EXPECT_LE(test, 0, idm_reserve_chunk_locked(idm, 1, 0)); + KUNIT_EXPECT_EQ(test, idm->used, idm->total); + for (n = 0; n < idm->total; n++) + idm_release_chunk_locked(idm, n, 1); + + mutex_unlock(idm_mutex(idm)); +} + +static struct kunit_case guc_id_mgr_test_cases[] = { + KUNIT_CASE(bad_init), + KUNIT_CASE(no_init), + KUNIT_CASE(init_fini), + KUNIT_CASE(check_used), + KUNIT_CASE(check_quota), + KUNIT_CASE_SLOW(check_all), + {} +}; + +static struct kunit_suite guc_id_mgr_suite = { + .name = "guc_idm", + .test_cases = guc_id_mgr_test_cases, + + .init = guc_id_mgr_test_init, + .exit = NULL, +}; + +kunit_test_suites(&guc_id_mgr_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c new file mode 100644 index 000000000..82869363a --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include +#include + +#define TEST_KEY (GUC_KLV_RESERVED_RANGE_START + 0x3de) +#define TEST_GROUP_KEY (GUC_KLV_RESERVED_RANGE_START + 0x3f0) +#define TEST_PAD 0xdeadbeef + +static bool fake_is_group_key(u16 key) +{ + return is_reserved_key(key) && key >= TEST_GROUP_KEY; +} + +static void test_count(struct kunit *test) +{ + u32 value = 0x12345678; + u16 key = TEST_KEY; + u32 klvs[] = { + PREP_GUC_KLV(key + 0, 0), + PREP_GUC_KLV(key + 1, 1), value, + PREP_GUC_KLV(key + 2, 2), value, value, + PREP_GUC_KLV(key + 3, 0), + 0, /* padding */ + }; + + KUNIT_EXPECT_EQ(test, 0, xe_guc_klv_count(klvs, 0)); + KUNIT_EXPECT_EQ(test, 1, xe_guc_klv_count(klvs, 1)); + KUNIT_EXPECT_GT(test, 0, xe_guc_klv_count(klvs, 2)); + KUNIT_EXPECT_EQ(test, 2, xe_guc_klv_count(klvs, 3)); + KUNIT_EXPECT_GT(test, 0, xe_guc_klv_count(klvs, 4)); + KUNIT_EXPECT_GT(test, 0, xe_guc_klv_count(klvs, 5)); + KUNIT_EXPECT_EQ(test, 3, xe_guc_klv_count(klvs, 6)); + KUNIT_EXPECT_EQ(test, 4, xe_guc_klv_count(klvs, 7)); + + /* 0 is treated as reserved KLV { KEY=0, LEN=0 } */ + KUNIT_EXPECT_EQ(test, 5, xe_guc_klv_count(klvs, 8)); +} + +static void test_encode_u32(struct kunit *test) +{ + u32 *fail = ERR_PTR(-ENOMEM); + u32 value = 0x12345678; + u16 key = TEST_KEY; + u32 klvs[16]; + + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u32(klvs, 0, key, value)); + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u32(klvs, 1, key, value)); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe_guc_klv_encode_u32(klvs, 2, key, value)); + KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, 1)); + KUNIT_EXPECT_EQ(test, klvs[1], value); + KUNIT_EXPECT_EQ(test, klvs[2], TEST_PAD); + KUNIT_EXPECT_PTR_EQ(test, &klvs[2], xe_guc_klv_encode_u32(klvs, 2, key, value)); + KUNIT_EXPECT_PTR_EQ(test, + xe_guc_klv_encode_u32(klvs, 2, key, value), + xe_guc_klv_encode_u32(klvs, ARRAY_SIZE(klvs), key, value)); + + KUNIT_ASSERT_PTR_EQ(test, fail, xe_guc_klv_encode_u32(fail, ARRAY_SIZE(klvs), key, value)); +} + +static void test_encode_u64(struct kunit *test) +{ + u64 value = 0x123456789abcdef0; + u32 *fail = ERR_PTR(-ENOMEM); + u16 key = TEST_KEY; + u32 klvs[16]; + + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u64(klvs, 0, key, value)); + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u64(klvs, 1, key, value)); + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u64(klvs, 2, key, value)); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe_guc_klv_encode_u64(klvs, 3, key, value)); + KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, 2)); + KUNIT_EXPECT_EQ(test, klvs[1], lower_32_bits(value)); + KUNIT_EXPECT_EQ(test, klvs[2], upper_32_bits(value)); + KUNIT_EXPECT_EQ(test, klvs[3], TEST_PAD); + KUNIT_EXPECT_PTR_EQ(test, &klvs[3], xe_guc_klv_encode_u64(klvs, 3, key, value)); + KUNIT_EXPECT_PTR_EQ(test, + xe_guc_klv_encode_u64(klvs, 3, key, value), + xe_guc_klv_encode_u64(klvs, ARRAY_SIZE(klvs), key, value)); + + KUNIT_ASSERT_PTR_EQ(test, fail, xe_guc_klv_encode_u64(fail, ARRAY_SIZE(klvs), key, value)); +} + +static u32 str_klv_size(const char *string) +{ + return GUC_KLV_LEN_MIN + to_num_dwords(strlen(string) + 1); +} + +static void test_encode_string(struct kunit *test) +{ + size_t longest_str = to_num_bytes(FIELD_MAX(GUC_KLV_0_LEN)) - 1; + u32 avail = GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN) + 1; + const char *string = "abcdefghijklmnopqrstvwxyz"; + u16 key = TEST_KEY; + u32 *klvs; + u32 *next; + char *buf; + u32 n; + + klvs = kunit_kcalloc(test, avail, sizeof(u32), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, klvs); + + buf = kunit_kzalloc(test, longest_str + 2, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf); + + /* empty string, no space, must fail */ + for (n = 0; n < str_klv_size(""); n++) { + klvs[0] = TEST_PAD; + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), + xe_guc_klv_encode_string(klvs, n, key, "")); + KUNIT_EXPECT_EQ(test, klvs[0], TEST_PAD); + } + + /* empty string, must pass */ + KUNIT_EXPECT_PTR_EQ(test, klvs + str_klv_size(""), + xe_guc_klv_encode_string(klvs, str_klv_size(""), key, "")); + KUNIT_EXPECT_PTR_EQ(test, klvs + str_klv_size(""), + xe_guc_klv_encode_string(klvs, avail, key, "")); + + /* demo string, no space, must fail */ + for (n = 0; n < str_klv_size(string); n++) { + klvs[0] = TEST_PAD; + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), + xe_guc_klv_encode_string(klvs, n, key, string)); + KUNIT_EXPECT_EQ(test, klvs[0], TEST_PAD); + } + + /* different string len, must pass */ + for (n = 0; n <= strlen(string); n++) { + strscpy(buf, string, n + 1); + kunit_info(test, "%u: '%s'\n", n, buf); + KUNIT_ASSERT_EQ(test, n, strlen(buf)); + memset32(klvs, TEST_PAD, avail); + + next = xe_guc_klv_encode_string(klvs, str_klv_size(buf), key, buf); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, next); + KUNIT_EXPECT_PTR_EQ(test, next, klvs + str_klv_size(buf)); + KUNIT_EXPECT_STREQ_MSG(test, buf, (char *)(klvs + GUC_KLV_LEN_MIN), "n=%u", n); + kunit_info(test, "%u: %*ph\n", n, (int)to_num_bytes(next - klvs), klvs); + KUNIT_EXPECT_NE(test, *(next - 1), TEST_PAD); + KUNIT_ASSERT_EQ(test, *next, TEST_PAD); + + /* bigger buf doesn't matter */ + KUNIT_EXPECT_PTR_EQ(test, + xe_guc_klv_encode_string(klvs, str_klv_size(buf), key, buf), + xe_guc_klv_encode_string(klvs, avail, key, buf)); + } + + /* don't crash if already failed */ + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-EROFS), + xe_guc_klv_encode_string(ERR_PTR(-EROFS), avail, key, "")); + + /* too long string, must fail */ + memset(buf, 'X', longest_str + 1); + buf[longest_str + 1] = '\0'; + KUNIT_EXPECT_LT(test, longest_str, strlen(buf)); + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-E2BIG), + xe_guc_klv_encode_string(klvs, avail, key, buf)); + + /* longest string, should pass */ + buf[longest_str] = '\0'; + KUNIT_EXPECT_EQ(test, longest_str, strlen(buf)); + KUNIT_EXPECT_PTR_EQ(test, klvs + str_klv_size(buf), + xe_guc_klv_encode_string(klvs, avail, key, buf)); +} + +struct some_object { + u32 value1; + u64 value2; +} __packed; + +static u32 *obj_raw_encoder(u32 *klvs, u32 avail, const void *arg) +{ + const struct some_object *obj = arg; + size_t sz = sizeof(*obj); + u32 dwords = to_num_dwords(sz); + + if (IS_ERR(klvs)) + return klvs; + if (dwords > avail) + return ERR_PTR(-ENOSPC); + memcpy(klvs, obj, sz); + return klvs + dwords; +} + +static u32 *obj_klv_encoder(u32 *klvs, u32 avail, const void *arg) +{ + const struct some_object *obj = arg; + u32 *end = klvs + avail; + + klvs = xe_guc_klv_encode_u32(klvs, end - klvs, TEST_KEY + 1, obj->value1); + klvs = xe_guc_klv_encode_u64(klvs, end - klvs, TEST_KEY + 2, obj->value2); + return klvs; +} + +static u32 *obj_nested_encoder(u32 *klvs, u32 avail, const void *arg) +{ + u32 *end = klvs + avail; + + klvs = xe_guc_klv_encode_object(klvs, end - klvs, TEST_GROUP_KEY + 1, + arg, obj_klv_encoder); + klvs = xe_guc_klv_encode_object(klvs, end - klvs, TEST_GROUP_KEY + 2, + arg, obj_klv_encoder); + return klvs; +} + +static void test_encode_object_raw(struct kunit *test) +{ + const struct some_object obj = { + .value1 = 0xdead1234, + .value2 = 0xdead87654321dead, + }; + u32 payload = to_num_dwords(sizeof(obj)); + u32 *err = ERR_PTR(-ENOSPC); + u16 key = TEST_KEY; + u32 klvs[16]; + u32 n; + + /* too small, must fail */ + for (n = 0; n < GUC_KLV_LEN_MIN + payload; n++) { + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + KUNIT_EXPECT_PTR_EQ_MSG(test, ERR_PTR(-ENOSPC), + xe_guc_klv_encode_object(klvs, n, key, &obj, + obj_raw_encoder), + "buf size=%u dwords", n); + } + + /* must pass */ + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, + xe_guc_klv_encode_object(klvs, GUC_KLV_LEN_MIN + payload, + key, &obj, obj_raw_encoder)); + KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, payload)); + KUNIT_EXPECT_MEMEQ(test, &klvs[1], &obj, sizeof(obj)); + + /* already failed, must fail */ + KUNIT_ASSERT_PTR_EQ(test, err, + xe_guc_klv_encode_object(err, ARRAY_SIZE(klvs), key, + &obj, obj_raw_encoder)); +} + +static void test_encode_object_klv(struct kunit *test) +{ + const struct some_object obj = { + .value1 = 0xdead1234, + .value2 = 0xdead87654321dead, + }; + u16 key = TEST_GROUP_KEY; + u32 payload = 0; + u32 klvs[16]; + u32 n; + + payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value1)); + payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value2)); + + /* too small, must fail */ + for (n = 0; n < GUC_KLV_LEN_MIN + payload; n++) { + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + KUNIT_EXPECT_PTR_EQ_MSG(test, ERR_PTR(-ENOSPC), + xe_guc_klv_encode_object(klvs, n, key, &obj, + obj_klv_encoder), + "buf size=%u dwords", n); + } + + /* must pass */ + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, + xe_guc_klv_encode_object(klvs, GUC_KLV_LEN_MIN + payload, + key, &obj, obj_klv_encoder)); + KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, payload)); + KUNIT_EXPECT_EQ(test, klvs[1], PREP_GUC_KLV(TEST_KEY + 1, 1)); + KUNIT_EXPECT_EQ(test, klvs[2], obj.value1); + KUNIT_EXPECT_EQ(test, klvs[3], PREP_GUC_KLV(TEST_KEY + 2, 2)); + KUNIT_EXPECT_EQ(test, klvs[4], lower_32_bits(obj.value2)); + KUNIT_EXPECT_EQ(test, klvs[5], upper_32_bits(obj.value2)); + KUNIT_EXPECT_EQ(test, klvs[6], TEST_PAD); +} + +static void test_encode_object_nested(struct kunit *test) +{ + const struct some_object obj = { + .value1 = 0xdead1234, + .value2 = 0xdead87654321dead, + }; + u16 key = TEST_GROUP_KEY; + u32 payload = 0; + u32 klvs[16]; + u32 n; + + payload += GUC_KLV_LEN_MIN; + payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value1)); + payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value2)); + payload *= 2; + + /* too small, must fail */ + for (n = 0; n < GUC_KLV_LEN_MIN + payload; n++) { + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + KUNIT_EXPECT_PTR_EQ_MSG(test, ERR_PTR(-ENOSPC), + xe_guc_klv_encode_object(klvs, n, key, &obj, + obj_nested_encoder), + "buf size=%u dwords", n); + } + + /* must pass */ + memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, + xe_guc_klv_encode_object(klvs, GUC_KLV_LEN_MIN + payload, + key, &obj, obj_nested_encoder)); + KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, payload)); + KUNIT_EXPECT_EQ(test, klvs[1], PREP_GUC_KLV(TEST_GROUP_KEY + 1, 5)); + KUNIT_EXPECT_EQ(test, klvs[2], PREP_GUC_KLV(TEST_KEY + 1, 1)); + KUNIT_EXPECT_EQ(test, klvs[3], obj.value1); + KUNIT_EXPECT_EQ(test, klvs[4], PREP_GUC_KLV(TEST_KEY + 2, 2)); + KUNIT_EXPECT_EQ(test, klvs[5], lower_32_bits(obj.value2)); + KUNIT_EXPECT_EQ(test, klvs[6], upper_32_bits(obj.value2)); + KUNIT_EXPECT_EQ(test, klvs[7], PREP_GUC_KLV(TEST_GROUP_KEY + 2, 5)); + KUNIT_EXPECT_EQ(test, klvs[8], PREP_GUC_KLV(TEST_KEY + 1, 1)); + KUNIT_EXPECT_EQ(test, klvs[9], obj.value1); + KUNIT_EXPECT_EQ(test, klvs[10], PREP_GUC_KLV(TEST_KEY + 2, 2)); + KUNIT_EXPECT_EQ(test, klvs[11], lower_32_bits(obj.value2)); + KUNIT_EXPECT_EQ(test, klvs[12], upper_32_bits(obj.value2)); + KUNIT_EXPECT_EQ(test, klvs[13], TEST_PAD); +} + +static u32 *obj_echo_encoder(u32 *klvs, u32 avail, const void *arg) +{ + return ERR_CAST(arg); +} + +static void test_encode_object_basic(struct kunit *test) +{ + u32 longest = GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN); + u32 avail = GUC_KLV_LEN_MIN + longest; + u16 key = TEST_GROUP_KEY; + u32 *klvs; + + klvs = kunit_kcalloc(test, avail, sizeof(u32), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, klvs); + + /* smallest */ + KUNIT_EXPECT_PTR_EQ(test, klvs + GUC_KLV_LEN_MIN, + xe_guc_klv_encode_object(klvs, avail, key, + klvs + GUC_KLV_LEN_MIN, + obj_echo_encoder)); + /* largest */ + KUNIT_EXPECT_PTR_EQ(test, klvs + longest, + xe_guc_klv_encode_object(klvs, avail, key, + klvs + longest, + obj_echo_encoder)); + /* already failed */ + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-EROFS), + xe_guc_klv_encode_object(ERR_PTR(-EROFS), avail, key, + klvs + GUC_KLV_LEN_MIN, + obj_echo_encoder)); + /* encoding error */ + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-EUCLEAN), + xe_guc_klv_encode_object(klvs, avail, key, + ERR_PTR(-EUCLEAN), + obj_echo_encoder)); + /* no space */ + KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), + xe_guc_klv_encode_object(klvs, 0, key, + klvs + GUC_KLV_LEN_MIN, + obj_echo_encoder)); +} + +static void __drm_printfn_kunit(struct drm_printer *p, struct va_format *vaf) +{ + struct kunit *test = p->arg; + + kunit_info(test, "%pV", vaf); +} + +static struct drm_printer drm_kunit_printer(void) +{ + struct drm_printer p = { + .printfn = __drm_printfn_kunit, + .arg = kunit_get_current_test(), + }; + return p; +} + +static void test_print(struct kunit *test) +{ + struct drm_printer p = drm_kunit_printer(); + u32 zeros[] = { 0, 0, 0, /* padding */ }; + u32 klvs[] = { + PREP_GUC_KLV(GUC_KLV_OPT_IN_FEATURE_EXT_CAT_ERR_TYPE_KEY, 0), + PREP_GUC_KLV(GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY, 1), 1234, + PREP_GUC_KLV(GUC_KLV_VF_CFG_GGTT_SIZE_KEY, 2), 0x4000, 0x0123, + PREP_GUC_KLV(TEST_KEY, 3), 1, 2, 3, + PREP_GUC_KLV(TEST_GROUP_KEY, 5), + PREP_GUC_KLV(TEST_KEY + 1, 1), 1, + PREP_GUC_KLV(TEST_KEY + 2, 2), 1, 2, + }; + + kunit_activate_static_stub(test, is_group_key, fake_is_group_key); + xe_guc_klv_print(zeros, ARRAY_SIZE(zeros), &p); + xe_guc_klv_print(klvs, ARRAY_SIZE(klvs), &p); +} + +static struct kunit_case guc_klv_helpers_test_cases[] = { + KUNIT_CASE(test_count), + KUNIT_CASE(test_encode_u32), + KUNIT_CASE(test_encode_u64), + KUNIT_CASE(test_encode_string), + KUNIT_CASE(test_encode_object_raw), + KUNIT_CASE(test_encode_object_klv), + KUNIT_CASE(test_encode_object_nested), + KUNIT_CASE(test_encode_object_basic), + KUNIT_CASE(test_print), + {} +}; + +static struct kunit_suite guc_klv_helpers_suite = { + .name = "guc_klv_helpers", + .test_cases = guc_klv_helpers_test_cases, +}; + +kunit_test_suite(guc_klv_helpers_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_guc_relay_test.c b/drivers/gpu/drm/xe/tests/xe_guc_relay_test.c new file mode 100644 index 000000000..13701451b --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_guc_relay_test.c @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include "xe_device.h" +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" + +#define TEST_RID 1234 +#define TEST_VFID 5 +#define TEST_LEN 6 +#define TEST_ACTION 0xa +#define TEST_DATA(n) (0xd0 + (n)) + +static int replacement_relay_get_totalvfs(struct xe_guc_relay *relay) +{ + return TEST_VFID; +} + +static int relay_test_init(struct kunit *test) +{ + struct xe_pci_fake_data fake = { + .sriov_mode = XE_SRIOV_MODE_PF, + .platform = XE_TIGERLAKE, /* some random platform */ + .subplatform = XE_SUBPLATFORM_NONE, + }; + struct xe_guc_relay *relay; + struct xe_device *xe; + + test->priv = &fake; + xe_kunit_helper_xe_device_test_init(test); + + xe = test->priv; + KUNIT_ASSERT_EQ(test, xe_sriov_init(xe), 0); + + relay = &xe_device_get_gt(xe, 0)->uc.guc.relay; + kunit_activate_static_stub(test, relay_get_totalvfs, + replacement_relay_get_totalvfs); + + KUNIT_ASSERT_EQ(test, xe_guc_relay_init(relay), 0); + KUNIT_EXPECT_TRUE(test, relay_is_ready(relay)); + relay->last_rid = TEST_RID - 1; + + test->priv = relay; + return 0; +} + +static const u32 TEST_MSG[TEST_LEN] = { + FIELD_PREP_CONST(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP_CONST(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) | + FIELD_PREP_CONST(GUC_HXG_EVENT_MSG_0_ACTION, TEST_ACTION) | + FIELD_PREP_CONST(GUC_HXG_EVENT_MSG_0_DATA0, TEST_DATA(0)), + TEST_DATA(1), TEST_DATA(2), TEST_DATA(3), TEST_DATA(4), +}; + +static int replacement_xe_guc_ct_send_recv_always_fails(struct xe_guc_ct *ct, + const u32 *msg, u32 len, + u32 *response_buffer) +{ + struct kunit *test = kunit_get_current_test(); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ct); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, msg); + KUNIT_ASSERT_GE(test, len, GUC_HXG_MSG_MIN_LEN); + + return -ECOMM; +} + +static int replacement_xe_guc_ct_send_recv_expects_pf2guc_relay(struct xe_guc_ct *ct, + const u32 *msg, u32 len, + u32 *response_buffer) +{ + struct kunit *test = kunit_get_current_test(); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ct); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, msg); + KUNIT_ASSERT_GE(test, len, PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN); + KUNIT_ASSERT_EQ(test, len, PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN + TEST_LEN); + KUNIT_EXPECT_EQ(test, GUC_HXG_ORIGIN_HOST, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0])); + KUNIT_EXPECT_EQ(test, GUC_HXG_TYPE_REQUEST, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])); + KUNIT_EXPECT_EQ(test, XE_GUC_ACTION_PF2GUC_RELAY_TO_VF, + FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0])); + KUNIT_EXPECT_EQ(test, TEST_VFID, + FIELD_GET(PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID, msg[1])); + KUNIT_EXPECT_EQ(test, TEST_RID, + FIELD_GET(PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID, msg[2])); + KUNIT_EXPECT_MEMEQ(test, TEST_MSG, msg + PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN, + sizeof(u32) * TEST_LEN); + return 0; +} + +static const u32 test_guc2pf[GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN] = { + /* transport */ + FIELD_PREP_CONST(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC) | + FIELD_PREP_CONST(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) | + FIELD_PREP_CONST(GUC_HXG_EVENT_MSG_0_ACTION, XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF), + FIELD_PREP_CONST(GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID, TEST_VFID), + FIELD_PREP_CONST(GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID, TEST_RID), + /* payload */ + FIELD_PREP_CONST(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP_CONST(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_SUCCESS), +}; + +static const u32 test_guc2vf[GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN] = { + /* transport */ + FIELD_PREP_CONST(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC) | + FIELD_PREP_CONST(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) | + FIELD_PREP_CONST(GUC_HXG_EVENT_MSG_0_ACTION, XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF), + FIELD_PREP_CONST(GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID, TEST_RID), + /* payload */ + FIELD_PREP_CONST(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP_CONST(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_SUCCESS), +}; + +static void pf_rejects_guc2pf_too_short(struct kunit *test) +{ + const u32 len = GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN - 1; + struct xe_guc_relay *relay = test->priv; + const u32 *msg = test_guc2pf; + + KUNIT_ASSERT_EQ(test, -EPROTO, xe_guc_relay_process_guc2pf(relay, msg, len)); +} + +static void pf_rejects_guc2pf_too_long(struct kunit *test) +{ + const u32 len = GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN + 1; + struct xe_guc_relay *relay = test->priv; + const u32 *msg = test_guc2pf; + + KUNIT_ASSERT_EQ(test, -EMSGSIZE, xe_guc_relay_process_guc2pf(relay, msg, len)); +} + +static void pf_rejects_guc2pf_no_payload(struct kunit *test) +{ + const u32 len = GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN; + struct xe_guc_relay *relay = test->priv; + const u32 *msg = test_guc2pf; + + KUNIT_ASSERT_EQ(test, -EPROTO, xe_guc_relay_process_guc2pf(relay, msg, len)); +} + +static void pf_fails_no_payload(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + const u32 msg = 0; + + KUNIT_ASSERT_EQ(test, -EPROTO, relay_process_msg(relay, TEST_VFID, TEST_RID, &msg, 0)); +} + +static void pf_fails_bad_origin(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + static const u32 msg[] = { + FIELD_PREP_CONST(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC) | + FIELD_PREP_CONST(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_SUCCESS), + }; + u32 len = ARRAY_SIZE(msg); + + KUNIT_ASSERT_EQ(test, -EPROTO, relay_process_msg(relay, TEST_VFID, TEST_RID, msg, len)); +} + +static void pf_fails_bad_type(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + const u32 msg[] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, 4), /* only 4 is undefined */ + }; + u32 len = ARRAY_SIZE(msg); + + KUNIT_ASSERT_EQ(test, -EBADRQC, relay_process_msg(relay, TEST_VFID, TEST_RID, msg, len)); +} + +static void pf_txn_reports_error(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + struct relay_transaction *txn; + + txn = __relay_get_transaction(relay, false, TEST_VFID, TEST_RID, + TEST_MSG, TEST_LEN, NULL, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, txn); + + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_always_fails); + KUNIT_EXPECT_EQ(test, -ECOMM, relay_send_transaction(relay, txn)); + + relay_release_transaction(relay, txn); +} + +static void pf_txn_sends_pf2guc(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + struct relay_transaction *txn; + + txn = __relay_get_transaction(relay, false, TEST_VFID, TEST_RID, + TEST_MSG, TEST_LEN, NULL, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, txn); + + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_expects_pf2guc_relay); + KUNIT_ASSERT_EQ(test, 0, relay_send_transaction(relay, txn)); + + relay_release_transaction(relay, txn); +} + +static void pf_sends_pf2guc(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_expects_pf2guc_relay); + KUNIT_ASSERT_EQ(test, 0, + xe_guc_relay_send_to_vf(relay, TEST_VFID, + TEST_MSG, TEST_LEN, NULL, 0)); +} + +static int replacement_xe_guc_ct_send_recv_loopback_relay(struct xe_guc_ct *ct, + const u32 *msg, u32 len, + u32 *response_buffer) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_guc_relay *relay = test->priv; + u32 *reply = kunit_kzalloc(test, len * sizeof(u32), GFP_KERNEL); + int (*guc2relay)(struct xe_guc_relay *, const u32 *, u32); + u32 action; + int err; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ct); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, msg); + KUNIT_ASSERT_GE(test, len, GUC_HXG_MSG_MIN_LEN); + KUNIT_ASSERT_EQ(test, GUC_HXG_TYPE_REQUEST, + FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])); + KUNIT_ASSERT_GE(test, len, GUC_HXG_REQUEST_MSG_MIN_LEN); + KUNIT_ASSERT_NOT_NULL(test, reply); + + switch (FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0])) { + case XE_GUC_ACTION_PF2GUC_RELAY_TO_VF: + KUNIT_ASSERT_GE(test, len, PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN); + action = XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF; + guc2relay = xe_guc_relay_process_guc2pf; + break; + case XE_GUC_ACTION_VF2GUC_RELAY_TO_PF: + KUNIT_ASSERT_GE(test, len, VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN); + action = XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF; + guc2relay = xe_guc_relay_process_guc2vf; + break; + default: + KUNIT_FAIL(test, "bad RELAY action %#x", msg[0]); + return -EINVAL; + } + + reply[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) | + FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION, action); + memcpy(reply + 1, msg + 1, sizeof(u32) * (len - 1)); + + err = guc2relay(relay, reply, len); + KUNIT_EXPECT_EQ(test, err, 0); + + return err; +} + +static void test_requires_relay_testloop(struct kunit *test) +{ + /* + * The debug relay action GUC_RELAY_ACTION_VFXPF_TESTLOOP is available + * only on builds with CONFIG_DRM_XE_DEBUG_SRIOV enabled. + * See "kunit.py --kconfig_add" option if it's missing. + */ + if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) + kunit_skip(test, "requires %s\n", __stringify(CONFIG_DRM_XE_DEBUG_SRIOV)); +} + +static void pf_loopback_nop(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + u32 request[] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_RELAY_ACTION_VFXPF_TESTLOOP) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_DATA0, VFXPF_TESTLOOP_OPCODE_NOP), + }; + u32 response[GUC_HXG_RESPONSE_MSG_MIN_LEN]; + int ret; + + test_requires_relay_testloop(test); + + kunit_activate_static_stub(test, relay_kick_worker, relay_process_incoming_action); + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_loopback_relay); + ret = xe_guc_relay_send_to_vf(relay, TEST_VFID, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + KUNIT_ASSERT_EQ(test, ret, GUC_HXG_RESPONSE_MSG_MIN_LEN); + KUNIT_EXPECT_EQ(test, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, response[0]), + GUC_HXG_ORIGIN_HOST); + KUNIT_EXPECT_EQ(test, FIELD_GET(GUC_HXG_MSG_0_TYPE, response[0]), + GUC_HXG_TYPE_RESPONSE_SUCCESS); + KUNIT_EXPECT_EQ(test, FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, response[0]), 0); +} + +static void pf_loopback_echo(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + u32 request[] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_RELAY_ACTION_VFXPF_TESTLOOP) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_DATA0, VFXPF_TESTLOOP_OPCODE_ECHO), + TEST_DATA(1), TEST_DATA(2), TEST_DATA(3), TEST_DATA(4), + }; + u32 response[ARRAY_SIZE(request)]; + unsigned int n; + int ret; + + test_requires_relay_testloop(test); + + kunit_activate_static_stub(test, relay_kick_worker, relay_process_incoming_action); + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_loopback_relay); + ret = xe_guc_relay_send_to_vf(relay, TEST_VFID, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + KUNIT_ASSERT_EQ(test, ret, ARRAY_SIZE(response)); + KUNIT_EXPECT_EQ(test, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, response[0]), + GUC_HXG_ORIGIN_HOST); + KUNIT_EXPECT_EQ(test, FIELD_GET(GUC_HXG_MSG_0_TYPE, response[0]), + GUC_HXG_TYPE_RESPONSE_SUCCESS); + KUNIT_EXPECT_EQ(test, FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, response[0]), + ARRAY_SIZE(response)); + for (n = GUC_HXG_RESPONSE_MSG_MIN_LEN; n < ret; n++) + KUNIT_EXPECT_EQ(test, request[n], response[n]); +} + +static void pf_loopback_fail(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + u32 request[] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_RELAY_ACTION_VFXPF_TESTLOOP) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_DATA0, VFXPF_TESTLOOP_OPCODE_FAIL), + }; + u32 response[GUC_HXG_RESPONSE_MSG_MIN_LEN]; + int ret; + + test_requires_relay_testloop(test); + + kunit_activate_static_stub(test, relay_kick_worker, relay_process_incoming_action); + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_loopback_relay); + ret = xe_guc_relay_send_to_vf(relay, TEST_VFID, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + KUNIT_ASSERT_EQ(test, ret, -EREMOTEIO); +} + +static void pf_loopback_busy(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + u32 request[] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_RELAY_ACTION_VFXPF_TESTLOOP) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_DATA0, VFXPF_TESTLOOP_OPCODE_BUSY), + TEST_DATA(0xb), + }; + u32 response[GUC_HXG_RESPONSE_MSG_MIN_LEN]; + int ret; + + test_requires_relay_testloop(test); + + kunit_activate_static_stub(test, relay_testonly_nop, relay_process_incoming_action); + kunit_activate_static_stub(test, relay_kick_worker, relay_process_incoming_action); + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_loopback_relay); + ret = xe_guc_relay_send_to_vf(relay, TEST_VFID, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + KUNIT_ASSERT_EQ(test, ret, GUC_HXG_RESPONSE_MSG_MIN_LEN); +} + +static void pf_loopback_retry(struct kunit *test) +{ + struct xe_guc_relay *relay = test->priv; + u32 request[] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_RELAY_ACTION_VFXPF_TESTLOOP) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_DATA0, VFXPF_TESTLOOP_OPCODE_RETRY), + TEST_DATA(0xd), TEST_DATA(0xd), + }; + u32 response[GUC_HXG_RESPONSE_MSG_MIN_LEN]; + int ret; + + test_requires_relay_testloop(test); + + kunit_activate_static_stub(test, relay_kick_worker, relay_process_incoming_action); + kunit_activate_static_stub(test, xe_guc_ct_send_recv, + replacement_xe_guc_ct_send_recv_loopback_relay); + ret = xe_guc_relay_send_to_vf(relay, TEST_VFID, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + KUNIT_ASSERT_EQ(test, ret, GUC_HXG_RESPONSE_MSG_MIN_LEN); +} + +static struct kunit_case pf_relay_test_cases[] = { + KUNIT_CASE(pf_rejects_guc2pf_too_short), + KUNIT_CASE(pf_rejects_guc2pf_too_long), + KUNIT_CASE(pf_rejects_guc2pf_no_payload), + KUNIT_CASE(pf_fails_no_payload), + KUNIT_CASE(pf_fails_bad_origin), + KUNIT_CASE(pf_fails_bad_type), + KUNIT_CASE(pf_txn_reports_error), + KUNIT_CASE(pf_txn_sends_pf2guc), + KUNIT_CASE(pf_sends_pf2guc), + KUNIT_CASE(pf_loopback_nop), + KUNIT_CASE(pf_loopback_echo), + KUNIT_CASE(pf_loopback_fail), + KUNIT_CASE_SLOW(pf_loopback_busy), + KUNIT_CASE_SLOW(pf_loopback_retry), + {} +}; + +static struct kunit_suite pf_relay_suite = { + .name = "pf_relay", + .test_cases = pf_relay_test_cases, + .init = relay_test_init, +}; + +static void vf_rejects_guc2vf_too_short(struct kunit *test) +{ + const u32 len = GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN - 1; + struct xe_guc_relay *relay = test->priv; + const u32 *msg = test_guc2vf; + + KUNIT_ASSERT_EQ(test, -EPROTO, xe_guc_relay_process_guc2vf(relay, msg, len)); +} + +static void vf_rejects_guc2vf_too_long(struct kunit *test) +{ + const u32 len = GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN + 1; + struct xe_guc_relay *relay = test->priv; + const u32 *msg = test_guc2vf; + + KUNIT_ASSERT_EQ(test, -EMSGSIZE, xe_guc_relay_process_guc2vf(relay, msg, len)); +} + +static void vf_rejects_guc2vf_no_payload(struct kunit *test) +{ + const u32 len = GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN; + struct xe_guc_relay *relay = test->priv; + const u32 *msg = test_guc2vf; + + KUNIT_ASSERT_EQ(test, -EPROTO, xe_guc_relay_process_guc2vf(relay, msg, len)); +} + +static struct kunit_case vf_relay_test_cases[] = { + KUNIT_CASE(vf_rejects_guc2vf_too_short), + KUNIT_CASE(vf_rejects_guc2vf_too_long), + KUNIT_CASE(vf_rejects_guc2vf_no_payload), + {} +}; + +static struct kunit_suite vf_relay_suite = { + .name = "vf_relay", + .test_cases = vf_relay_test_cases, + .init = relay_test_init, +}; + +static void xe_drops_guc2pf_if_not_ready(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct xe_guc_relay *relay = &xe_device_get_gt(xe, 0)->uc.guc.relay; + const u32 *msg = test_guc2pf; + u32 len = GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN + GUC_RELAY_MSG_MIN_LEN; + + KUNIT_ASSERT_EQ(test, -ENODEV, xe_guc_relay_process_guc2pf(relay, msg, len)); +} + +static void xe_drops_guc2vf_if_not_ready(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct xe_guc_relay *relay = &xe_device_get_gt(xe, 0)->uc.guc.relay; + const u32 *msg = test_guc2vf; + u32 len = GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN + GUC_RELAY_MSG_MIN_LEN; + + KUNIT_ASSERT_EQ(test, -ENODEV, xe_guc_relay_process_guc2vf(relay, msg, len)); +} + +static void xe_rejects_send_if_not_ready(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct xe_guc_relay *relay = &xe_device_get_gt(xe, 0)->uc.guc.relay; + u32 msg[GUC_RELAY_MSG_MIN_LEN]; + u32 len = ARRAY_SIZE(msg); + + KUNIT_ASSERT_EQ(test, -ENODEV, xe_guc_relay_send_to_pf(relay, msg, len, NULL, 0)); + KUNIT_ASSERT_EQ(test, -ENODEV, relay_send_to(relay, TEST_VFID, msg, len, NULL, 0)); +} + +static struct kunit_case no_relay_test_cases[] = { + KUNIT_CASE(xe_drops_guc2pf_if_not_ready), + KUNIT_CASE(xe_drops_guc2vf_if_not_ready), + KUNIT_CASE(xe_rejects_send_if_not_ready), + {} +}; + +static struct kunit_suite no_relay_suite = { + .name = "no_relay", + .test_cases = no_relay_test_cases, + .init = xe_kunit_helper_xe_device_test_init, +}; + +kunit_test_suites(&no_relay_suite, + &pf_relay_suite, + &vf_relay_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c b/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c new file mode 100644 index 000000000..bc5156966 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include +#include + +#include "tests/xe_kunit_helpers.h" +#include "tests/xe_pci_test.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_pm.h" + +/** + * xe_kunit_helper_alloc_xe_device - Allocate a &xe_device for a KUnit test. + * @test: the &kunit where this &xe_device will be used + * @dev: The parent device object + * + * This function allocates xe_device using drm_kunit_helper_alloc_device(). + * The xe_device allocation is managed by the test. + * + * @dev should be allocated using drm_kunit_helper_alloc_device(). + * + * This function uses KUNIT_ASSERT to detect any allocation failures. + * + * Return: A pointer to the new &xe_device. + */ +struct xe_device *xe_kunit_helper_alloc_xe_device(struct kunit *test, + struct device *dev) +{ + struct xe_device *xe; + + xe = drm_kunit_helper_alloc_drm_device(test, dev, + struct xe_device, + drm, DRIVER_GEM); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe); + return xe; +} +EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_alloc_xe_device); + +static void kunit_action_restore_priv(void *priv) +{ + struct kunit *test = kunit_get_current_test(); + + test->priv = priv; +} + +/** + * xe_kunit_helper_xe_device_test_init - Prepare a &xe_device for a KUnit test. + * @test: the &kunit where this fake &xe_device will be used + * + * This function allocates and initializes a fake &xe_device and stores its + * pointer as &kunit.priv to allow the test code to access it. + * + * This function can be directly used as custom implementation of + * &kunit_suite.init. + * + * It is possible to prepare specific variant of the fake &xe_device by passing + * in &kunit.priv pointer to the struct xe_pci_fake_data supplemented with + * desired parameters prior to calling this function. + * + * This function uses KUNIT_ASSERT to detect any failures. + * + * Return: Always 0. + */ +int xe_kunit_helper_xe_device_test_init(struct kunit *test) +{ + struct xe_device *xe; + struct device *dev; + int err; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + xe = xe_kunit_helper_alloc_xe_device(test, dev); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe); + + err = xe_pci_fake_device_init(xe); + KUNIT_ASSERT_EQ(test, err, 0); + + err = kunit_add_action_or_reset(test, kunit_action_restore_priv, test->priv); + KUNIT_ASSERT_EQ(test, err, 0); + + test->priv = xe; + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_xe_device_test_init); + +KUNIT_DEFINE_ACTION_WRAPPER(put_xe_pm_runtime, xe_pm_runtime_put, struct xe_device *); + +/** + * xe_kunit_helper_xe_device_live_test_init - Prepare a &xe_device for + * use in a live KUnit test. + * @test: the &kunit where live &xe_device will be used + * + * This function expects pointer to the &xe_device in the &test.param_value, + * like it is prepared by the &xe_pci_live_device_gen_param and stores that + * pointer as &kunit.priv to allow the test code to access it. + * + * This function makes sure that device is not wedged and then resumes it + * to avoid waking up the device inside the test. It uses deferred cleanup + * action to release a runtime_pm reference. + * + * This function can be used as custom implementation of &kunit_suite.init. + * + * This function uses KUNIT_ASSERT to detect any failures. + * + * Return: Always 0. + */ +int xe_kunit_helper_xe_device_live_test_init(struct kunit *test) +{ + struct xe_device *xe = xe_device_const_cast(test->param_value); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe); + kunit_info(test, "running on %s device\n", xe->info.platform_name); + + KUNIT_ASSERT_FALSE(test, xe_device_wedged(xe)); + xe_pm_runtime_get(xe); + KUNIT_ASSERT_EQ(test, 0, kunit_add_action_or_reset(test, put_xe_pm_runtime, xe)); + + test->priv = xe; + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_xe_device_live_test_init); diff --git a/drivers/gpu/drm/xe/tests/xe_kunit_helpers.h b/drivers/gpu/drm/xe/tests/xe_kunit_helpers.h new file mode 100644 index 000000000..83665f7b1 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_kunit_helpers.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_KUNIT_HELPERS_H_ +#define _XE_KUNIT_HELPERS_H_ + +struct device; +struct kunit; +struct xe_device; + +struct xe_device *xe_kunit_helper_alloc_xe_device(struct kunit *test, + struct device *dev); +int xe_kunit_helper_xe_device_test_init(struct kunit *test); + +int xe_kunit_helper_xe_device_live_test_init(struct kunit *test); + +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_live_test_mod.c b/drivers/gpu/drm/xe/tests/xe_live_test_mod.c new file mode 100644 index 000000000..87cd7db20 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_live_test_mod.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2023 Intel Corporation + */ +#include +#include + +extern struct kunit_suite xe_bo_test_suite; +extern struct kunit_suite xe_bo_shrink_test_suite; +extern struct kunit_suite xe_dma_buf_test_suite; +extern struct kunit_suite xe_migrate_test_suite; +extern struct kunit_suite xe_mocs_test_suite; +extern struct kunit_suite xe_guc_g2g_test_suite; +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +extern struct kunit_suite xe_bo_page_size_alloc_suite; +#endif + +kunit_test_suite(xe_bo_test_suite); +kunit_test_suite(xe_bo_shrink_test_suite); +kunit_test_suite(xe_dma_buf_test_suite); +kunit_test_suite(xe_migrate_test_suite); +kunit_test_suite(xe_mocs_test_suite); +kunit_test_suite(xe_guc_g2g_test_suite); +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +kunit_test_suite(xe_bo_page_size_alloc_suite); +#endif + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("xe live kunit tests"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/gpu/drm/xe/tests/xe_lmtt_test.c b/drivers/gpu/drm/xe/tests/xe_lmtt_test.c new file mode 100644 index 000000000..1f1557c45 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_lmtt_test.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +static const struct lmtt_ops_param { + const char *desc; + const struct xe_lmtt_ops *ops; +} lmtt_ops_params[] = { + { "2-level", &lmtt_2l_ops, }, + { "multi-level", &lmtt_ml_ops, }, +}; + +static void lmtt_ops_param_get_desc(const struct lmtt_ops_param *p, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", p->desc); +} + +KUNIT_ARRAY_PARAM(lmtt_ops, lmtt_ops_params, lmtt_ops_param_get_desc); + +static void test_ops(struct kunit *test) +{ + const struct lmtt_ops_param *p = test->param_value; + const struct xe_lmtt_ops *ops = p->ops; + unsigned int n; + + KUNIT_ASSERT_NOT_NULL(test, ops->lmtt_root_pd_level); + KUNIT_ASSERT_NOT_NULL(test, ops->lmtt_pte_num); + KUNIT_ASSERT_NOT_NULL(test, ops->lmtt_pte_size); + KUNIT_ASSERT_NOT_NULL(test, ops->lmtt_pte_shift); + KUNIT_ASSERT_NOT_NULL(test, ops->lmtt_pte_index); + KUNIT_ASSERT_NOT_NULL(test, ops->lmtt_pte_encode); + + KUNIT_EXPECT_NE(test, ops->lmtt_root_pd_level(), 0); + + for (n = 0; n <= ops->lmtt_root_pd_level(); n++) { + KUNIT_EXPECT_NE_MSG(test, ops->lmtt_pte_num(n), 0, + "level=%u", n); + KUNIT_EXPECT_NE_MSG(test, ops->lmtt_pte_size(n), 0, + "level=%u", n); + KUNIT_EXPECT_NE_MSG(test, ops->lmtt_pte_encode(0, n), LMTT_PTE_INVALID, + "level=%u", n); + } + + for (n = 0; n < ops->lmtt_root_pd_level(); n++) { + u64 addr = BIT_ULL(ops->lmtt_pte_shift(n)); + + KUNIT_EXPECT_NE_MSG(test, ops->lmtt_pte_shift(n), 0, + "level=%u", n); + KUNIT_EXPECT_EQ_MSG(test, ops->lmtt_pte_index(addr - 1, n), 0, + "addr=%#llx level=%u", addr, n); + KUNIT_EXPECT_EQ_MSG(test, ops->lmtt_pte_index(addr + 1, n), 1, + "addr=%#llx level=%u", addr, n); + KUNIT_EXPECT_EQ_MSG(test, ops->lmtt_pte_index(addr * 2 - 1, n), 1, + "addr=%#llx level=%u", addr, n); + KUNIT_EXPECT_EQ_MSG(test, ops->lmtt_pte_index(addr * 2, n), 2, + "addr=%#llx level=%u", addr, n); + } +} + +static struct kunit_case lmtt_test_cases[] = { + KUNIT_CASE_PARAM(test_ops, lmtt_ops_gen_params), + {} +}; + +static struct kunit_suite lmtt_suite = { + .name = "lmtt", + .test_cases = lmtt_test_cases, +}; + +kunit_test_suites(&lmtt_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c new file mode 100644 index 000000000..f10d95137 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -0,0 +1,785 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020-2022 Intel Corporation + */ + +#include +#include + +#include "tests/xe_kunit_helpers.h" +#include "tests/xe_pci_test.h" + +#include "xe_pat.h" +#include "xe_pci.h" +#include "xe_pm.h" + +static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence, + const char *str, struct kunit *test) +{ + long ret; + + if (IS_ERR(fence)) { + KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str, + PTR_ERR(fence)); + return true; + } + if (!fence) + return true; + + ret = dma_fence_wait_timeout(fence, false, 5 * HZ); + if (ret <= 0) { + KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret); + return true; + } + + return false; +} + +static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe, + struct xe_bb *bb, u32 second_idx, const char *str, + struct kunit *test) +{ + u64 batch_base = xe_migrate_batch_base(m, xe->info.has_usm); + struct xe_sched_job *job = xe_bb_create_migration_job(m->q, bb, + batch_base, + second_idx); + struct dma_fence *fence; + + if (IS_ERR(job)) { + KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", + PTR_ERR(job)); + return PTR_ERR(job); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + if (sanity_fence_failed(xe, fence, str, test)) + return -ETIMEDOUT; + + dma_fence_put(fence); + kunit_info(test, "%s: Job completed\n", str); + return 0; +} + +#define check(_retval, _expected, str, _test) \ + do { if ((_retval) != (_expected)) { \ + KUNIT_FAIL(_test, "Sanity check failed: " str \ + " expected %llx, got %llx\n", \ + (u64)(_expected), (u64)(_retval)); \ + } } while (0) + +static void test_copy(struct xe_migrate *m, struct xe_bo *bo, + struct kunit *test, u32 region, struct drm_exec *exec) +{ + struct xe_device *xe = tile_to_xe(m->tile); + u64 retval, expected = 0; + bool big = xe_bo_size(bo) >= SZ_2M; + struct dma_fence *fence; + const char *str = big ? "Copying big bo" : "Copying small bo"; + int err; + + struct xe_bo *remote = xe_bo_create_locked(xe, m->tile, NULL, + xe_bo_size(bo), + ttm_bo_type_kernel, + region | + XE_BO_FLAG_NEEDS_CPU_ACCESS | + XE_BO_FLAG_PINNED, + exec); + if (IS_ERR(remote)) { + KUNIT_FAIL(test, "Failed to allocate remote bo for %s: %pe\n", + str, remote); + return; + } + + err = xe_bo_validate(remote, NULL, false, exec); + if (err) { + KUNIT_FAIL(test, "Failed to validate system bo for %s: %i\n", + str, err); + goto out_unlock; + } + + err = xe_bo_vmap(remote); + if (err) { + KUNIT_FAIL(test, "Failed to vmap system bo for %s: %i\n", + str, err); + goto out_unlock; + } + + xe_map_memset(xe, &remote->vmap, 0, 0xd0, xe_bo_size(remote)); + fence = xe_migrate_clear(m, remote, remote->ttm.resource, + XE_MIGRATE_CLEAR_FLAG_FULL); + if (!sanity_fence_failed(xe, fence, big ? "Clearing remote big bo" : + "Clearing remote small bo", test)) { + retval = xe_map_rd(xe, &remote->vmap, 0, u64); + check(retval, expected, "remote first offset should be cleared", + test); + retval = xe_map_rd(xe, &remote->vmap, xe_bo_size(remote) - 8, u64); + check(retval, expected, "remote last offset should be cleared", + test); + } + dma_fence_put(fence); + + /* Try to copy 0xc0 from remote to vram with 2MB or 64KiB/4KiB pages */ + xe_map_memset(xe, &remote->vmap, 0, 0xc0, xe_bo_size(remote)); + xe_map_memset(xe, &bo->vmap, 0, 0xd0, xe_bo_size(bo)); + + expected = 0xc0c0c0c0c0c0c0c0; + fence = xe_migrate_copy(m, remote, bo, remote->ttm.resource, + bo->ttm.resource, false); + if (!sanity_fence_failed(xe, fence, big ? "Copying big bo remote -> vram" : + "Copying small bo remote -> vram", test)) { + retval = xe_map_rd(xe, &bo->vmap, 0, u64); + check(retval, expected, + "remote -> vram bo first offset should be copied", test); + retval = xe_map_rd(xe, &bo->vmap, xe_bo_size(bo) - 8, u64); + check(retval, expected, + "remote -> vram bo offset should be copied", test); + } + dma_fence_put(fence); + + /* And other way around.. slightly hacky.. */ + xe_map_memset(xe, &remote->vmap, 0, 0xd0, xe_bo_size(remote)); + xe_map_memset(xe, &bo->vmap, 0, 0xc0, xe_bo_size(bo)); + + fence = xe_migrate_copy(m, bo, remote, bo->ttm.resource, + remote->ttm.resource, false); + if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> remote" : + "Copying small bo vram -> remote", test)) { + retval = xe_map_rd(xe, &remote->vmap, 0, u64); + check(retval, expected, + "vram -> remote bo first offset should be copied", test); + retval = xe_map_rd(xe, &remote->vmap, xe_bo_size(bo) - 8, u64); + check(retval, expected, + "vram -> remote bo last offset should be copied", test); + } + dma_fence_put(fence); + + xe_bo_vunmap(remote); +out_unlock: + xe_bo_unlock(remote); + xe_bo_put(remote); +} + +static void test_copy_sysmem(struct xe_migrate *m, struct xe_bo *bo, + struct drm_exec *exec, struct kunit *test) +{ + test_copy(m, bo, test, XE_BO_FLAG_SYSTEM, exec); +} + +static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo, + struct drm_exec *exec, struct kunit *test) +{ + u32 region; + + if (bo->ttm.resource->mem_type == XE_PL_SYSTEM) + return; + + if (bo->ttm.resource->mem_type == XE_PL_VRAM0) + region = XE_BO_FLAG_VRAM1; + else + region = XE_BO_FLAG_VRAM0; + test_copy(m, bo, test, region, exec); +} + +static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test, + struct drm_exec *exec) +{ + struct xe_tile *tile = m->tile; + struct xe_device *xe = tile_to_xe(tile); + struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny; + struct xe_res_cursor src_it; + struct dma_fence *fence; + u64 retval, expected; + struct xe_bb *bb; + int err; + u8 id = tile->id; + + err = xe_bo_vmap(bo); + if (err) { + KUNIT_FAIL(test, "Failed to vmap our pagetables: %d\n", err); + return; + } + + big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile), + exec); + if (IS_ERR(big)) { + KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big)); + goto vunmap; + } + + pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile), + exec); + if (IS_ERR(pt)) { + KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", + PTR_ERR(pt)); + goto free_big; + } + + tiny = xe_bo_create_pin_map(xe, tile, m->q->vm, + 2 * SZ_4K, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile), + exec); + if (IS_ERR(tiny)) { + KUNIT_FAIL(test, "Failed to allocate tiny fake pt: %li\n", + PTR_ERR(tiny)); + goto free_pt; + } + + bb = xe_bb_new(tile->primary_gt, 32, xe->info.has_usm); + if (IS_ERR(bb)) { + KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n", + PTR_ERR(bb)); + goto free_tiny; + } + + kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n", + (unsigned long)xe_bo_main_addr(m->q->vm->pt_root[id]->bo, XE_PAGE_SIZE), + (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE)); + + /* First part of the test, are we updating our pagetable bo with a new entry? */ + xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, + 0xdeaddeadbeefbeef); + expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe_cache_pat_idx(xe, XE_CACHE_WB), 0); + if (m->q->vm->flags & XE_VM_FLAG_64K) + expected |= XE_PTE_PS64; + if (xe_bo_is_vram(pt)) + xe_res_first(pt->ttm.resource, 0, xe_bo_size(pt), &src_it); + else + xe_res_first_sg(xe_bo_sg(pt), 0, xe_bo_size(pt), &src_it); + + emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), false, + &src_it, XE_PAGE_SIZE, pt->ttm.resource); + + run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test); + + retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), + u64); + check(retval, expected, "PTE entry write", test); + + /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */ + bb->len = 0; + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead); + expected = 0; + + emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4, + IS_DGFX(xe)); + run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable", + test); + + retval = xe_map_rd(xe, &pt->vmap, 0, u32); + check(retval, expected, "Write to PT after adding PTE", test); + + /* Sanity checks passed, try the full ones! */ + + /* Clear a small bo */ + kunit_info(test, "Clearing small buffer object\n"); + xe_map_memset(xe, &tiny->vmap, 0, 0x22, xe_bo_size(tiny)); + expected = 0; + fence = xe_migrate_clear(m, tiny, tiny->ttm.resource, + XE_MIGRATE_CLEAR_FLAG_FULL); + if (sanity_fence_failed(xe, fence, "Clearing small bo", test)) + goto out; + + dma_fence_put(fence); + retval = xe_map_rd(xe, &tiny->vmap, 0, u32); + check(retval, expected, "Command clear small first value", test); + retval = xe_map_rd(xe, &tiny->vmap, xe_bo_size(tiny) - 4, u32); + check(retval, expected, "Command clear small last value", test); + + kunit_info(test, "Copying small buffer object to system\n"); + test_copy_sysmem(m, tiny, exec, test); + if (xe->info.tile_count > 1) { + kunit_info(test, "Copying small buffer object to other vram\n"); + test_copy_vram(m, tiny, exec, test); + } + + /* Clear a big bo */ + kunit_info(test, "Clearing big buffer object\n"); + xe_map_memset(xe, &big->vmap, 0, 0x11, xe_bo_size(big)); + expected = 0; + fence = xe_migrate_clear(m, big, big->ttm.resource, + XE_MIGRATE_CLEAR_FLAG_FULL); + if (sanity_fence_failed(xe, fence, "Clearing big bo", test)) + goto out; + + dma_fence_put(fence); + retval = xe_map_rd(xe, &big->vmap, 0, u32); + check(retval, expected, "Command clear big first value", test); + retval = xe_map_rd(xe, &big->vmap, xe_bo_size(big) - 4, u32); + check(retval, expected, "Command clear big last value", test); + + kunit_info(test, "Copying big buffer object to system\n"); + test_copy_sysmem(m, big, exec, test); + if (xe->info.tile_count > 1) { + kunit_info(test, "Copying big buffer object to other vram\n"); + test_copy_vram(m, big, exec, test); + } + +out: + xe_bb_free(bb, NULL); +free_tiny: + xe_bo_unpin(tiny); + xe_bo_put(tiny); +free_pt: + xe_bo_unpin(pt); + xe_bo_put(pt); +free_big: + xe_bo_unpin(big); + xe_bo_put(big); +vunmap: + xe_bo_vunmap(m->pt_bo); +} + +static int migrate_test_run_device(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_tile *tile; + int id; + + guard(xe_pm_runtime)(xe); + for_each_tile(tile, xe, id) { + struct xe_migrate *m = tile->migrate; + struct drm_exec *exec = XE_VALIDATION_OPT_OUT; + + kunit_info(test, "Testing tile id %d.\n", id); + xe_vm_lock(m->q->vm, false); + xe_migrate_sanity_test(m, test, exec); + xe_vm_unlock(m->q->vm); + } + + return 0; +} + +static void xe_migrate_sanity_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + migrate_test_run_device(xe); +} + +static struct dma_fence *blt_copy(struct xe_tile *tile, + struct xe_bo *src_bo, struct xe_bo *dst_bo, + bool copy_only_ccs, const char *str, struct kunit *test) +{ + struct xe_gt *gt = tile->primary_gt; + struct xe_migrate *m = tile->migrate; + struct xe_device *xe = gt_to_xe(gt); + struct dma_fence *fence = NULL; + u64 size = xe_bo_size(src_bo); + struct xe_res_cursor src_it, dst_it; + struct ttm_resource *src = src_bo->ttm.resource, *dst = dst_bo->ttm.resource; + u64 src_L0_ofs, dst_L0_ofs; + u32 src_L0_pt, dst_L0_pt; + u64 src_L0, dst_L0; + int err; + bool src_is_vram = mem_type_is_vram(src->mem_type); + bool dst_is_vram = mem_type_is_vram(dst->mem_type); + + if (!src_is_vram) + xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); + else + xe_res_first(src, 0, size, &src_it); + + if (!dst_is_vram) + xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); + else + xe_res_first(dst, 0, size, &dst_it); + + while (size) { + u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ + struct xe_sched_job *job; + struct xe_bb *bb; + u32 flush_flags = 0; + u32 update_idx; + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + u32 pte_flags; + + src_L0 = xe_migrate_res_sizes(m, &src_it); + dst_L0 = xe_migrate_res_sizes(m, &dst_it); + + src_L0 = min(src_L0, dst_L0); + + pte_flags = src_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM | + PTE_UPDATE_FLAG_IS_COMP_PTE) : 0; + batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + avail_pts); + + pte_flags = dst_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM | + PTE_UPDATE_FLAG_IS_COMP_PTE) : 0; + batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0, + &dst_L0_ofs, &dst_L0_pt, 0, + avail_pts, avail_pts); + + /* Add copy commands size here */ + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + + ((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0); + + bb = xe_bb_new(gt, batch_size, xe->info.has_usm); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + goto err_sync; + } + + if (src_is_vram) + xe_res_next(&src_it, src_L0); + else + emit_pte(m, bb, src_L0_pt, src_is_vram, false, + &src_it, src_L0, src); + + if (dst_is_vram) + xe_res_next(&dst_it, src_L0); + else + emit_pte(m, bb, dst_L0_pt, dst_is_vram, false, + &dst_it, src_L0, dst); + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + if (!copy_only_ccs) + emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); + + if (copy_only_ccs) + flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, + src_is_vram, dst_L0_ofs, + dst_is_vram, src_L0, dst_L0_ofs, + copy_only_ccs); + + job = xe_bb_create_migration_job(m->q, bb, + xe_migrate_batch_base(m, xe->info.has_usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err; + } + + xe_sched_job_add_migrate_flush(job, flush_flags); + + mutex_lock(&m->job_mutex); + xe_sched_job_arm(job); + dma_fence_put(fence); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + size -= src_L0; + continue; + +err: + xe_bb_free(bb, NULL); + +err_sync: + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + return ERR_PTR(err); + } + + return fence; +} + +static void test_migrate(struct xe_device *xe, struct xe_tile *tile, + struct xe_bo *sys_bo, struct xe_bo *vram_bo, struct xe_bo *ccs_bo, + struct drm_exec *exec, struct kunit *test) +{ + struct dma_fence *fence; + u64 expected, retval; + long timeout; + long ret; + + expected = 0xd0d0d0d0d0d0d0d0; + xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, xe_bo_size(sys_bo)); + + fence = blt_copy(tile, sys_bo, vram_bo, false, "Blit copy from sysmem to vram", test); + if (!sanity_fence_failed(xe, fence, "Blit copy from sysmem to vram", test)) { + retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64); + if (retval == expected) + KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n"); + } + dma_fence_put(fence); + + kunit_info(test, "Evict vram buffer object\n"); + ret = xe_bo_evict(vram_bo, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to evict bo.\n"); + return; + } + + ret = xe_bo_vmap(vram_bo); + if (ret) { + KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret); + return; + } + + retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64); + check(retval, expected, "Clear evicted vram data first value", test); + retval = xe_map_rd(xe, &vram_bo->vmap, xe_bo_size(vram_bo) - 8, u64); + check(retval, expected, "Clear evicted vram data last value", test); + + fence = blt_copy(tile, vram_bo, ccs_bo, + true, "Blit surf copy from vram to sysmem", test); + if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) { + retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64); + check(retval, 0, "Clear ccs data first value", test); + + retval = xe_map_rd(xe, &ccs_bo->vmap, xe_bo_size(ccs_bo) - 8, u64); + check(retval, 0, "Clear ccs data last value", test); + } + dma_fence_put(fence); + + kunit_info(test, "Restore vram buffer object\n"); + ret = xe_bo_validate(vram_bo, NULL, false, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret); + return; + } + + /* Sync all migration blits */ + timeout = dma_resv_wait_timeout(vram_bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL, + true, + 5 * HZ); + if (timeout <= 0) { + KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); + return; + } + + ret = xe_bo_vmap(vram_bo); + if (ret) { + KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret); + return; + } + + retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64); + check(retval, expected, "Restored value must be equal to initial value", test); + retval = xe_map_rd(xe, &vram_bo->vmap, xe_bo_size(vram_bo) - 8, u64); + check(retval, expected, "Restored value must be equal to initial value", test); + + fence = blt_copy(tile, vram_bo, ccs_bo, + true, "Blit surf copy from vram to sysmem", test); + if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) { + retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64); + check(retval, 0, "Clear ccs data first value", test); + retval = xe_map_rd(xe, &ccs_bo->vmap, xe_bo_size(ccs_bo) - 8, u64); + check(retval, 0, "Clear ccs data last value", test); + } + dma_fence_put(fence); +} + +static void test_clear(struct xe_device *xe, struct xe_tile *tile, + struct xe_bo *sys_bo, struct xe_bo *vram_bo, struct kunit *test) +{ + struct dma_fence *fence; + u64 expected, retval; + + expected = 0xd0d0d0d0d0d0d0d0; + xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, xe_bo_size(sys_bo)); + + fence = blt_copy(tile, sys_bo, vram_bo, false, "Blit copy from sysmem to vram", test); + if (!sanity_fence_failed(xe, fence, "Blit copy from sysmem to vram", test)) { + retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64); + if (retval == expected) + KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n"); + } + dma_fence_put(fence); + + fence = blt_copy(tile, vram_bo, sys_bo, false, "Blit copy from vram to sysmem", test); + if (!sanity_fence_failed(xe, fence, "Blit copy from vram to sysmem", test)) { + retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64); + check(retval, expected, "Decompressed value must be equal to initial value", test); + retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64); + check(retval, expected, "Decompressed value must be equal to initial value", test); + } + dma_fence_put(fence); + + kunit_info(test, "Clear vram buffer object\n"); + expected = 0x0000000000000000; + fence = xe_migrate_clear(tile->migrate, vram_bo, vram_bo->ttm.resource, + XE_MIGRATE_CLEAR_FLAG_FULL); + if (sanity_fence_failed(xe, fence, "Clear vram_bo", test)) + return; + dma_fence_put(fence); + + fence = blt_copy(tile, vram_bo, sys_bo, + false, "Blit copy from vram to sysmem", test); + if (!sanity_fence_failed(xe, fence, "Clear main buffer data", test)) { + retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64); + check(retval, expected, "Clear main buffer first value", test); + retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64); + check(retval, expected, "Clear main buffer last value", test); + } + dma_fence_put(fence); + + fence = blt_copy(tile, vram_bo, sys_bo, + true, "Blit surf copy from vram to sysmem", test); + if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) { + retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64); + check(retval, expected, "Clear ccs data first value", test); + retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64); + check(retval, expected, "Clear ccs data last value", test); + } + dma_fence_put(fence); +} + +static void validate_ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile, + struct kunit *test) +{ + struct xe_bo *sys_bo, *vram_bo = NULL, *ccs_bo = NULL; + unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); + struct drm_exec *exec; + long ret; + + sys_bo = xe_bo_create_user(xe, NULL, SZ_4M, + DRM_XE_GEM_CPU_CACHING_WC, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_NEEDS_CPU_ACCESS | + XE_BO_FLAG_PINNED, NULL); + + if (IS_ERR(sys_bo)) { + KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", + PTR_ERR(sys_bo)); + return; + } + + exec = XE_VALIDATION_OPT_OUT; + xe_bo_lock(sys_bo, false); + ret = xe_bo_validate(sys_bo, NULL, false, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret); + goto free_sysbo; + } + + ret = xe_bo_vmap(sys_bo); + if (ret) { + KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret); + goto free_sysbo; + } + xe_bo_unlock(sys_bo); + + ccs_bo = xe_bo_create_user(xe, NULL, SZ_4M, + DRM_XE_GEM_CPU_CACHING_WC, + bo_flags | XE_BO_FLAG_NEEDS_CPU_ACCESS | + XE_BO_FLAG_PINNED, NULL); + + if (IS_ERR(ccs_bo)) { + KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", + PTR_ERR(ccs_bo)); + return; + } + + xe_bo_lock(ccs_bo, false); + ret = xe_bo_validate(ccs_bo, NULL, false, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret); + goto free_ccsbo; + } + + ret = xe_bo_vmap(ccs_bo); + if (ret) { + KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret); + goto free_ccsbo; + } + xe_bo_unlock(ccs_bo); + + vram_bo = xe_bo_create_user(xe, NULL, SZ_4M, + DRM_XE_GEM_CPU_CACHING_WC, + bo_flags | XE_BO_FLAG_NEEDS_CPU_ACCESS | + XE_BO_FLAG_PINNED, NULL); + if (IS_ERR(vram_bo)) { + KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", + PTR_ERR(vram_bo)); + return; + } + + xe_bo_lock(vram_bo, false); + ret = xe_bo_validate(vram_bo, NULL, false, exec); + if (ret) { + KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret); + goto free_vrambo; + } + + ret = xe_bo_vmap(vram_bo); + if (ret) { + KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret); + goto free_vrambo; + } + + test_clear(xe, tile, sys_bo, vram_bo, test); + test_migrate(xe, tile, sys_bo, vram_bo, ccs_bo, exec, test); + xe_bo_unlock(vram_bo); + + xe_bo_lock(vram_bo, false); + xe_bo_vunmap(vram_bo); + xe_bo_unlock(vram_bo); + + xe_bo_lock(ccs_bo, false); + xe_bo_vunmap(ccs_bo); + xe_bo_unlock(ccs_bo); + + xe_bo_lock(sys_bo, false); + xe_bo_vunmap(sys_bo); + xe_bo_unlock(sys_bo); +free_vrambo: + xe_bo_put(vram_bo); +free_ccsbo: + xe_bo_put(ccs_bo); +free_sysbo: + xe_bo_put(sys_bo); +} + +static int validate_ccs_test_run_device(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_tile *tile; + int id; + + if (!xe_device_has_flat_ccs(xe)) { + kunit_skip(test, "non-flat-ccs device\n"); + return 0; + } + + if (!(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))) { + kunit_skip(test, "non-xe2 discrete device\n"); + return 0; + } + + guard(xe_pm_runtime)(xe); + for_each_tile(tile, xe, id) + validate_ccs_test_run_tile(xe, tile, test); + + return 0; +} + +static void xe_validate_ccs_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + validate_ccs_test_run_device(xe); +} + +static struct kunit_case xe_migrate_tests[] = { + KUNIT_CASE_PARAM(xe_migrate_sanity_kunit, xe_pci_live_device_gen_param), + KUNIT_CASE_PARAM(xe_validate_ccs_kunit, xe_pci_live_device_gen_param), + {} +}; + +VISIBLE_IF_KUNIT +struct kunit_suite xe_migrate_test_suite = { + .name = "xe_migrate", + .test_cases = xe_migrate_tests, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_migrate_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_mocs.c b/drivers/gpu/drm/xe/tests/xe_mocs.c new file mode 100644 index 000000000..daf3c6836 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_mocs.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include "tests/xe_kunit_helpers.h" +#include "tests/xe_pci_test.h" +#include "tests/xe_test.h" + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_mocs.h" +#include "xe_pci.h" +#include "xe_pm.h" + +struct live_mocs { + struct xe_mocs_info table; +}; + +static int live_mocs_init(struct live_mocs *arg, struct xe_gt *gt) +{ + unsigned int flags; + struct kunit *test = kunit_get_current_test(); + + memset(arg, 0, sizeof(*arg)); + + flags = get_mocs_settings(gt_to_xe(gt), &arg->table); + + kunit_info(test, "gt %d", gt->info.id); + kunit_info(test, "gt type %d", gt->info.type); + kunit_info(test, "table size %d", arg->table.table_size); + kunit_info(test, "table uc_index %d", arg->table.uc_index); + kunit_info(test, "table num_mocs_regs %d", arg->table.num_mocs_regs); + + return flags; +} + +static void read_l3cc_table(struct xe_gt *gt, + const struct xe_mocs_info *info) +{ + struct kunit *test = kunit_get_current_test(); + u32 l3cc, l3cc_expected; + unsigned int i; + u32 reg_val; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + KUNIT_FAIL_AND_ABORT(test, "Forcewake Failed.\n"); + + for (i = 0; i < info->num_mocs_regs; i++) { + if (!(i & 1)) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i >> 1)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i >> 1)); + + mocs_dbg(gt, "reg_val=0x%x\n", reg_val); + } else { + /* Just reuse value read on previous iteration */ + reg_val >>= 16; + } + + l3cc_expected = get_entry_l3cc(info, i); + l3cc = reg_val & 0xffff; + + mocs_dbg(gt, "[%u] expected=0x%x actual=0x%x\n", + i, l3cc_expected, l3cc); + + KUNIT_EXPECT_EQ_MSG(test, l3cc_expected, l3cc, + "l3cc idx=%u has incorrect val.\n", i); + } +} + +static void read_mocs_table(struct xe_gt *gt, + const struct xe_mocs_info *info) +{ + struct kunit *test = kunit_get_current_test(); + u32 mocs, mocs_expected; + unsigned int i; + u32 reg_val; + + KUNIT_EXPECT_TRUE_MSG(test, info->unused_entries_index, + "Unused entries index should have been defined\n"); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + KUNIT_ASSERT_NE_MSG(test, fw_ref.domains, 0, "Forcewake Failed.\n"); + + for (i = 0; i < info->num_mocs_regs; i++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); + + mocs_expected = get_entry_control(info, i); + mocs = reg_val; + + mocs_dbg(gt, "[%u] expected=0x%x actual=0x%x\n", + i, mocs_expected, mocs); + + KUNIT_EXPECT_EQ_MSG(test, mocs_expected, mocs, + "mocs reg 0x%x has incorrect val.\n", i); + } +} + +static int mocs_kernel_test_run_device(struct xe_device *xe) +{ + /* Basic check the system is configured with the expected mocs table */ + + struct live_mocs mocs; + struct xe_gt *gt; + + unsigned int flags; + int id; + + guard(xe_pm_runtime)(xe); + for_each_gt(gt, xe, id) { + flags = live_mocs_init(&mocs, gt); + if (flags & HAS_GLOBAL_MOCS) + read_mocs_table(gt, &mocs.table); + if (flags & HAS_LNCF_MOCS) + read_l3cc_table(gt, &mocs.table); + } + + return 0; +} + +static void xe_live_mocs_kernel_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + if (IS_SRIOV_VF(xe)) + kunit_skip(test, "this test is N/A for VF"); + + mocs_kernel_test_run_device(xe); +} + +static int mocs_reset_test_run_device(struct xe_device *xe) +{ + /* Check the mocs setup is retained over GT reset */ + + struct live_mocs mocs; + struct xe_gt *gt; + unsigned int flags; + int id; + struct kunit *test = kunit_get_current_test(); + + guard(xe_pm_runtime)(xe); + for_each_gt(gt, xe, id) { + flags = live_mocs_init(&mocs, gt); + kunit_info(test, "mocs_reset_test before reset\n"); + if (flags & HAS_GLOBAL_MOCS) + read_mocs_table(gt, &mocs.table); + if (flags & HAS_LNCF_MOCS) + read_l3cc_table(gt, &mocs.table); + + xe_gt_reset(gt); + + kunit_info(test, "mocs_reset_test after reset\n"); + if (flags & HAS_GLOBAL_MOCS) + read_mocs_table(gt, &mocs.table); + if (flags & HAS_LNCF_MOCS) + read_l3cc_table(gt, &mocs.table); + } + + return 0; +} + +static void xe_live_mocs_reset_kunit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + if (IS_SRIOV_VF(xe)) + kunit_skip(test, "this test is N/A for VF"); + + mocs_reset_test_run_device(xe); +} + +static struct kunit_case xe_mocs_tests[] = { + KUNIT_CASE_PARAM(xe_live_mocs_kernel_kunit, xe_pci_live_device_gen_param), + KUNIT_CASE_PARAM(xe_live_mocs_reset_kunit, xe_pci_live_device_gen_param), + {} +}; + +VISIBLE_IF_KUNIT +struct kunit_suite xe_mocs_test_suite = { + .name = "xe_mocs", + .test_cases = xe_mocs_tests, + .init = xe_kunit_helper_xe_device_live_test_init, +}; +EXPORT_SYMBOL_IF_KUNIT(xe_mocs_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c new file mode 100644 index 000000000..bb0393475 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "tests/xe_pci_test.h" + +#include "tests/xe_test.h" + +#include +#include +#include + +#define PLATFORM_CASE(platform__, graphics_step__) \ + { \ + .platform = XE_ ## platform__, \ + .subplatform = XE_SUBPLATFORM_NONE, \ + .step = { .graphics = STEP_ ## graphics_step__ } \ + } + +#define SUBPLATFORM_CASE(platform__, subplatform__, graphics_step__) \ + { \ + .platform = XE_ ## platform__, \ + .subplatform = XE_SUBPLATFORM_ ## platform__ ## _ ## subplatform__, \ + .step = { .graphics = STEP_ ## graphics_step__ } \ + } + +#define GMDID_CASE(platform__, graphics_verx100__, graphics_step__, \ + media_verx100__, media_step__) \ + { \ + .platform = XE_ ## platform__, \ + .subplatform = XE_SUBPLATFORM_NONE, \ + .graphics_verx100 = graphics_verx100__, \ + .media_verx100 = media_verx100__, \ + .step = { .graphics = STEP_ ## graphics_step__, \ + .media = STEP_ ## media_step__ } \ + } + +static const struct xe_pci_fake_data cases[] = { + PLATFORM_CASE(TIGERLAKE, B0), + PLATFORM_CASE(DG1, A0), + PLATFORM_CASE(DG1, B0), + PLATFORM_CASE(ALDERLAKE_S, A0), + PLATFORM_CASE(ALDERLAKE_S, B0), + PLATFORM_CASE(ALDERLAKE_S, C0), + PLATFORM_CASE(ALDERLAKE_S, D0), + PLATFORM_CASE(ALDERLAKE_P, A0), + PLATFORM_CASE(ALDERLAKE_P, B0), + PLATFORM_CASE(ALDERLAKE_P, C0), + SUBPLATFORM_CASE(ALDERLAKE_S, RPLS, D0), + SUBPLATFORM_CASE(ALDERLAKE_P, RPLU, E0), + SUBPLATFORM_CASE(DG2, G10, C0), + SUBPLATFORM_CASE(DG2, G11, B1), + SUBPLATFORM_CASE(DG2, G12, A1), + GMDID_CASE(METEORLAKE, 1270, A0, 1300, A0), + GMDID_CASE(METEORLAKE, 1271, A0, 1300, A0), + GMDID_CASE(METEORLAKE, 1274, A0, 1300, A0), + GMDID_CASE(LUNARLAKE, 2004, A0, 2000, A0), + GMDID_CASE(LUNARLAKE, 2004, B0, 2000, A0), + GMDID_CASE(BATTLEMAGE, 2001, A0, 1301, A1), + GMDID_CASE(PANTHERLAKE, 3000, A0, 3000, A0), +}; + +KUNIT_ARRAY_PARAM(platform, cases, xe_pci_fake_data_desc); + +/** + * xe_pci_fake_data_gen_params - Generate struct xe_pci_fake_data parameters + * @test: test context object + * @prev: the pointer to the previous parameter to iterate from or NULL + * @desc: output buffer with minimum size of KUNIT_PARAM_DESC_SIZE + * + * This function prepares struct xe_pci_fake_data parameter. + * + * To be used only as a parameter generator function in &KUNIT_CASE_PARAM. + * + * Return: pointer to the next parameter or NULL if no more parameters + */ +const void *xe_pci_fake_data_gen_params(struct kunit *test, const void *prev, char *desc) +{ + return platform_gen_params(test, prev, desc); +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_fake_data_gen_params); + +static const struct xe_device_desc *lookup_desc(enum xe_platform p) +{ + const struct xe_device_desc *desc; + const struct pci_device_id *ids; + + for (ids = pciidlist; ids->driver_data; ids++) { + desc = (const void *)ids->driver_data; + if (desc->platform == p) + return desc; + } + return NULL; +} + +static const struct xe_subplatform_desc *lookup_sub_desc(enum xe_platform p, enum xe_subplatform s) +{ + const struct xe_device_desc *desc = lookup_desc(p); + const struct xe_subplatform_desc *spd; + + if (desc && desc->subplatforms) + for (spd = desc->subplatforms; spd->subplatform; spd++) + if (spd->subplatform == s) + return spd; + return NULL; +} + +static const char *lookup_platform_name(enum xe_platform p) +{ + const struct xe_device_desc *desc = lookup_desc(p); + + return desc ? desc->platform_name : "INVALID"; +} + +static const char *__lookup_subplatform_name(enum xe_platform p, enum xe_subplatform s) +{ + const struct xe_subplatform_desc *desc = lookup_sub_desc(p, s); + + return desc ? desc->name : "INVALID"; +} + +static const char *lookup_subplatform_name(enum xe_platform p, enum xe_subplatform s) +{ + return s == XE_SUBPLATFORM_NONE ? "" : __lookup_subplatform_name(p, s); +} + +static const char *subplatform_prefix(enum xe_subplatform s) +{ + return s == XE_SUBPLATFORM_NONE ? "" : " "; +} + +static const char *step_prefix(enum intel_step step) +{ + return step == STEP_NONE ? "" : " "; +} + +static const char *step_name(enum intel_step step) +{ + return step == STEP_NONE ? "" : xe_step_name(step); +} + +static const char *sriov_prefix(enum xe_sriov_mode mode) +{ + return mode <= XE_SRIOV_MODE_NONE ? "" : " "; +} + +static const char *sriov_name(enum xe_sriov_mode mode) +{ + return mode <= XE_SRIOV_MODE_NONE ? "" : xe_sriov_mode_to_string(mode); +} + +static const char *lookup_graphics_name(unsigned int verx100) +{ + const struct xe_ip *ip = find_graphics_ip(verx100); + + return ip ? ip->name : ""; +} + +static const char *lookup_media_name(unsigned int verx100) +{ + const struct xe_ip *ip = find_media_ip(verx100); + + return ip ? ip->name : ""; +} + +/** + * xe_pci_fake_data_desc - Describe struct xe_pci_fake_data parameter + * @param: the &struct xe_pci_fake_data parameter to describe + * @desc: output buffer with minimum size of KUNIT_PARAM_DESC_SIZE + * + * This function prepares description of the struct xe_pci_fake_data parameter. + * + * It is tailored for use in parameterized KUnit tests where parameter generator + * is based on the struct xe_pci_fake_data arrays. + */ +void xe_pci_fake_data_desc(const struct xe_pci_fake_data *param, char *desc) +{ + if (param->graphics_verx100 || param->media_verx100) + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s%s%s %u.%02u(%s)%s%s %u.%02u(%s)%s%s%s%s", + lookup_platform_name(param->platform), + subplatform_prefix(param->subplatform), + lookup_subplatform_name(param->platform, param->subplatform), + param->graphics_verx100 / 100, param->graphics_verx100 % 100, + lookup_graphics_name(param->graphics_verx100), + step_prefix(param->step.graphics), step_name(param->step.graphics), + param->media_verx100 / 100, param->media_verx100 % 100, + lookup_media_name(param->media_verx100), + step_prefix(param->step.media), step_name(param->step.media), + sriov_prefix(param->sriov_mode), sriov_name(param->sriov_mode)); + else + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s%s%s%s%s%s%s", + lookup_platform_name(param->platform), + subplatform_prefix(param->subplatform), + lookup_subplatform_name(param->platform, param->subplatform), + step_prefix(param->step.graphics), step_name(param->step.graphics), + sriov_prefix(param->sriov_mode), sriov_name(param->sriov_mode)); +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_fake_data_desc); + +static void xe_ip_kunit_desc(const struct xe_ip *param, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%u.%02u %s", + param->verx100 / 100, param->verx100 % 100, param->name); +} + +/* + * Pre-GMDID Graphics and Media IPs definitions. + * + * Mimic the way GMDID IPs are declared so the same + * param generator can be used for both + */ +static const struct xe_ip pre_gmdid_graphics_ips[] = { + { 1200, "Xe_LP", &graphics_xelp }, + { 1210, "Xe_LP+", &graphics_xelp }, + { 1255, "Xe_HPG", &graphics_xehpg }, + { 1260, "Xe_HPC", &graphics_xehpc }, +}; + +static const struct xe_ip pre_gmdid_media_ips[] = { + { 1200, "Xe_M", &media_xem }, + { 1255, "Xe_HPM", &media_xem }, +}; + +KUNIT_ARRAY_PARAM(pre_gmdid_graphics_ip, pre_gmdid_graphics_ips, xe_ip_kunit_desc); +KUNIT_ARRAY_PARAM(pre_gmdid_media_ip, pre_gmdid_media_ips, xe_ip_kunit_desc); + +KUNIT_ARRAY_PARAM(graphics_ip, graphics_ips, xe_ip_kunit_desc); +KUNIT_ARRAY_PARAM(media_ip, media_ips, xe_ip_kunit_desc); + +static void xe_pci_id_kunit_desc(const struct pci_device_id *param, char *desc) +{ + const struct xe_device_desc *dev_desc = + (const struct xe_device_desc *)param->driver_data; + + if (dev_desc) + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "0x%X (%s)", + param->device, dev_desc->platform_name); +} + +KUNIT_ARRAY_PARAM(pci_id, pciidlist, xe_pci_id_kunit_desc); + +/** + * xe_pci_graphics_ip_gen_param - Generate graphics struct xe_ip parameters + * @test: test context object + * @prev: the pointer to the previous parameter to iterate from or NULL + * @desc: output buffer with minimum size of KUNIT_PARAM_DESC_SIZE + * + * This function prepares struct xe_ip parameter. + * + * To be used only as a parameter generator function in &KUNIT_CASE_PARAM. + * + * Return: pointer to the next parameter or NULL if no more parameters + */ +const void *xe_pci_graphics_ip_gen_param(struct kunit *test, const void *prev, char *desc) +{ + const void *next = pre_gmdid_graphics_ip_gen_params(test, prev, desc); + + if (next) + return next; + if (is_insidevar(prev, pre_gmdid_graphics_ips)) + prev = NULL; + + return graphics_ip_gen_params(test, prev, desc); +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_graphics_ip_gen_param); + +/** + * xe_pci_media_ip_gen_param - Generate media struct xe_ip parameters + * @test: test context object + * @prev: the pointer to the previous parameter to iterate from or NULL + * @desc: output buffer with minimum size of KUNIT_PARAM_DESC_SIZE + * + * This function prepares struct xe_ip parameter. + * + * To be used only as a parameter generator function in &KUNIT_CASE_PARAM. + * + * Return: pointer to the next parameter or NULL if no more parameters + */ +const void *xe_pci_media_ip_gen_param(struct kunit *test, const void *prev, char *desc) +{ + const void *next = pre_gmdid_media_ip_gen_params(test, prev, desc); + + if (next) + return next; + if (is_insidevar(prev, pre_gmdid_media_ips)) + prev = NULL; + + return media_ip_gen_params(test, prev, desc); +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_media_ip_gen_param); + +/** + * xe_pci_id_gen_param - Generate struct pci_device_id parameters + * @test: test context object + * @prev: the pointer to the previous parameter to iterate from or NULL + * @desc: output buffer with minimum size of KUNIT_PARAM_DESC_SIZE + * + * This function prepares struct pci_device_id parameter. + * + * To be used only as a parameter generator function in &KUNIT_CASE_PARAM. + * + * Return: pointer to the next parameter or NULL if no more parameters + */ +const void *xe_pci_id_gen_param(struct kunit *test, const void *prev, char *desc) +{ + const struct pci_device_id *pci = pci_id_gen_params(test, prev, desc); + + return pci->driver_data ? pci : NULL; +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_id_gen_param); + +static int fake_probe_info(struct xe_device *xe, + const struct xe_device_desc *desc, + struct xe_pci_fake_data *data, + struct xe_probed_info *probed_info) +{ + probed_info->tile_count = 1 + desc->max_remote_tiles; + + if (!data || desc->pre_gmdid_graphics_ip) { + probed_info->graphics_ip = desc->pre_gmdid_graphics_ip; + probed_info->media_ip = desc->pre_gmdid_media_ip; + } else { + probed_info->graphics_ip = find_graphics_ip(data->graphics_verx100); + + if (data->media_verx100) { + probed_info->media_ip = find_media_ip(data->media_verx100); + xe_assert(xe, probed_info->media_ip); + } + } + + xe_assert(xe, probed_info->graphics_ip); + if (!probed_info->graphics_ip) + return -ENODEV; + + if (data) + probed_info->step = data->step; + + return 0; +} + +int xe_pci_fake_device_init(struct xe_device *xe) +{ + struct kunit *test = kunit_get_current_test(); + struct xe_pci_fake_data *data = test->priv; + struct xe_probed_info probed_info = {}; + const struct pci_device_id *ent = pciidlist; + const struct xe_device_desc *desc; + const struct xe_subplatform_desc *subplatform_desc; + int err; + + if (!data) { + desc = (const void *)ent->driver_data; + subplatform_desc = NULL; + goto done; + } + + for (ent = pciidlist; ent->device; ent++) { + desc = (const void *)ent->driver_data; + if (desc->platform == data->platform) + break; + } + + if (!ent->device) + return -ENODEV; + + for (subplatform_desc = desc->subplatforms; + subplatform_desc && subplatform_desc->subplatform; + subplatform_desc++) + if (subplatform_desc->subplatform == data->subplatform) + break; + + if (data->subplatform != XE_SUBPLATFORM_NONE && !subplatform_desc) + return -ENODEV; + +done: + xe->sriov.__mode = data && data->sriov_mode ? + data->sriov_mode : XE_SRIOV_MODE_NONE; + + err = fake_probe_info(xe, desc, data, &probed_info); + if (err) + return err; + + xe_info_init_early(xe, desc, subplatform_desc, &probed_info); + xe_info_init(xe, desc, &probed_info); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_fake_device_init); + +/** + * xe_pci_live_device_gen_param - Helper to iterate Xe devices as KUnit parameters + * @test: test context object + * @prev: the previously returned value, or NULL for the first iteration + * @desc: the buffer for a parameter name + * + * Iterates over the available Xe devices on the system. Uses the device name + * as the parameter name. + * + * To be used only as a parameter generator function in &KUNIT_CASE_PARAM. + * + * Return: pointer to the next &struct xe_device ready to be used as a parameter + * or NULL if there are no more Xe devices on the system. + */ +const void *xe_pci_live_device_gen_param(struct kunit *test, const void *prev, char *desc) +{ + const struct xe_device *xe = prev; + struct device *dev = xe ? xe->drm.dev : NULL; + struct device *next; + + next = driver_find_next_device(&xe_pci_driver.driver, dev); + if (dev) + put_device(dev); + if (!next) + return NULL; + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", dev_name(next)); + return pdev_to_xe_device(to_pci_dev(next)); +} +EXPORT_SYMBOL_IF_KUNIT(xe_pci_live_device_gen_param); diff --git a/drivers/gpu/drm/xe/tests/xe_pci_test.c b/drivers/gpu/drm/xe/tests/xe_pci_test.c new file mode 100644 index 000000000..acf5a5555 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_pci_test.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include + +#include + +#include "tests/xe_test.h" + +#include "xe_device.h" +#include "xe_pci_test.h" +#include "xe_pci_types.h" + +static void check_graphics_ip(struct kunit *test) +{ + const struct xe_ip *param = test->param_value; + const struct xe_graphics_desc *graphics = param->desc; + u64 mask = graphics->hw_engine_mask; + u8 fuse_regs = graphics->num_geometry_xecore_fuse_regs + + graphics->num_compute_xecore_fuse_regs; + + /* RCS, CCS, and BCS engines are allowed on the graphics IP */ + mask &= ~(XE_HW_ENGINE_RCS_MASK | + XE_HW_ENGINE_CCS_MASK | + XE_HW_ENGINE_BCS_MASK); + + /* Any remaining engines are an error */ + KUNIT_ASSERT_EQ(test, mask, 0); + + /* + * All graphics IP should have at least one geometry and/or compute + * XeCore fuse register. + */ + KUNIT_ASSERT_GE(test, fuse_regs, 1); +} + +static void check_media_ip(struct kunit *test) +{ + const struct xe_ip *param = test->param_value; + const struct xe_media_desc *media = param->desc; + u64 mask = media->hw_engine_mask; + + /* VCS, VECS and GSCCS engines are allowed on the media IP */ + mask &= ~(XE_HW_ENGINE_VCS_MASK | + XE_HW_ENGINE_VECS_MASK | + XE_HW_ENGINE_GSCCS_MASK); + + /* Any remaining engines are an error */ + KUNIT_ASSERT_EQ(test, mask, 0); +} + +static void check_platform_desc(struct kunit *test) +{ + const struct pci_device_id *pci = test->param_value; + const struct xe_device_desc *desc = + (const struct xe_device_desc *)pci->driver_data; + + KUNIT_EXPECT_GT(test, desc->dma_mask_size, 0); + + KUNIT_EXPECT_GT(test, (unsigned int)desc->max_gt_per_tile, 0); + KUNIT_EXPECT_LE(test, (unsigned int)desc->max_gt_per_tile, XE_MAX_GT_PER_TILE); + + KUNIT_EXPECT_GT(test, desc->va_bits, 0); + KUNIT_EXPECT_LE(test, desc->va_bits, 64); + + KUNIT_EXPECT_GT(test, desc->vm_max_level, 0); +} + +static struct kunit_case xe_pci_tests[] = { + KUNIT_CASE_PARAM(check_graphics_ip, xe_pci_graphics_ip_gen_param), + KUNIT_CASE_PARAM(check_media_ip, xe_pci_media_ip_gen_param), + KUNIT_CASE_PARAM(check_platform_desc, xe_pci_id_gen_param), + {} +}; + +static struct kunit_suite xe_pci_test_suite = { + .name = "xe_pci", + .test_cases = xe_pci_tests, +}; + +kunit_test_suite(xe_pci_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_pci_test.h b/drivers/gpu/drm/xe/tests/xe_pci_test.h new file mode 100644 index 000000000..30505d1cb --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_pci_test.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_PCI_TEST_H_ +#define _XE_PCI_TEST_H_ + +#include +#include + +#include "xe_platform_types.h" +#include "xe_sriov_types.h" +#include "xe_step_types.h" + +struct xe_device; + +struct xe_pci_fake_data { + enum xe_sriov_mode sriov_mode; + enum xe_platform platform; + enum xe_subplatform subplatform; + struct xe_step_info step; + u32 graphics_verx100; + u32 media_verx100; +}; + +int xe_pci_fake_device_init(struct xe_device *xe); +const void *xe_pci_fake_data_gen_params(struct kunit *test, const void *prev, char *desc); +void xe_pci_fake_data_desc(const struct xe_pci_fake_data *param, char *desc); + +const void *xe_pci_graphics_ip_gen_param(struct kunit *test, const void *prev, char *desc); +const void *xe_pci_media_ip_gen_param(struct kunit *test, const void *prev, char *desc); +const void *xe_pci_id_gen_param(struct kunit *test, const void *prev, char *desc); +const void *xe_pci_live_device_gen_param(struct kunit *test, const void *prev, char *desc); + +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_rtp.c b/drivers/gpu/drm/xe/tests/xe_rtp.c new file mode 100644 index 000000000..b3a8b7593 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_rtp.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "tests/xe_rtp_test.h" + +#include + +/** + * xe_rtp_rule_matches - Check if a set of RTP rule set match against the + * device/GT/hwe + * @xe: The xe device + * @gt: The GT struct (may be NULL) + * @hwe: The hw_engine (may be NULL) + * @rules: The array of rules to match against + * @n_rules: Number of items in @rules + * @err: Pointer (may be NULL) to set error number. + * + * This parses the set of rules and check if they match against the passed + * parameters. + * + * If passed, @err is updated with a non-zero negative error number or zero if + * no errors were found during the parsing/evaluation of rules. + * + * Returns true if there is a match and false if there is no match or if an + * error was found. + */ +bool xe_rtp_rule_matches(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules, + int *err) +{ + return rule_matches_with_err(xe, gt, hwe, rules, n_rules, err); +} +EXPORT_SYMBOL_IF_KUNIT(xe_rtp_rule_matches); diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c new file mode 100644 index 000000000..7e2fc39ac --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2026 Intel Corporation + */ + +#include + +#include "xe_reg_whitelist.h" +#include "xe_rtp_types.h" +#include "xe_tuning.h" +#include "xe_wa.h" + +#define RTP_TABLE_PARAM(table) \ + static const void *table##_gen_params(struct kunit *test, \ + const void *prev, char *desc) \ + { \ + typeof((table.entries)[0]) *__next = prev ? \ + ((typeof(__next))prev) + 1 : (table.entries); \ + if (__next - table.entries < table.n_entries) { \ + scnprintf(desc, KUNIT_PARAM_DESC_SIZE, #table "/%s", __next->name); \ + return __next; \ + } \ + return NULL; \ + } + +static void xe_rtp_table_gt_test(struct kunit *test) +{ + const struct xe_rtp_entry_sr *entry = test->param_value; + + for (int i = 0; i < entry->n_rules; i++) { + KUNIT_EXPECT_TRUE(test, + entry->rules[i].match_type != XE_RTP_MATCH_ENGINE_CLASS || + entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE); + KUNIT_EXPECT_TRUE(test, + entry->rules[i].match_type != XE_RTP_MATCH_NOT_ENGINE_CLASS || + entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE); + } +} + +RTP_TABLE_PARAM(gt_was); +RTP_TABLE_PARAM(gt_tunings); + +static void xe_rtp_table_oob_test(struct kunit *test) +{ + const struct xe_rtp_entry *entry = test->param_value; + + for (int i = 0; i < entry->n_rules; i++) { + u8 match_type = entry->rules[i].match_type; + + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_ENGINE_CLASS); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_NOT_ENGINE_CLASS); + } +} + +RTP_TABLE_PARAM(oob_was); + +static void xe_rtp_table_dev_oob_test(struct kunit *test) +{ + const struct xe_rtp_entry *entry = test->param_value; + + for (int i = 0; i < entry->n_rules; i++) { + u8 match_type = entry->rules[i].match_type; + + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_ENGINE_CLASS); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_NOT_ENGINE_CLASS); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_VERSION); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_VERSION_RANGE); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_STEP); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_VERSION); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_VERSION_RANGE); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_VERSION_ANY_GT); + KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_STEP); + } +} + +RTP_TABLE_PARAM(device_oob_was); + +static void xe_rtp_table_missing_upper_bound_test(struct kunit *test) +{ + const struct xe_rtp_entry_sr *entry = test->param_value; + + for (int i = 0; i < entry->n_rules; i++) { + u8 match_type = entry->rules[i].match_type; + + KUNIT_EXPECT_FALSE(test, + match_type == XE_RTP_MATCH_GRAPHICS_VERSION_RANGE && + entry->rules[i].ver_end == XE_RTP_END_VERSION_UNDEFINED); + KUNIT_EXPECT_FALSE(test, + match_type == XE_RTP_MATCH_MEDIA_VERSION_RANGE && + entry->rules[i].ver_end == XE_RTP_END_VERSION_UNDEFINED); + } +} + +RTP_TABLE_PARAM(register_whitelist); + +static struct kunit_case xe_rtp_table_tests[] = { + KUNIT_CASE_PARAM(xe_rtp_table_gt_test, gt_was_gen_params), + KUNIT_CASE_PARAM(xe_rtp_table_gt_test, gt_tunings_gen_params), + KUNIT_CASE_PARAM(xe_rtp_table_oob_test, oob_was_gen_params), + KUNIT_CASE_PARAM(xe_rtp_table_dev_oob_test, device_oob_was_gen_params), + KUNIT_CASE_PARAM(xe_rtp_table_missing_upper_bound_test, + register_whitelist_gen_params), + {} +}; + +static struct kunit_suite xe_rtp_tables_test_suite = { + .name = "xe_rtp_tables_test", + .test_cases = xe_rtp_table_tests, +}; + +kunit_test_suite(xe_rtp_tables_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_test.c new file mode 100644 index 000000000..367811621 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.c @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include + +#include +#include + +#include +#include + +#include "regs/xe_gt_regs.h" +#include "regs/xe_reg_defs.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_gt_mcr.h" +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" +#include "xe_reg_sr.h" +#include "xe_rtp.h" +#include "xe_rtp_test.h" + +#define REGULAR_REG1 XE_REG(1) +#define REGULAR_REG2 XE_REG(2) +#define REGULAR_REG3 XE_REG(3) +#define REGULAR_REG4 XE_REG(4) +#define BAD_REGULAR_REG5 XE_REG(5) +#define MCR_REG1 XE_REG_MCR(1) +#define MCR_REG2 XE_REG_MCR(2) +#define MCR_REG3 XE_REG_MCR(3) +#define BAD_MCR_REG4 XE_REG_MCR(4) +#define MCR_REG5 XE_REG_MCR(5) +#define MASKED_REG1 XE_REG(1, XE_REG_OPTION_MASKED) + +#undef XE_REG_MCR +#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) + +struct rtp_rules_test_case { + const char *name; + bool expected_match; + int expected_err; + const struct xe_rtp_rule *rules; + u8 n_rules; +}; + +struct rtp_to_sr_test_case { + const char *name; + struct xe_reg expected_reg; + u32 expected_set_bits; + u32 expected_clr_bits; + unsigned long expected_count_sr_entries; + unsigned int expected_sr_errors; + unsigned long expected_active; + const struct xe_rtp_table_sr table; +}; + +struct rtp_test_case { + const char *name; + unsigned long expected_active; + const struct xe_rtp_table table; +}; + +static bool fake_xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) +{ + /* + * All supported platforms in this imaginary setup will always have REG4 + * as a non-MCR register and REG5 as MCR, meaning that BAD_MCR_REG4 and + * BAD_REGULAR_REG5 represent programming errors to be captured by our + * tests. + */ + if (reg.raw == BAD_REGULAR_REG5.raw) + return true; + + if (reg.raw == BAD_MCR_REG4.raw) + return false; + + return reg.mcr; +} + +static bool match_yes(const struct xe_device *xe, const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return true; +} + +static bool match_no(const struct xe_device *xe, const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return false; +} + +static const struct rtp_rules_test_case rtp_rules_cases[] = { + /* + * Single rules. + * + * TODO: Include other types of rules as well: GRAPHICS_VERSION(), + * MEDIA_VERSION(), etc. + */ + { + .name = "no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no)), + }, + { + .name = "yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes)), + }, + + /* Conjunctions with 2 operands. */ + { + .name = "no-and-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), FUNC(match_no)), + }, + { + .name = "no-and-yes", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), FUNC(match_yes)), + }, + { + .name = "yes-and-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_no)), + }, + { + .name = "yes-and-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_yes)), + }, + + /* Disjunctions with 2 operands. */ + { + .name = "no-or-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), OR, FUNC(match_no)), + }, + { + .name = "no-or-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_no), OR, FUNC(match_yes)), + }, + { + .name = "yes-or-no", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_no)), + }, + { + .name = "yes-or-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_yes)), + }, + + /* Conjunction and disjunctions. */ + { + .name = "no-yes-or-yes-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), FUNC(match_yes), OR, + FUNC(match_yes), FUNC(match_no)), + }, + { + .name = "no-yes-or-yes-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_no), FUNC(match_yes), OR, + FUNC(match_yes), FUNC(match_yes)), + }, + { + .name = "yes-yes-or-no-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_yes), OR, + FUNC(match_no), FUNC(match_yes)), + }, + { + .name = "yes-yes-or-yes-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_yes), OR, + FUNC(match_yes), FUNC(match_yes)), + }, + { + .name = "no-no-or-yes-or-no", + .expected_match = true, + XE_RTP_RULES(FUNC(match_no), FUNC(match_no), OR, + FUNC(match_yes), OR, + FUNC(match_no)), + }, + + /* Syntax errors. */ + { + .name = "or", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(OR), + }, + { + .name = "or-yes", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(OR, FUNC(match_yes)), + }, + { + .name = "or-no", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(OR, FUNC(match_no)), + }, + { + .name = "yes-or", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_yes), OR), + }, + { + .name = "no-or", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_no), OR), + }, + { + .name = "no-or-or-yes", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_no), OR, OR, FUNC(match_yes)), + }, + { + .name = "yes-or-or-no", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_yes), OR, OR, FUNC(match_no)), + }, + { + .name = "no-or-or-no", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_no), OR, OR, FUNC(match_no)), + }, + + /* No match because hwe is NULL. */ + { + .name = "missing-context-engine-class", + .expected_match = false, + XE_RTP_RULES(ENGINE_CLASS(RENDER)), + }, + + /* + * Missing context (hwe==NULL) does not cause parsing to stop, hence we + * expect a match. + */ + { + .name = "missing-context-engine-class-or-yes", + .expected_match = true, + XE_RTP_RULES(ENGINE_CLASS(RENDER), OR, FUNC(match_yes)), + }, + + /* + * Missing context (hwe==NULL) does not cause parsing to stop, hence we + * expect a syntax error. + */ + { + .name = "missing-context-engine-class-or-or-yes", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(ENGINE_CLASS(RENDER), OR, OR, FUNC(match_yes)), + }, +}; + +static void xe_rtp_rules_tests(struct kunit *test) +{ + const struct rtp_rules_test_case *param = test->param_value; + struct xe_device *xe = test->priv; + struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt; + int err; + bool match; + + match = xe_rtp_rule_matches(xe, gt, NULL, param->rules, param->n_rules, &err); + + KUNIT_EXPECT_EQ(test, match, param->expected_match); + KUNIT_EXPECT_EQ(test, err, param->expected_err); +} + +static u32 bits_2_3_set(struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + return REG_BIT(2) | REG_BIT(3); +} + +static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { + { + .name = "coalesce-same-reg", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0) | REG_BIT(1), + .expected_clr_bits = REG_BIT(0) | REG_BIT(1), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 1, + /* Different bits on the same register: create a single entry */ + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) + }, + ), + }, + { + .name = "coalesce-same-reg-literal-and-func", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0) | REG_BIT(1) | REG_BIT(2) | REG_BIT(3), + .expected_clr_bits = REG_BIT(0) | REG_BIT(1) | REG_BIT(2) | REG_BIT(3), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 1, + /* Different bits on the same register: create a single entry */ + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(FIELD_SET(REGULAR_REG1, + REG_BIT(0) | REG_BIT(1), + REG_BIT(0) | REG_BIT(1))) + }, + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(FIELD_SET_FUNC(REGULAR_REG1, + REG_BIT(2) | REG_BIT(3), + bits_2_3_set)) + }, + ), + }, + { + .name = "no-match-no-add", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + /* Don't coalesce second entry since rules don't match */ + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_no)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) + }, + ), + }, + { + .name = "two-regs-two-entries", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 2, + /* Same bits on different registers are not coalesced */ + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG2, REG_BIT(0))) + }, + ), + }, + { + .name = "clr-one-set-other", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(1) | REG_BIT(0), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 1, + /* Check clr vs set actions on different bits */ + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(CLR(REGULAR_REG1, REG_BIT(1))) + }, + ), + }, + { +#define TEMP_MASK REG_GENMASK(10, 8) +#define TEMP_FIELD REG_FIELD_PREP(TEMP_MASK, 2) + .name = "set-field", + .expected_reg = REGULAR_REG1, + .expected_set_bits = TEMP_FIELD, + .expected_clr_bits = TEMP_MASK, + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + /* Check FIELD_SET works */ + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(FIELD_SET(REGULAR_REG1, + TEMP_MASK, TEMP_FIELD)) + }, + ), +#undef TEMP_MASK +#undef TEMP_FIELD + }, + { + .name = "conflict-duplicate", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + /* drop: setting same values twice */ + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + ), + }, + { + .name = "conflict-not-disjoint", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + /* drop: bits are not disjoint with previous entries */ + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(CLR(REGULAR_REG1, REG_GENMASK(1, 0))) + }, + ), + }, + { + .name = "conflict-not-disjoint-literal-and-func", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(1) | REG_BIT(2), + .expected_clr_bits = REG_BIT(1) | REG_BIT(2), + .expected_active = BIT(0) | BIT(1), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(FIELD_SET(REGULAR_REG1, + REG_BIT(1) | REG_BIT(2), + REG_BIT(1) | REG_BIT(2))) + }, + /* drop: bits are not disjoint with previous entries */ + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(FIELD_SET_FUNC(REGULAR_REG1, + REG_BIT(2) | REG_BIT(3), + bits_2_3_set)) + }, + ), + }, + { + .name = "conflict-reg-type", + .expected_reg = REGULAR_REG1, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0) | BIT(1) | BIT(2), + .expected_count_sr_entries = 1, + .expected_sr_errors = 2, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("basic-1"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) + }, + /* drop: regular vs MCR */ + { XE_RTP_NAME("basic-2"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(MCR_REG1, REG_BIT(1))) + }, + /* drop: regular vs masked */ + { XE_RTP_NAME("basic-3"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(MASKED_REG1, REG_BIT(0))) + }, + ), + }, + { + .name = "bad-mcr-reg-forced-to-regular", + .expected_reg = REGULAR_REG4, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("bad-mcr-regular-reg"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(BAD_MCR_REG4, REG_BIT(0))) + }, + ), + }, + { + .name = "bad-regular-reg-forced-to-mcr", + .expected_reg = MCR_REG5, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("bad-regular-reg"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(BAD_REGULAR_REG5, REG_BIT(0))) + }, + ), + }, +}; + +static void xe_rtp_process_to_sr_tests(struct kunit *test) +{ + const struct rtp_to_sr_test_case *param = test->param_value; + struct xe_device *xe = test->priv; + struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt; + struct xe_reg_sr *reg_sr = >->reg_sr; + const struct xe_reg_sr_entry *sre, *sr_entry = NULL; + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); + unsigned long idx, count_sr_entries = 0, active = 0; + + xe_reg_sr_init(reg_sr, "xe_rtp_to_sr_tests", xe); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, param->table.n_entries); + xe_rtp_process_to_sr(&ctx, ¶m->table, reg_sr, false); + + xa_for_each(®_sr->xa, idx, sre) { + if (idx == param->expected_reg.addr) + sr_entry = sre; + + count_sr_entries++; + } + + KUNIT_EXPECT_EQ(test, active, param->expected_active); + + KUNIT_EXPECT_EQ(test, count_sr_entries, param->expected_count_sr_entries); + if (count_sr_entries) { + KUNIT_EXPECT_EQ(test, sr_entry->clr_bits, param->expected_clr_bits); + KUNIT_EXPECT_EQ(test, sr_entry->set_bits, param->expected_set_bits); + KUNIT_EXPECT_EQ(test, sr_entry->reg.raw, param->expected_reg.raw); + } else { + KUNIT_EXPECT_NULL(test, sr_entry); + } + + KUNIT_EXPECT_EQ(test, reg_sr->errors, param->expected_sr_errors); +} + +/* + * Entries below follow the logic used with xe_wa_oob.rules: + * 1) Entries with empty name are OR'ed: all entries marked active since the + * last entry with a name + * 2) There are no action associated with rules + */ +static const struct rtp_test_case rtp_cases[] = { + { + .name = "active1", + .expected_active = BIT(0), + .table = XE_RTP_TABLE( + { XE_RTP_NAME("r1"), + XE_RTP_RULES(FUNC(match_yes)), + }, + ), + }, + { + .name = "active2", + .expected_active = BIT(0) | BIT(1), + .table = XE_RTP_TABLE( + { XE_RTP_NAME("r1"), + XE_RTP_RULES(FUNC(match_yes)), + }, + { XE_RTP_NAME("r2"), + XE_RTP_RULES(FUNC(match_yes)), + }, + ), + }, + { + .name = "active-inactive", + .expected_active = BIT(0), + .table = XE_RTP_TABLE( + { XE_RTP_NAME("r1"), + XE_RTP_RULES(FUNC(match_yes)), + }, + { XE_RTP_NAME("r2"), + XE_RTP_RULES(FUNC(match_no)), + }, + ), + }, + { + .name = "inactive-active", + .expected_active = BIT(1), + .table = XE_RTP_TABLE( + { XE_RTP_NAME("r1"), + XE_RTP_RULES(FUNC(match_no)), + }, + { XE_RTP_NAME("r2"), + XE_RTP_RULES(FUNC(match_yes)), + }, + ), + }, + { + .name = "inactive-active-inactive", + .expected_active = BIT(1), + .table = XE_RTP_TABLE( + { XE_RTP_NAME("r1"), + XE_RTP_RULES(FUNC(match_no)), + }, + { XE_RTP_NAME("r2"), + XE_RTP_RULES(FUNC(match_yes)), + }, + { XE_RTP_NAME("r3"), + XE_RTP_RULES(FUNC(match_no)), + }, + ), + }, + { + .name = "inactive-inactive-inactive", + .expected_active = 0, + .table = XE_RTP_TABLE( + { XE_RTP_NAME("r1"), + XE_RTP_RULES(FUNC(match_no)), + }, + { XE_RTP_NAME("r2"), + XE_RTP_RULES(FUNC(match_no)), + }, + { XE_RTP_NAME("r3"), + XE_RTP_RULES(FUNC(match_no)), + }, + ), + }, +}; + +static void xe_rtp_process_tests(struct kunit *test) +{ + const struct rtp_test_case *param = test->param_value; + struct xe_device *xe = test->priv; + struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt; + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); + unsigned long active = 0; + + xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, param->table.n_entries); + xe_rtp_process(&ctx, ¶m->table); + + KUNIT_EXPECT_EQ(test, active, param->expected_active); +} + +static void rtp_rules_desc(const struct rtp_rules_test_case *t, char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(rtp_rules, rtp_rules_cases, rtp_rules_desc); + +static void rtp_to_sr_desc(const struct rtp_to_sr_test_case *t, char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(rtp_to_sr, rtp_to_sr_cases, rtp_to_sr_desc); + +static void rtp_desc(const struct rtp_test_case *t, char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(rtp, rtp_cases, rtp_desc); + +static int xe_rtp_test_init(struct kunit *test) +{ + struct xe_device *xe; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + xe = xe_kunit_helper_alloc_xe_device(test, dev); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe); + + /* Initialize an empty device */ + test->priv = NULL; + ret = xe_pci_fake_device_init(xe); + KUNIT_ASSERT_EQ(test, ret, 0); + + xe->drm.dev = dev; + test->priv = xe; + + kunit_activate_static_stub(test, xe_gt_mcr_check_reg, fake_xe_gt_mcr_check_reg); + + return 0; +} + +static void xe_rtp_test_exit(struct kunit *test) +{ + struct xe_device *xe = test->priv; + + drm_kunit_helper_free_device(test, xe->drm.dev); +} + +static struct kunit_case xe_rtp_tests[] = { + KUNIT_CASE_PARAM(xe_rtp_rules_tests, rtp_rules_gen_params), + KUNIT_CASE_PARAM(xe_rtp_process_to_sr_tests, rtp_to_sr_gen_params), + KUNIT_CASE_PARAM(xe_rtp_process_tests, rtp_gen_params), + {} +}; + +static struct kunit_suite xe_rtp_test_suite = { + .name = "xe_rtp", + .init = xe_rtp_test_init, + .exit = xe_rtp_test_exit, + .test_cases = xe_rtp_tests, +}; + +kunit_test_suite(xe_rtp_test_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.h b/drivers/gpu/drm/xe/tests/xe_rtp_test.h new file mode 100644 index 000000000..26becc1e2 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RTP_TEST_H_ +#define _XE_RTP_TEST_H_ + +#include + +struct xe_device; +struct xe_gt; +struct xe_hw_engine; +struct xe_rtp_rule; + +bool xe_rtp_rule_matches(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules, + int *err); + +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c b/drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c new file mode 100644 index 000000000..c2720b461 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_guc_klv_helpers.h" +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" + +#define TEST_VF VFID(1) + +static int sriov_packet_test_init(struct kunit *test) +{ + struct xe_pci_fake_data fake = { + .sriov_mode = XE_SRIOV_MODE_PF, + .platform = XE_PANTHERLAKE, /* we need MEMIRQ */ + .subplatform = XE_SUBPLATFORM_NONE, + .graphics_verx100 = 3000, + .media_verx100 = 3000, + }; + struct xe_device *xe; + + test->priv = &fake; + xe_kunit_helper_xe_device_test_init(test); + xe = test->priv; + + /* pretend we can support at least VF1 */ + xe->sriov.pf.device_total_vfs = 1; + xe->sriov.pf.driver_max_vfs = 1; + + KUNIT_ASSERT_EQ(test, 0, xe_sriov_init(xe)); + KUNIT_ASSERT_TRUE(test, xe_sriov_pf_migration_supported(xe)); + + return 0; +} + +static void test_descriptor_init(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct xe_sriov_packet **desc; + + /* note: with lock held we should avoid KUNIT_ASSERT() */ + guard(mutex)(pf_migration_mutex(xe, TEST_VF)); + + KUNIT_EXPECT_EQ(test, 0, pf_descriptor_init(xe, TEST_VF)); + desc = pf_pick_descriptor(xe, TEST_VF); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, *desc); + if (!*desc) + return; + KUNIT_EXPECT_NE(test, (*desc)->hdr.version, 0); + KUNIT_EXPECT_EQ(test, (*desc)->hdr.version, XE_SRIOV_PACKET_SUPPORTED_VERSION); + KUNIT_EXPECT_EQ(test, (*desc)->hdr.type, XE_SRIOV_PACKET_TYPE_DESCRIPTOR); + KUNIT_EXPECT_NE(test, (*desc)->hdr.size, 0); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, (*desc)->vaddr); + if (!(*desc)->vaddr) + return; + KUNIT_EXPECT_EQ(test, 0, xe_sriov_packet_process_descriptor(xe, TEST_VF, *desc)); + + switch ((*desc)->hdr.version) { + case 1: + /* v1 is KLV based */ + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((*desc)->hdr.size, sizeof(u32))); + /* v1 has at least DEVID and REVID KLVs */ + KUNIT_EXPECT_LE(test, 2, + xe_guc_klv_count((*desc)->vaddr, + (*desc)->hdr.size / sizeof(u32))); + break; + default: + kunit_mark_skipped(test, "no test code for version %u\n", (*desc)->hdr.version); + return; + } +} + +static struct kunit_case sriov_packet_test_cases[] = { + KUNIT_CASE(test_descriptor_init), + {} +}; + +static struct kunit_suite sriov_packet_suite = { + .name = "sriov_packet", + .test_cases = sriov_packet_test_cases, + .init = sriov_packet_test_init, +}; + +kunit_test_suite(sriov_packet_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_sriov_pf_service_kunit.c b/drivers/gpu/drm/xe/tests/xe_sriov_pf_service_kunit.c new file mode 100644 index 000000000..ba95e29b5 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_sriov_pf_service_kunit.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2024-2025 Intel Corporation + */ + +#include + +#include "xe_device.h" +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" + +static int pf_service_test_init(struct kunit *test) +{ + struct xe_pci_fake_data fake = { + .sriov_mode = XE_SRIOV_MODE_PF, + .platform = XE_TIGERLAKE, /* some random platform */ + .subplatform = XE_SUBPLATFORM_NONE, + }; + struct xe_device *xe; + + test->priv = &fake; + xe_kunit_helper_xe_device_test_init(test); + + xe = test->priv; + KUNIT_ASSERT_EQ(test, xe_sriov_init(xe), 0); + + xe_sriov_pf_service_init(xe); + /* + * sanity check: + * - all supported platforms VF/PF ABI versions must be defined + * - base version can't be newer than latest + */ + KUNIT_ASSERT_NE(test, 0, xe->sriov.pf.service.version.base.major); + KUNIT_ASSERT_NE(test, 0, xe->sriov.pf.service.version.latest.major); + KUNIT_ASSERT_LE(test, xe->sriov.pf.service.version.base.major, + xe->sriov.pf.service.version.latest.major); + if (xe->sriov.pf.service.version.base.major == xe->sriov.pf.service.version.latest.major) + KUNIT_ASSERT_LE(test, xe->sriov.pf.service.version.base.minor, + xe->sriov.pf.service.version.latest.minor); + return 0; +} + +static void pf_negotiate_any(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, VF2PF_HANDSHAKE_MAJOR_ANY, + VF2PF_HANDSHAKE_MINOR_ANY, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.latest.major); + KUNIT_ASSERT_EQ(test, minor, xe->sriov.pf.service.version.latest.minor); +} + +static void pf_negotiate_base_match(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.base.major, + xe->sriov.pf.service.version.base.minor, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.base.major); + KUNIT_ASSERT_EQ(test, minor, xe->sriov.pf.service.version.base.minor); +} + +static void pf_negotiate_base_newer(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.base.major, + xe->sriov.pf.service.version.base.minor + 1, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.base.major); + KUNIT_ASSERT_GE(test, minor, xe->sriov.pf.service.version.base.minor); + if (xe->sriov.pf.service.version.base.major == xe->sriov.pf.service.version.latest.major) + KUNIT_ASSERT_LE(test, minor, xe->sriov.pf.service.version.latest.minor); + else + KUNIT_FAIL(test, "FIXME: don't know how to test multi-version yet!\n"); +} + +static void pf_negotiate_base_next(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.base.major + 1, 0, + &major, &minor)); + KUNIT_ASSERT_GE(test, major, xe->sriov.pf.service.version.base.major); + KUNIT_ASSERT_LE(test, major, xe->sriov.pf.service.version.latest.major); + if (major == xe->sriov.pf.service.version.latest.major) + KUNIT_ASSERT_LE(test, minor, xe->sriov.pf.service.version.latest.minor); + else + KUNIT_FAIL(test, "FIXME: don't know how to test multi-version yet!\n"); +} + +static void pf_negotiate_base_older(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + if (!xe->sriov.pf.service.version.base.minor) + kunit_skip(test, "no older minor\n"); + + KUNIT_ASSERT_NE(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.base.major, + xe->sriov.pf.service.version.base.minor - 1, + &major, &minor)); +} + +static void pf_negotiate_base_prev(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_NE(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.base.major - 1, 1, + &major, &minor)); +} + +static void pf_negotiate_latest_match(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.latest.major, + xe->sriov.pf.service.version.latest.minor, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.latest.major); + KUNIT_ASSERT_EQ(test, minor, xe->sriov.pf.service.version.latest.minor); +} + +static void pf_negotiate_latest_newer(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.latest.major, + xe->sriov.pf.service.version.latest.minor + 1, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.latest.major); + KUNIT_ASSERT_EQ(test, minor, xe->sriov.pf.service.version.latest.minor); +} + +static void pf_negotiate_latest_next(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.latest.major + 1, 0, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.latest.major); + KUNIT_ASSERT_EQ(test, minor, xe->sriov.pf.service.version.latest.minor); +} + +static void pf_negotiate_latest_older(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + if (!xe->sriov.pf.service.version.latest.minor) + kunit_skip(test, "no older minor\n"); + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.latest.major, + xe->sriov.pf.service.version.latest.minor - 1, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.latest.major); + KUNIT_ASSERT_EQ(test, minor, xe->sriov.pf.service.version.latest.minor - 1); +} + +static void pf_negotiate_latest_prev(struct kunit *test) +{ + struct xe_device *xe = test->priv; + u32 major, minor; + + if (xe->sriov.pf.service.version.base.major == xe->sriov.pf.service.version.latest.major) + kunit_skip(test, "no prev major"); + + KUNIT_ASSERT_EQ(test, 0, + pf_negotiate_version(xe, + xe->sriov.pf.service.version.latest.major - 1, + xe->sriov.pf.service.version.base.minor + 1, + &major, &minor)); + KUNIT_ASSERT_EQ(test, major, xe->sriov.pf.service.version.latest.major - 1); + KUNIT_ASSERT_GE(test, major, xe->sriov.pf.service.version.base.major); +} + +static struct kunit_case pf_service_test_cases[] = { + KUNIT_CASE(pf_negotiate_any), + KUNIT_CASE(pf_negotiate_base_match), + KUNIT_CASE(pf_negotiate_base_newer), + KUNIT_CASE(pf_negotiate_base_next), + KUNIT_CASE(pf_negotiate_base_older), + KUNIT_CASE(pf_negotiate_base_prev), + KUNIT_CASE(pf_negotiate_latest_match), + KUNIT_CASE(pf_negotiate_latest_newer), + KUNIT_CASE(pf_negotiate_latest_next), + KUNIT_CASE(pf_negotiate_latest_older), + KUNIT_CASE(pf_negotiate_latest_prev), + {} +}; + +static struct kunit_suite pf_service_suite = { + .name = "pf_service", + .test_cases = pf_service_test_cases, + .init = pf_service_test_init, +}; + +kunit_test_suite(pf_service_suite); diff --git a/drivers/gpu/drm/xe/tests/xe_test.h b/drivers/gpu/drm/xe/tests/xe_test.h new file mode 100644 index 000000000..9c23ad9db --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_test.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TEST_H_ +#define _XE_TEST_H_ + +#include + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include +#include + +/* + * Each test that provides a kunit private test structure, place a test id + * here and point the kunit->priv to an embedded struct xe_test_priv. + */ +enum xe_test_priv_id { + XE_TEST_LIVE_DMA_BUF, + XE_TEST_LIVE_MIGRATE, +}; + +/** + * struct xe_test_priv - Base class for test private info + * @id: enum xe_test_priv_id to identify the subclass. + */ +struct xe_test_priv { + enum xe_test_priv_id id; +}; + +#define XE_TEST_DECLARE(x) x +#define XE_TEST_ONLY(x) unlikely(x) + +/** + * xe_cur_kunit_priv - Obtain the struct xe_test_priv pointed to by + * current->kunit->priv if it exists and is embedded in the expected subclass. + * @id: Id of the expected subclass. + * + * Return: NULL if the process is not a kunit test, and NULL if the + * current kunit->priv pointer is not pointing to an object of the expected + * subclass. A pointer to the embedded struct xe_test_priv otherwise. + */ +static inline struct xe_test_priv * +xe_cur_kunit_priv(enum xe_test_priv_id id) +{ + struct xe_test_priv *priv; + + if (!kunit_get_current_test()) + return NULL; + + priv = kunit_get_current_test()->priv; + return priv->id == id ? priv : NULL; +} + +#else /* if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) */ + +#define XE_TEST_DECLARE(x) +#define XE_TEST_ONLY(x) 0 +#define xe_cur_kunit_priv(_id) NULL + +#endif +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_test_mod.c b/drivers/gpu/drm/xe/tests/xe_test_mod.c new file mode 100644 index 000000000..93081bcf2 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_test_mod.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2023 Intel Corporation + */ +#include + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("xe kunit tests"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c new file mode 100644 index 000000000..21601e9df --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include + +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_kunit_helpers.h" +#include "xe_pci_test.h" +#include "xe_reg_sr.h" +#include "xe_tuning.h" +#include "xe_wa.h" + +static int xe_wa_test_init(struct kunit *test) +{ + const struct xe_pci_fake_data *param = test->param_value; + struct xe_pci_fake_data data = *param; + struct device *dev; + struct xe_device *xe; + struct xe_gt *gt; + int id; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + xe = xe_kunit_helper_alloc_xe_device(test, dev); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe); + + test->priv = &data; + ret = xe_pci_fake_device_init(xe); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* Needed for sanitize_mcr(). */ + for_each_gt(gt, xe, id) { + xe_gt_mcr_init_early(gt); + xe_gt_mmio_init(gt); + } + + /* TODO: init hw engines for engine/LRC WAs */ + xe->drm.dev = dev; + test->priv = xe; + + return 0; +} + +static void xe_wa_gt(struct kunit *test) +{ + struct xe_device *xe = test->priv; + struct xe_gt *gt; + int id; + + for_each_gt(gt, xe, id) { + xe_reg_sr_init(>->reg_sr, "GT", xe); + + xe_wa_process_gt(gt); + xe_tuning_process_gt(gt); + + KUNIT_EXPECT_EQ(test, gt->reg_sr.errors, 0); + } +} + +static struct kunit_case xe_wa_tests[] = { + KUNIT_CASE_PARAM(xe_wa_gt, xe_pci_fake_data_gen_params), + {} +}; + +static struct kunit_suite xe_rtp_test_suite = { + .name = "xe_wa", + .init = xe_wa_test_init, + .test_cases = xe_wa_tests, +}; + +kunit_test_suite(xe_rtp_test_suite); diff --git a/drivers/gpu/drm/xe/xe_amc.c b/drivers/gpu/drm/xe/xe_amc.c new file mode 100644 index 000000000..8ecadee6e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_amc.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "regs/xe_i2c_regs.h" + +#include "xe_amc.h" +#include "xe_device.h" +#include "xe_i2c.h" +#include "xe_mmio.h" + +/** + * DOC: Add-In Management Controller (AMC) + * + * Handler for the SMBus Alerts from the AMC. All the alerts from AMC will cause + * the device to be declared wedged. + */ + +#define AMC_COMMAND 0x0f +#define AMC_GPU_I2C_ADDR 0x8f +#define AMC_VERSION_V1 0x01 +#define AMC_DESTINATION_ID 12 +#define AMC_SOURCE_ID 8 +#define AMC_FLAGS 0xc8 + +#define AMC_MSG_TYPE 0x7e +#define AMC_GET_ALERT_REASON 0x01 + +enum xe_amc_alert { + AMC_ALERT_UNKNOWN, + AMC_ALERT_FW_DOWNLOAD, + AMC_ALERT_THERMAL_TRIP, + AMC_ALERT_OOB_REQUEST, + AMC_ALERT_OOB_RESET, + AMC_ALERT_CATERR, +}; + +static const char * const amc_alert[] = { + [AMC_ALERT_FW_DOWNLOAD] = "Firmware Download", + [AMC_ALERT_THERMAL_TRIP] = "Thermal Trip", + [AMC_ALERT_OOB_REQUEST] = "OOB Request", + [AMC_ALERT_OOB_RESET] = "OOB Reset", + [AMC_ALERT_CATERR] = "Catastrophic", +}; + +struct xe_amc { + struct xe_i2c *i2c; + struct work_struct work; +}; + +struct amc_header { + u8 command; + u8 len; + u8 address; + u8 version; + u8 destination; + u8 source; + u8 flags; +} __packed; + +struct amc_message { + u8 type; + u16 vendor; + u8 command; +} __packed; + +struct amc_request { + struct amc_header header; + struct amc_message message; + u32 reserved; +} __packed; + +struct amc_response { + struct amc_header header; + struct amc_message message; + u8 error; + u8 value; +} __packed; + +static const struct amc_request amc_get_alert_reason = { + .header = { + .command = AMC_COMMAND, + .len = sizeof(struct amc_request) - 2, + .address = AMC_GPU_I2C_ADDR, + .version = AMC_VERSION_V1, + .destination = AMC_DESTINATION_ID, + .source = AMC_SOURCE_ID, + .flags = AMC_FLAGS, + }, + .message = { + .type = AMC_MSG_TYPE, + .vendor = htons(PCI_VENDOR_ID_INTEL), + .command = AMC_GET_ALERT_REASON, + }, +}; + +static void xe_amc_work(struct work_struct *work) +{ + const struct amc_request *request = &amc_get_alert_reason; + struct xe_amc *amc = from_work(amc, work, work); + u8 alert_reason = AMC_ALERT_UNKNOWN; + struct amc_response response; + struct i2c_client *client; + int ret; + + client = amc->i2c->client[XE_I2C_CLIENT_AMC]; + if (IS_ERR_OR_NULL(client)) + goto out_reassert_interrupt; + + ret = i2c_master_send(client, (u8 *)request, sizeof(*request)); + if (ret < 0) { + dev_err(&client->dev, "failed to send request (%d)\n", ret); + goto out_reassert_interrupt; + } + + /* AMC needs 20ms to generate the response. */ + fsleep(20 * USEC_PER_MSEC); + + ret = i2c_master_recv(client, (u8 *)&response, sizeof(response)); + if (ret < 0) { + dev_err(&client->dev, "failed to read response (%d)\n", ret); + goto out_reassert_interrupt; + } + + if (!response.header.len) { + dev_err(&client->dev, "empty response from AMC\n"); + goto out_reassert_interrupt; + } + + if (memcmp(&response.message, &request->message, sizeof(struct amc_message))) { + dev_err(&client->dev, "response does not match the request\n"); + goto out_reassert_interrupt; + } + + if (response.error) { + dev_err(&client->dev, "AMC error 0x%02x\n", response.error); + goto out_reassert_interrupt; + } + + alert_reason = response.value; + dev_dbg(&client->dev, "Alert reason: %d\n", alert_reason); + +out_reassert_interrupt: + xe_mmio_rmw32(amc->i2c->mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); + + switch (alert_reason) { + case AMC_ALERT_FW_DOWNLOAD: + case AMC_ALERT_THERMAL_TRIP: + case AMC_ALERT_OOB_REQUEST: + case AMC_ALERT_OOB_RESET: + case AMC_ALERT_CATERR: + dev_warn(amc->i2c->drm_dev, "AMC Alert: %s\n", amc_alert[alert_reason]); + xe_device_declare_wedged(i2c_client_to_xe_device(client)); + break; + default: + dev_warn(amc->i2c->drm_dev, "unknown AMC alert: %d\n", alert_reason); + break; + } +} + +void xe_amc_handle_alert(struct xe_i2c *i2c) +{ + queue_work(system_long_wq, &i2c->amc->work); +} + +int xe_amc_init(struct xe_i2c *i2c) +{ + struct xe_amc *amc; + + amc = kzalloc(sizeof(*amc), GFP_KERNEL); + if (!amc) + return -ENOMEM; + + INIT_WORK(&amc->work, xe_amc_work); + i2c->amc = amc; + amc->i2c = i2c; + + return 0; +} + +void xe_amc_exit(struct xe_i2c *i2c) +{ + if (i2c->amc) { + cancel_work_sync(&i2c->amc->work); + kfree(i2c->amc); + } +} diff --git a/drivers/gpu/drm/xe/xe_amc.h b/drivers/gpu/drm/xe/xe_amc.h new file mode 100644 index 000000000..b1d5311fe --- /dev/null +++ b/drivers/gpu/drm/xe/xe_amc.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XE_AMC_H_ +#define _XE_AMC_H_ + +#include + +#include "xe_device.h" + +struct xe_i2c; + +static inline struct xe_device *i2c_adapter_to_xe_device(struct i2c_adapter *adapter) +{ + return kdev_to_xe_device(adapter->dev.parent->parent); +} + +static inline struct xe_device *i2c_client_to_xe_device(struct i2c_client *client) +{ + return i2c_adapter_to_xe_device(client->adapter); +} + +int xe_amc_init(struct xe_i2c *i2c); +void xe_amc_exit(struct xe_i2c *i2c); +void xe_amc_handle_alert(struct xe_i2c *i2c); + +#endif /* _XE_AMC_H_ */ diff --git a/drivers/gpu/drm/xe/xe_args.h b/drivers/gpu/drm/xe/xe_args.h new file mode 100644 index 000000000..f550b5e3b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_args.h @@ -0,0 +1,170 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_ARGS_H_ +#define _XE_ARGS_H_ + +#include + +/* + * Why don't the following macros have the XE prefix? + * + * Once we find more potential users outside of the Xe driver, we plan to move + * all of the following macros unchanged to linux/args.h. + */ + +/** + * CALL_ARGS - Invoke a macro, but allow parameters to be expanded beforehand. + * @f: name of the macro to invoke + * @args: arguments for the macro + * + * This macro allows calling macros which names might generated or we want to + * make sure it's arguments will be correctly expanded. + * + * Example: + * + * #define foo X,Y,Z,Q + * #define bar COUNT_ARGS(foo) + * #define buz CALL_ARGS(COUNT_ARGS, foo) + * + * With above definitions bar expands to 1 while buz expands to 4. + */ +#define CALL_ARGS(f, args...) __CALL_ARGS(f, args) +#define __CALL_ARGS(f, args...) f(args) + +/** + * DROP_FIRST_ARG - Returns all arguments except the first one. + * @args: arguments + * + * This helper macro allows manipulation the argument list before passing it + * to the next level macro. + * + * Example: + * + * #define foo X,Y,Z,Q + * #define bar CALL_ARGS(COUNT_ARGS, DROP_FIRST_ARG(foo)) + * + * With above definitions bar expands to 3. + */ +#define DROP_FIRST_ARG(args...) __DROP_FIRST_ARG(args) +#define __DROP_FIRST_ARG(a, b...) b + +/** + * FIRST_ARG - Returns the first argument. + * @args: arguments + * + * This helper macro allows manipulation the argument list before passing it + * to the next level macro. + * + * Example: + * + * #define foo X,Y,Z,Q + * #define bar FIRST_ARG(foo) + * + * With above definitions bar expands to X. + */ +#define FIRST_ARG(args...) __FIRST_ARG(args) +#define __FIRST_ARG(a, b...) a + +/** + * LAST_ARG - Returns the last argument. + * @args: arguments + * + * This helper macro allows manipulation the argument list before passing it + * to the next level macro. + * + * Like COUNT_ARGS() this macro works up to 12 arguments. + * + * Example: + * + * #define foo X,Y,Z,Q + * #define bar LAST_ARG(foo) + * + * With above definitions bar expands to Q. + */ +#define LAST_ARG(args...) __LAST_ARG(args) +#define __LAST_ARG(args...) PICK_ARG(COUNT_ARGS(args), args) + +/** + * PICK_ARG - Returns the n-th argument. + * @n: argument number to be returned + * @args: arguments + * + * This helper macro allows manipulation the argument list before passing it + * to the next level macro. + * + * Like COUNT_ARGS() this macro supports n up to 12. + * Specialized macros PICK_ARG1() to PICK_ARG12() are also available. + * + * Example: + * + * #define foo X,Y,Z,Q + * #define bar PICK_ARG(2, foo) + * #define buz PICK_ARG3(foo) + * + * With above definitions bar expands to Y and buz expands to Z. + */ +#define PICK_ARG(n, args...) __PICK_ARG(n, args) +#define __PICK_ARG(n, args...) CALL_ARGS(CONCATENATE(PICK_ARG, n), args) +#define PICK_ARG1(args...) FIRST_ARG(args) +#define PICK_ARG2(args...) PICK_ARG1(DROP_FIRST_ARG(args)) +#define PICK_ARG3(args...) PICK_ARG2(DROP_FIRST_ARG(args)) +#define PICK_ARG4(args...) PICK_ARG3(DROP_FIRST_ARG(args)) +#define PICK_ARG5(args...) PICK_ARG4(DROP_FIRST_ARG(args)) +#define PICK_ARG6(args...) PICK_ARG5(DROP_FIRST_ARG(args)) +#define PICK_ARG7(args...) PICK_ARG6(DROP_FIRST_ARG(args)) +#define PICK_ARG8(args...) PICK_ARG7(DROP_FIRST_ARG(args)) +#define PICK_ARG9(args...) PICK_ARG8(DROP_FIRST_ARG(args)) +#define PICK_ARG10(args...) PICK_ARG9(DROP_FIRST_ARG(args)) +#define PICK_ARG11(args...) PICK_ARG10(DROP_FIRST_ARG(args)) +#define PICK_ARG12(args...) PICK_ARG11(DROP_FIRST_ARG(args)) + +/** + * IF_ARGS() - Make selection based on optional argument list. + * @then: token to return if arguments are present + * @else: token to return if arguments are empty + * @...: arguments to check (optional) + * + * This macro allows to select a token based on the presence of the argument list. + * + * Example: + * + * #define foo X, Y + * #define bar IF_ARGS(Z, Q, foo) + * #define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo))) + * + * With above definitions bar expands to Z while buz expands to Q. + */ +#if defined(CONFIG_CC_IS_CLANG) || GCC_VERSION >= 100100 +#define IF_ARGS(then, else, ...) FIRST_ARG(__VA_OPT__(then,) else) +#else +#define IF_ARGS(then, else, ...) _IF_ARGS(then, else, CALL_ARGS(FIRST_ARG, __VA_ARGS__)) +#define _IF_ARGS(then, else, ...) __IF_ARGS(then, else, CALL_ARGS(COUNT_ARGS, __VA_ARGS__)) +#define __IF_ARGS(then, else, n) ___IF_ARGS(then, else, CALL_ARGS(CONCATENATE, ___IF_ARG, n)) +#define ___IF_ARGS(then, else, if) CALL_ARGS(if, then, else) +#define ___IF_ARG1(then, else) then +#define ___IF_ARG0(then, else) else +#endif + +/** + * ARGS_SEP_COMMA - Definition of a comma character. + * + * This definition can be used in cases where any intermediate macro expects + * fixed number of arguments, but we want to pass more arguments which can + * be properly evaluated only by the next level macro. + * + * Example: + * + * #define foo(f) f(X) f(Y) f(Z) f(Q) + * #define bar DROP_FIRST_ARG(foo(ARGS_SEP_COMMA __stringify)) + * #define buz CALL_ARGS(COUNT_ARGS, DROP_FIRST_ARG(foo(ARGS_SEP_COMMA))) + * + * With above definitions bar expands to + * "X", "Y", "Z", "Q" + * and buz expands to 4. + */ +#define ARGS_SEP_COMMA , + +#endif diff --git a/drivers/gpu/drm/xe/xe_assert.h b/drivers/gpu/drm/xe/xe_assert.h new file mode 100644 index 000000000..a818eaa05 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_assert.h @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_ASSERT_H_ +#define _XE_ASSERT_H_ + +#include + +#include + +#include "xe_gt_types.h" +#include "xe_step.h" +#include "xe_vram.h" + +/** + * DOC: Xe Asserts + * + * While Xe driver aims to be simpler than legacy i915 driver it is still + * complex enough that some changes introduced while adding new functionality + * could break the existing code. + * + * Adding &drm_WARN or &drm_err to catch unwanted programming usage could lead + * to undesired increased driver footprint and may impact production driver + * performance as this additional code will be always present. + * + * To allow annotate functions with additional detailed debug checks to assert + * that all prerequisites are satisfied, without worrying about footprint or + * performance penalty on production builds where all potential misuses + * introduced during code integration were already fixed, we introduce family + * of Xe assert macros that try to follow classic assert() utility: + * + * * xe_assert() + * * xe_tile_assert() + * * xe_gt_assert() + * + * These macros are implemented on top of &drm_WARN, but unlikely to the origin, + * warning is triggered when provided condition is false. Additionally all above + * assert macros cannot be used in expressions or as a condition, since + * underlying code will be compiled out on non-debug builds. + * + * Note that these macros are not intended for use to cover known gaps in the + * implementation; for such cases use regular &drm_WARN or &drm_err and provide + * valid safe fallback. + * + * Also in cases where performance or footprint is not an issue, developers + * should continue to use the regular &drm_WARN or &drm_err to ensure that bug + * reports from production builds will contain meaningful diagnostics data. + * + * Below code shows how asserts could help in debug to catch unplanned use:: + * + * static void one_igfx(struct xe_device *xe) + * { + * xe_assert(xe, xe->info.is_dgfx == false); + * xe_assert(xe, xe->info.tile_count == 1); + * } + * + * static void two_dgfx(struct xe_device *xe) + * { + * xe_assert(xe, xe->info.is_dgfx); + * xe_assert(xe, xe->info.tile_count == 2); + * } + * + * void foo(struct xe_device *xe) + * { + * if (xe->info.dgfx) + * return two_dgfx(xe); + * return one_igfx(xe); + * } + * + * void bar(struct xe_device *xe) + * { + * if (drm_WARN_ON(xe->drm, xe->info.tile_count > 2)) + * return; + * + * if (xe->info.tile_count == 2) + * return two_dgfx(xe); + * return one_igfx(xe); + * } + */ + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define __xe_assert_msg(xe, condition, msg, arg...) ({ \ + (void)drm_WARN(&(xe)->drm, !(condition), "Assertion `%s` failed!\n" msg, \ + __stringify(condition), ## arg); \ +}) +#else +#define __xe_assert_msg(xe, condition, msg, arg...) ({ \ + typecheck(const struct xe_device *, xe); \ + BUILD_BUG_ON_INVALID(condition); \ +}) +#endif + +/** + * xe_assert - warn if condition is false when debugging. + * @xe: the &struct xe_device pointer to which &condition applies + * @condition: condition to check + * + * xe_assert() uses &drm_WARN to emit a warning and print additional information + * that could be read from the &xe pointer if provided &condition is false. + * + * Contrary to &drm_WARN, xe_assert() is effective only on debug builds + * (&CONFIG_DRM_XE_DEBUG must be enabled) and cannot be used in expressions + * or as a condition. + * + * See `Xe Asserts`_ for general usage guidelines. + */ +#define xe_assert(xe, condition) xe_assert_msg((xe), condition, "") +#define xe_assert_msg(xe, condition, msg, arg...) ({ \ + const struct xe_device *__xe = (xe); \ + __xe_assert_msg(__xe, condition, \ + "platform: %s subplatform: %d\n" \ + "graphics: %s %u.%02u step %s\n" \ + "media: %s %u.%02u step %s\n" \ + msg, \ + __xe->info.platform_name, __xe->info.subplatform, \ + __xe->info.graphics_name, \ + __xe->info.graphics_verx100 / 100, \ + __xe->info.graphics_verx100 % 100, \ + xe_step_name(__xe->info.step.graphics), \ + __xe->info.media_name, \ + __xe->info.media_verx100 / 100, \ + __xe->info.media_verx100 % 100, \ + xe_step_name(__xe->info.step.media), \ + ## arg); \ +}) + +/** + * xe_tile_assert - warn if condition is false when debugging. + * @tile: the &struct xe_tile pointer to which &condition applies + * @condition: condition to check + * + * xe_tile_assert() uses &drm_WARN to emit a warning and print additional + * information that could be read from the &tile pointer if provided &condition + * is false. + * + * Contrary to &drm_WARN, xe_tile_assert() is effective only on debug builds + * (&CONFIG_DRM_XE_DEBUG must be enabled) and cannot be used in expressions + * or as a condition. + * + * See `Xe Asserts`_ for general usage guidelines. + */ +#define xe_tile_assert(tile, condition) xe_tile_assert_msg((tile), condition, "") +#define xe_tile_assert_msg(tile, condition, msg, arg...) ({ \ + const struct xe_tile *__tile = (tile); \ + char __buf[10] __maybe_unused; \ + xe_assert_msg(tile_to_xe(__tile), condition, "tile: %u VRAM %s\n" msg, \ + __tile->id, ({ string_get_size( \ + xe_vram_region_actual_physical_size(__tile->mem.vram), 1, \ + STRING_UNITS_2, __buf, sizeof(__buf)); __buf; }), ## arg); \ +}) + +/** + * xe_gt_assert - warn if condition is false when debugging. + * @gt: the &struct xe_gt pointer to which &condition applies + * @condition: condition to check + * + * xe_gt_assert() uses &drm_WARN to emit a warning and print additional + * information that could be safetely read from the > pointer if provided + * &condition is false. + * + * Contrary to &drm_WARN, xe_gt_assert() is effective only on debug builds + * (&CONFIG_DRM_XE_DEBUG must be enabled) and cannot be used in expressions + * or as a condition. + * + * See `Xe Asserts`_ for general usage guidelines. + */ +#define xe_gt_assert(gt, condition) xe_gt_assert_msg((gt), condition, "") +#define xe_gt_assert_msg(gt, condition, msg, arg...) ({ \ + const struct xe_gt *__gt = (gt); \ + xe_tile_assert_msg(gt_to_tile(__gt), condition, "GT: %u type %d\n" msg, \ + __gt->info.id, __gt->info.type, ## arg); \ +}) + +#endif diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c new file mode 100644 index 000000000..b896b6f66 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bb.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bb.h" + +#include "instructions/xe_mi_commands.h" +#include "xe_assert.h" +#include "xe_device_types.h" +#include "xe_exec_queue_types.h" +#include "xe_gt.h" +#include "xe_sa.h" +#include "xe_sched_job.h" +#include "xe_vm_types.h" + +static int bb_prefetch(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (GRAPHICS_VERx100(xe) >= 1250 && xe_gt_is_main_type(gt)) + /* + * RCS and CCS require 1K, although other engines would be + * okay with 512. + */ + return SZ_1K; + else + return SZ_512; +} + +struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bb *bb = kmalloc_obj(*bb); + int err; + + if (!bb) + return ERR_PTR(-ENOMEM); + + /* + * We need to allocate space for the requested number of dwords, + * one additional MI_BATCH_BUFFER_END dword, and additional buffer + * space to accommodate the platform-specific hardware prefetch + * requirements. + */ + bb->bo = xe_sa_bo_new(!usm ? tile->mem.kernel_bb_pool : gt->usm.bb_pool, + 4 * (dwords + 1) + bb_prefetch(gt)); + if (IS_ERR(bb->bo)) { + err = PTR_ERR(bb->bo); + goto err; + } + + bb->cs = xe_sa_bo_cpu_addr(bb->bo); + bb->len = 0; + + return bb; +err: + kfree(bb); + return ERR_PTR(err); +} + +/** + * xe_bb_alloc() - Allocate a new batch buffer structure + * @gt: the &xe_gt + * + * Allocates and initializes a new xe_bb structure with an associated + * uninitialized suballoc object. + * + * Returns: Batch buffer structure or an ERR_PTR(-ENOMEM). + */ +struct xe_bb *xe_bb_alloc(struct xe_gt *gt) +{ + struct xe_bb *bb = kmalloc_obj(*bb); + int err; + + if (!bb) + return ERR_PTR(-ENOMEM); + + bb->bo = xe_sa_bo_alloc(GFP_KERNEL); + if (IS_ERR(bb->bo)) { + err = PTR_ERR(bb->bo); + goto err; + } + + return bb; + +err: + kfree(bb); + return ERR_PTR(err); +} + +/** + * xe_bb_init() - Initialize a batch buffer with memory from a sub-allocator pool + * @bb: Batch buffer structure to initialize + * @bb_pool: Suballoc memory pool to allocate from + * @dwords: Number of dwords to be allocated + * + * Initializes the batch buffer by allocating memory from the specified + * suballoc pool. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_bb_init(struct xe_bb *bb, struct xe_sa_manager *bb_pool, u32 dwords) +{ + int err; + + /* + * We need to allocate space for the requested number of dwords & + * one additional MI_BATCH_BUFFER_END dword. Since the whole SA + * is submitted to HW, we need to make sure that the last instruction + * is not over written when the last chunk of SA is allocated for BB. + * So, this extra DW acts as a guard here. + */ + err = xe_sa_bo_init(bb_pool, bb->bo, 4 * (dwords + 1)); + if (err) + return err; + + bb->cs = xe_sa_bo_cpu_addr(bb->bo); + bb->len = 0; + + return 0; +} + +static struct xe_sched_job * +__xe_bb_create_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 *addr) +{ + u32 size = drm_suballoc_size(bb->bo); + + if (bb->len == 0 || bb->cs[bb->len - 1] != MI_BATCH_BUFFER_END) + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + + xe_gt_assert(q->gt, bb->len * 4 + bb_prefetch(q->gt) <= size); + + xe_sa_bo_flush_write(bb->bo); + + return xe_sched_job_create(q, addr); +} + +struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q, + struct xe_bb *bb, + u64 batch_base_ofs, + u32 second_idx) +{ + u64 addr[2] = { + batch_base_ofs + drm_suballoc_soffset(bb->bo), + batch_base_ofs + drm_suballoc_soffset(bb->bo) + + 4 * second_idx, + }; + + xe_gt_assert(q->gt, second_idx <= bb->len); + xe_gt_assert(q->gt, xe_sched_job_is_migration(q)); + xe_gt_assert(q->gt, q->width == 1); + + return __xe_bb_create_job(q, bb, addr); +} + +struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q, + struct xe_bb *bb) +{ + u64 addr = xe_sa_bo_gpu_addr(bb->bo); + + xe_gt_assert(q->gt, !xe_sched_job_is_migration(q)); + xe_gt_assert(q->gt, q->width == 1); + return __xe_bb_create_job(q, bb, &addr); +} + +void xe_bb_free(struct xe_bb *bb, struct dma_fence *fence) +{ + if (!bb) + return; + + xe_sa_bo_free(bb->bo, fence); + kfree(bb); +} diff --git a/drivers/gpu/drm/xe/xe_bb.h b/drivers/gpu/drm/xe/xe_bb.h new file mode 100644 index 000000000..231870b24 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bb.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BB_H_ +#define _XE_BB_H_ + +#include "xe_bb_types.h" + +struct dma_fence; + +struct xe_gt; +struct xe_exec_queue; +struct xe_sa_manager; +struct xe_sched_job; + +struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm); +struct xe_bb *xe_bb_alloc(struct xe_gt *gt); +int xe_bb_init(struct xe_bb *bb, struct xe_sa_manager *bb_pool, u32 dwords); +struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q, + struct xe_bb *bb); +struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q, + struct xe_bb *bb, u64 batch_ofs, + u32 second_idx); +void xe_bb_free(struct xe_bb *bb, struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/xe/xe_bb_types.h b/drivers/gpu/drm/xe/xe_bb_types.h new file mode 100644 index 000000000..b7d30308c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bb_types.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BB_TYPES_H_ +#define _XE_BB_TYPES_H_ + +#include + +struct drm_suballoc; + +struct xe_bb { + struct drm_suballoc *bo; + + u32 *cs; + u32 len; /* in dwords */ +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c new file mode 100644 index 000000000..dde309821 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -0,0 +1,4092 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_bo.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "xe_device.h" +#include "xe_dma_buf.h" +#include "xe_drm_client.h" +#include "xe_ggtt.h" +#include "xe_map.h" +#include "xe_migrate.h" +#include "xe_pat.h" +#include "xe_pm.h" +#include "xe_preempt_fence.h" +#include "xe_pxp.h" +#include "xe_res_cursor.h" +#include "xe_shrinker.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_tile.h" +#include "xe_trace_bo.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_vm.h" +#include "xe_vram_types.h" + +const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES] = { + [XE_PL_SYSTEM] = "system", + [XE_PL_TT] = "gtt", + [XE_PL_VRAM0] = "vram0", + [XE_PL_VRAM1] = "vram1", + [XE_PL_STOLEN] = "stolen" +}; + +static const struct ttm_place sys_placement_flags = { + .fpfn = 0, + .lpfn = 0, + .mem_type = XE_PL_SYSTEM, + .flags = 0, +}; + +static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_placement_flags, +}; + +static struct ttm_placement purge_placement; + +static const struct ttm_place tt_placement_flags[] = { + { + .fpfn = 0, + .lpfn = 0, + .mem_type = XE_PL_TT, + .flags = TTM_PL_FLAG_DESIRED, + }, + { + .fpfn = 0, + .lpfn = 0, + .mem_type = XE_PL_SYSTEM, + .flags = TTM_PL_FLAG_FALLBACK, + } +}; + +static struct ttm_placement tt_placement = { + .num_placement = 2, + .placement = tt_placement_flags, +}; + +#define for_each_set_bo_vram_flag(bit__, bo_flags__) \ + for (unsigned int __bit_tmp = BIT(0); __bit_tmp <= XE_BO_FLAG_VRAM_MASK; __bit_tmp <<= 1) \ + for_each_if(((bit__) = __bit_tmp) & (bo_flags__) & XE_BO_FLAG_VRAM_MASK) + +bool mem_type_is_vram(u32 mem_type) +{ + return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN; +} + +static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res) +{ + return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe); +} + +static bool resource_is_vram(struct ttm_resource *res) +{ + return mem_type_is_vram(res->mem_type); +} + +bool xe_bo_is_vram(struct xe_bo *bo) +{ + return resource_is_vram(bo->ttm.resource) || + resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource); +} + +bool xe_bo_is_stolen(struct xe_bo *bo) +{ + return bo->ttm.resource->mem_type == XE_PL_STOLEN; +} + +/** + * xe_bo_has_single_placement - check if BO is placed only in one memory location + * @bo: The BO + * + * This function checks whether a given BO is placed in only one memory location. + * + * Returns: true if the BO is placed in a single memory location, false otherwise. + * + */ +bool xe_bo_has_single_placement(struct xe_bo *bo) +{ + return bo->placement.num_placement == 1; +} + +/** + * xe_bo_is_stolen_devmem - check if BO is of stolen type accessed via PCI BAR + * @bo: The BO + * + * The stolen memory is accessed through the PCI BAR for both DGFX and some + * integrated platforms that have a dedicated bit in the PTE for devmem (DM). + * + * Returns: true if it's stolen memory accessed via PCI BAR, false otherwise. + */ +bool xe_bo_is_stolen_devmem(struct xe_bo *bo) +{ + return xe_bo_is_stolen(bo) && + GRAPHICS_VERx100(xe_bo_device(bo)) >= 1270; +} + +/** + * xe_bo_is_vm_bound - check if BO has any mappings through VM_BIND + * @bo: The BO + * + * Check if a given bo is bound through VM_BIND. This requires the + * reservation lock for the BO to be held. + * + * Returns: boolean + */ +bool xe_bo_is_vm_bound(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + return !list_empty(&bo->ttm.base.gpuva.list); +} + +static bool xe_bo_is_user(struct xe_bo *bo) +{ + return bo->flags & XE_BO_FLAG_USER; +} + +static struct xe_migrate * +mem_type_to_migrate(struct xe_device *xe, u32 mem_type) +{ + struct xe_tile *tile; + + xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type)); + tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)]; + return tile->migrate; +} + +static void try_add_system(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags, u32 *c) +{ + if (bo_flags & XE_BO_FLAG_SYSTEM) { + xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); + + bo->placements[*c] = (struct ttm_place) { + .mem_type = XE_PL_TT, + .flags = (bo_flags & XE_BO_FLAG_VRAM_MASK) ? + TTM_PL_FLAG_FALLBACK : 0, + }; + *c += 1; + } +} + +static bool force_contiguous(u32 bo_flags) +{ + if (bo_flags & XE_BO_FLAG_STOLEN) + return true; /* users expect this */ + else if (bo_flags & XE_BO_FLAG_PINNED && + !(bo_flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) + return true; /* needs vmap */ + else if (bo_flags & XE_BO_FLAG_CPU_ADDR_MIRROR) + return true; + + /* + * For eviction / restore on suspend / resume objects pinned in VRAM + * must be contiguous, also only contiguous BOs support xe_bo_vmap. + */ + return bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS && + bo_flags & XE_BO_FLAG_PINNED; +} + +static u8 vram_bo_flag_to_tile_id(struct xe_device *xe, u32 vram_bo_flag) +{ + xe_assert(xe, vram_bo_flag & XE_BO_FLAG_VRAM_MASK); + xe_assert(xe, (vram_bo_flag & (vram_bo_flag - 1)) == 0); + + return __ffs(vram_bo_flag >> (__ffs(XE_BO_FLAG_VRAM0) - 1)) - 1; +} + +static u32 bo_vram_flags_to_vram_placement(struct xe_device *xe, u32 bo_flags, u32 vram_flag, + enum ttm_bo_type type) +{ + u8 tile_id = vram_bo_flag_to_tile_id(xe, vram_flag); + + xe_assert(xe, tile_id < xe->info.tile_count); + + if (type == ttm_bo_type_kernel && !(bo_flags & XE_BO_FLAG_FORCE_USER_VRAM)) + return xe->tiles[tile_id].mem.kernel_vram->placement; + else + return xe->tiles[tile_id].mem.vram->placement; +} + +static void add_vram(struct xe_device *xe, struct xe_bo *bo, + struct ttm_place *places, u32 bo_flags, u32 mem_type, u32 *c) +{ + struct ttm_place place = { .mem_type = mem_type }; + struct ttm_resource_manager *mgr = ttm_manager_type(&xe->ttm, mem_type); + struct xe_ttm_vram_mgr *vram_mgr = to_xe_ttm_vram_mgr(mgr); + + struct xe_vram_region *vram; + u64 io_size; + + xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); + + vram = container_of(vram_mgr, struct xe_vram_region, ttm); + xe_assert(xe, vram && vram->usable_size); + io_size = vram->io_size; + + if (force_contiguous(bo_flags)) + place.flags |= TTM_PL_FLAG_CONTIGUOUS; + + if (io_size < vram->usable_size) { + if (bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) { + place.fpfn = 0; + place.lpfn = io_size >> PAGE_SHIFT; + } else { + place.flags |= TTM_PL_FLAG_TOPDOWN; + } + } + places[*c] = place; + *c += 1; +} + +static void try_add_vram(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags, enum ttm_bo_type type, u32 *c) +{ + u32 vram_flag; + + for_each_set_bo_vram_flag(vram_flag, bo_flags) { + u32 pl = bo_vram_flags_to_vram_placement(xe, bo_flags, vram_flag, type); + + add_vram(xe, bo, bo->placements, bo_flags, pl, c); + } +} + +static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags, u32 *c) +{ + if (bo_flags & XE_BO_FLAG_STOLEN) { + xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); + + bo->placements[*c] = (struct ttm_place) { + .mem_type = XE_PL_STOLEN, + .flags = force_contiguous(bo_flags) ? + TTM_PL_FLAG_CONTIGUOUS : 0, + }; + *c += 1; + } +} + +static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags, enum ttm_bo_type type) +{ + u32 c = 0; + + try_add_vram(xe, bo, bo_flags, type, &c); + try_add_system(xe, bo, bo_flags, &c); + try_add_stolen(xe, bo, bo_flags, &c); + + if (!c) + return -EINVAL; + + bo->placement = (struct ttm_placement) { + .num_placement = c, + .placement = bo->placements, + }; + + return 0; +} + +int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags, enum ttm_bo_type type) +{ + xe_bo_assert_held(bo); + return __xe_bo_placement_for_flags(xe, bo, bo_flags, type); +} + +static void xe_evict_flags(struct ttm_buffer_object *tbo, + struct ttm_placement *placement) +{ + struct xe_device *xe = container_of(tbo->bdev, typeof(*xe), ttm); + bool device_unplugged = drm_dev_is_unplugged(&xe->drm); + struct xe_bo *bo; + + if (!xe_bo_is_xe_bo(tbo)) { + /* Don't handle scatter gather BOs */ + if (tbo->type == ttm_bo_type_sg) { + placement->num_placement = 0; + return; + } + + *placement = device_unplugged ? purge_placement : sys_placement; + return; + } + + bo = ttm_to_xe_bo(tbo); + if (bo->flags & XE_BO_FLAG_CPU_ADDR_MIRROR) { + *placement = sys_placement; + return; + } + + if (device_unplugged && !tbo->base.dma_buf) { + *placement = purge_placement; + return; + } + + if (xe_bo_madv_is_dontneed(bo)) { + /* + * We can't use purge_placement here, since we need to trigger + * our own purge procedure at the start of xe_bo_move(), which + * would otherwise be skipped. At the same time we don't want + * ttm to then populate the tt with dst pages, before the move + * callback, hence use sys_placement here. + */ + *placement = sys_placement; + return; + } + + /* + * For xe, sg bos that are evicted to system just triggers a + * rebind of the sg list upon subsequent validation to XE_PL_TT. + */ + switch (tbo->resource->mem_type) { + case XE_PL_VRAM0: + case XE_PL_VRAM1: + case XE_PL_STOLEN: + *placement = tt_placement; + break; + case XE_PL_TT: + default: + *placement = sys_placement; + break; + } +} + +/* struct xe_ttm_tt - Subclassed ttm_tt for xe */ +struct xe_ttm_tt { + struct ttm_tt ttm; + struct sg_table sgt; + struct sg_table *sg; + /** @purgeable: Whether the content of the pages of @ttm is purgeable. */ + bool purgeable; +}; + +static int xe_tt_map_sg(struct xe_device *xe, struct ttm_tt *tt) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + unsigned long num_pages = tt->num_pages; + int ret; + + XE_WARN_ON((tt->page_flags & TTM_TT_FLAG_EXTERNAL) && + !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE)); + + if (xe_tt->sg) + return 0; + + ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages, + num_pages, 0, + (u64)num_pages << PAGE_SHIFT, + xe_sg_segment_size(xe->drm.dev), + GFP_KERNEL); + if (ret) + return ret; + + xe_tt->sg = &xe_tt->sgt; + ret = dma_map_sgtable(xe->drm.dev, xe_tt->sg, DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (ret) { + sg_free_table(xe_tt->sg); + xe_tt->sg = NULL; + return ret; + } + + return 0; +} + +static void xe_tt_unmap_sg(struct xe_device *xe, struct ttm_tt *tt) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + + if (xe_tt->sg) { + dma_unmap_sgtable(xe->drm.dev, xe_tt->sg, + DMA_BIDIRECTIONAL, 0); + sg_free_table(xe_tt->sg); + xe_tt->sg = NULL; + } +} + +struct sg_table *xe_bo_sg(struct xe_bo *bo) +{ + struct ttm_tt *tt = bo->ttm.ttm; + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + + return xe_tt->sg; +} + +/* + * Account ttm pages against the device shrinker's shrinkable and + * purgeable counts. + */ +static void xe_ttm_tt_account_add(struct xe_device *xe, struct ttm_tt *tt) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + + if (xe_tt->purgeable) + xe_shrinker_mod_pages(xe->mem.shrinker, 0, tt->num_pages); + else + xe_shrinker_mod_pages(xe->mem.shrinker, tt->num_pages, 0); +} + +static void xe_ttm_tt_account_subtract(struct xe_device *xe, struct ttm_tt *tt) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + + if (xe_tt->purgeable) + xe_shrinker_mod_pages(xe->mem.shrinker, 0, -(long)tt->num_pages); + else + xe_shrinker_mod_pages(xe->mem.shrinker, -(long)tt->num_pages, 0); +} + +static void update_global_total_pages(struct ttm_device *ttm_dev, + long num_pages) +{ +#if IS_ENABLED(CONFIG_TRACE_GPU_MEM) + struct xe_device *xe = ttm_to_xe_device(ttm_dev); + u64 global_total_pages = + atomic64_add_return(num_pages, &xe->global_total_pages); + + trace_gpu_mem_total(xe->drm.primary->index, 0, + global_total_pages << PAGE_SHIFT); +#endif +} + +static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, + u32 page_flags) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct xe_device *xe = xe_bo_device(bo); + struct xe_ttm_tt *xe_tt; + struct ttm_tt *tt; + unsigned long extra_pages; + enum ttm_caching caching = ttm_cached; + int err; + + xe_tt = kzalloc_obj(*xe_tt); + if (!xe_tt) + return NULL; + + tt = &xe_tt->ttm; + + extra_pages = 0; + if (xe_bo_needs_ccs_pages(bo)) + extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, xe_bo_size(bo)), + PAGE_SIZE); + + /* + * DGFX system memory is always WB / ttm_cached, since + * other caching modes are only supported on x86. DGFX + * GPU system memory accesses are always coherent with the + * CPU. + */ + if (!IS_DGFX(xe)) { + switch (bo->cpu_caching) { + case DRM_XE_GEM_CPU_CACHING_WC: + caching = ttm_write_combined; + break; + default: + caching = ttm_cached; + break; + } + + WARN_ON((bo->flags & XE_BO_FLAG_USER) && !bo->cpu_caching); + + /* + * For Xe_LPG and beyond up to NVL-P (excluding), PPGTT PTE + * lookups are also non-coherent and require a CPU:WC mapping. + */ + if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_FORCE_WC) || + (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE)) + caching = ttm_write_combined; + } + + if (bo->flags & XE_BO_FLAG_NEEDS_UC) { + /* + * Valid only for internally-created buffers only, for + * which cpu_caching is never initialized. + */ + xe_assert(xe, bo->cpu_caching == 0); + caching = ttm_uncached; + } + + if (ttm_bo->type != ttm_bo_type_sg) + page_flags |= TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE; + + err = ttm_tt_init(tt, &bo->ttm, page_flags, caching, extra_pages); + if (err) { + kfree(xe_tt); + return NULL; + } + + if (ttm_bo->type != ttm_bo_type_sg) { + err = ttm_tt_setup_backup(tt); + if (err) { + ttm_tt_fini(tt); + kfree(xe_tt); + return NULL; + } + } + + return tt; +} + +static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + int err; + + /* + * dma-bufs are not populated with pages, and the dma- + * addresses are set up when moved to XE_PL_TT. + */ + if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) && + !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE)) + return 0; + + if (ttm_tt_is_backed_up(tt) && !xe_tt->purgeable) { + err = ttm_tt_restore(ttm_dev, tt, ctx); + } else { + ttm_tt_clear_backed_up(tt); + err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx); + } + if (err) + return err; + + xe_tt->purgeable = false; + xe_ttm_tt_account_add(ttm_to_xe_device(ttm_dev), tt); + update_global_total_pages(ttm_dev, tt->num_pages); + + return 0; +} + +static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt) +{ + struct xe_device *xe = ttm_to_xe_device(ttm_dev); + + if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) && + !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE)) + return; + + xe_tt_unmap_sg(xe, tt); + + ttm_pool_free(&ttm_dev->pool, tt); + xe_ttm_tt_account_subtract(xe, tt); + update_global_total_pages(ttm_dev, -(long)tt->num_pages); +} + +static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt) +{ + ttm_tt_fini(tt); + kfree(tt); +} + +static bool xe_ttm_resource_visible(struct xe_device *xe, struct ttm_resource *mem) +{ + struct xe_ttm_vram_mgr_resource *vres; + + if (mem->mem_type == XE_PL_STOLEN) { + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; + + return mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe); + } + + vres = to_xe_ttm_vram_mgr_resource(mem); + return vres->used_visible_size == mem->size; +} + +/** + * xe_bo_is_visible_vram - check if BO is placed entirely in visible VRAM. + * @bo: The BO + * + * This function checks whether a given BO resides entirely in memory visible from the CPU + * + * Returns: true if the BO is entirely visible, false otherwise. + * + */ +bool xe_bo_is_visible_vram(struct xe_bo *bo) +{ + if (drm_WARN_ON(bo->ttm.base.dev, !xe_bo_is_vram(bo))) + return false; + + return xe_ttm_resource_visible(xe_bo_device(bo), bo->ttm.resource); +} + +static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, + struct ttm_resource *mem) +{ + struct xe_device *xe = ttm_to_xe_device(bdev); + + switch (mem->mem_type) { + case XE_PL_SYSTEM: + case XE_PL_TT: + return 0; + case XE_PL_VRAM0: + case XE_PL_VRAM1: { + struct xe_vram_region *vram = xe_map_resource_to_region(mem); + + if (!xe_ttm_resource_visible(xe, mem)) + return -EINVAL; + + mem->bus.offset = mem->start << PAGE_SHIFT; + + if (vram->mapping && + mem->placement & TTM_PL_FLAG_CONTIGUOUS) + mem->bus.addr = (u8 __force *)vram->mapping + + mem->bus.offset; + + mem->bus.offset += vram->io_start; + mem->bus.is_iomem = true; + +#if !IS_ENABLED(CONFIG_X86) + mem->bus.caching = ttm_write_combined; +#endif + return 0; + } case XE_PL_STOLEN: + return xe_ttm_stolen_io_mem_reserve(xe, mem); + default: + return -EINVAL; + } +} + +static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo, + const struct ttm_operation_ctx *ctx) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + struct drm_gem_object *obj = &bo->ttm.base; + struct drm_gpuvm_bo *vm_bo; + bool idle = false; + int ret = 0; + + dma_resv_assert_held(bo->ttm.base.resv); + + if (!list_empty(&bo->ttm.base.gpuva.list)) { + dma_resv_iter_begin(&cursor, bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) + dma_fence_enable_signaling(fence); + dma_resv_iter_end(&cursor); + } + + drm_gem_for_each_gpuvm_bo(vm_bo, obj) { + struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); + struct drm_gpuva *gpuva; + + if (!xe_vm_in_fault_mode(vm)) { + drm_gpuvm_bo_evict(vm_bo, true); + /* + * L2 cache may not be flushed, so ensure that is done in + * xe_vm_invalidate_vma() below + */ + if (!xe_device_is_l2_flush_optimized(xe)) + continue; + } + + if (!idle) { + long timeout; + + if (ctx->no_wait_gpu && + !dma_resv_test_signaled(bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + return -EBUSY; + + timeout = dma_resv_wait_timeout(bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, + MAX_SCHEDULE_TIMEOUT); + if (!timeout) + return -ETIME; + if (timeout < 0) + return timeout; + + idle = true; + } + + drm_gpuvm_bo_for_each_va(gpuva, vm_bo) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + trace_xe_vma_evict(vma); + ret = xe_vm_invalidate_vma(vma); + if (XE_WARN_ON(ret)) + return ret; + } + } + + return ret; +} + +/* + * The dma-buf map_attachment() / unmap_attachment() is hooked up here. + * Note that unmapping the attachment is deferred to the next + * map_attachment time, or to bo destroy (after idling) whichever comes first. + * This is to avoid syncing before unmap_attachment(), assuming that the + * caller relies on idling the reservation object before moving the + * backing store out. Should that assumption not hold, then we will be able + * to unconditionally call unmap_attachment() when moving out to system. + */ +static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo, + struct ttm_resource *new_res) +{ + struct dma_buf_attachment *attach = ttm_bo->base.import_attach; + struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt, + ttm); + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + bool device_unplugged = drm_dev_is_unplugged(&xe->drm); + struct sg_table *sg; + + xe_assert(xe, attach); + xe_assert(xe, ttm_bo->ttm); + + if (device_unplugged && new_res->mem_type == XE_PL_SYSTEM && + ttm_bo->sg) { + dma_resv_wait_timeout(ttm_bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL); + ttm_bo->sg = NULL; + } + + if (new_res->mem_type == XE_PL_SYSTEM) + goto out; + + if (ttm_bo->sg) { + dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL); + ttm_bo->sg = NULL; + } + + sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); + if (IS_ERR(sg)) + return PTR_ERR(sg); + + ttm_bo->sg = sg; + xe_tt->sg = sg; + +out: + ttm_bo_move_null(ttm_bo, new_res); + + return 0; +} + +/** + * xe_bo_move_notify - Notify subsystems of a pending move + * @bo: The buffer object + * @ctx: The struct ttm_operation_ctx controlling locking and waits. + * + * This function notifies subsystems of an upcoming buffer move. + * Upon receiving such a notification, subsystems should schedule + * halting access to the underlying pages and optionally add a fence + * to the buffer object's dma_resv object, that signals when access is + * stopped. The caller will wait on all dma_resv fences before + * starting the move. + * + * A subsystem may commence access to the object after obtaining + * bindings to the new backing memory under the object lock. + * + * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode, + * negative error code on error. + */ +static int xe_bo_move_notify(struct xe_bo *bo, + const struct ttm_operation_ctx *ctx) +{ + struct ttm_buffer_object *ttm_bo = &bo->ttm; + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + struct ttm_resource *old_mem = ttm_bo->resource; + u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM; + int ret; + + /* + * If this starts to call into many components, consider + * using a notification chain here. + */ + + if (xe_bo_is_pinned(bo)) + return -EINVAL; + + xe_bo_vunmap(bo); + ret = xe_bo_trigger_rebind(xe, bo, ctx); + if (ret) + return ret; + + /* Don't call move_notify() for imported dma-bufs. */ + if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach) + dma_buf_invalidate_mappings(ttm_bo->base.dma_buf); + + /* + * TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual), + * so if we moved from VRAM make sure to unlink this from the userfault + * tracking. + */ + if (mem_type_is_vram(old_mem_type)) { + mutex_lock(&xe->mem_access.vram_userfault.lock); + if (!list_empty(&bo->vram_userfault_link)) + list_del_init(&bo->vram_userfault_link); + mutex_unlock(&xe->mem_access.vram_userfault.lock); + } + + return 0; +} + +/** + * xe_bo_set_purgeable_shrinker() - Update shrinker accounting for purgeable state + * @bo: Buffer object + * @new_state: New purgeable state being set + * + * Transfers pages between shrinkable and purgeable buckets when the BO + * purgeable state changes. Called automatically from xe_bo_set_purgeable_state(). + */ +static void xe_bo_set_purgeable_shrinker(struct xe_bo *bo, + enum xe_madv_purgeable_state new_state) +{ + struct ttm_buffer_object *ttm_bo = &bo->ttm; + struct ttm_tt *tt = ttm_bo->ttm; + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + struct xe_ttm_tt *xe_tt; + long tt_pages; + + xe_bo_assert_held(bo); + + if (!tt || !ttm_tt_is_populated(tt)) + return; + + xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + tt_pages = tt->num_pages; + + if (!xe_tt->purgeable && new_state == XE_MADV_PURGEABLE_DONTNEED) { + xe_tt->purgeable = true; + /* Transfer pages from shrinkable to purgeable count */ + xe_shrinker_mod_pages(xe->mem.shrinker, -tt_pages, tt_pages); + } else if (xe_tt->purgeable && new_state == XE_MADV_PURGEABLE_WILLNEED) { + xe_tt->purgeable = false; + /* Transfer pages from purgeable to shrinkable count */ + xe_shrinker_mod_pages(xe->mem.shrinker, tt_pages, -tt_pages); + } +} + +/** + * xe_bo_set_purgeable_state() - Set BO purgeable state with validation + * @bo: Buffer object + * @new_state: New purgeable state + * + * Sets the purgeable state with lockdep assertions and validates state + * transitions. Once a BO is PURGED, it cannot transition to any other state. + * Invalid transitions are caught with xe_assert(). Shrinker page accounting + * is updated automatically. + */ +void xe_bo_set_purgeable_state(struct xe_bo *bo, + enum xe_madv_purgeable_state new_state) +{ + struct xe_device *xe = xe_bo_device(bo); + + xe_bo_assert_held(bo); + + /* Validate state is one of the known values */ + xe_assert(xe, new_state == XE_MADV_PURGEABLE_WILLNEED || + new_state == XE_MADV_PURGEABLE_DONTNEED || + new_state == XE_MADV_PURGEABLE_PURGED); + + /* Once purged, always purged - cannot transition out */ + xe_assert(xe, !(bo->purgeable.state == XE_MADV_PURGEABLE_PURGED && + new_state != XE_MADV_PURGEABLE_PURGED)); + + bo->purgeable.state = new_state; + xe_bo_set_purgeable_shrinker(bo, new_state); +} + +/** + * xe_ttm_bo_purge() - Purge buffer object backing store + * @ttm_bo: The TTM buffer object to purge + * @ctx: TTM operation context + * + * This function purges the backing store of a BO marked as DONTNEED and + * triggers rebind to invalidate stale GPU mappings. For fault-mode VMs, + * this zaps the PTEs. The next GPU access will trigger a page fault and + * perform NULL rebind (scratch pages or clear PTEs based on VM config). + * + * Return: 0 on success, negative error code on failure + */ +static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct ttm_placement place = {}; + int ret; + + xe_bo_assert_held(bo); + + if (!ttm_bo->ttm) + return 0; + + if (!xe_bo_madv_is_dontneed(bo)) + return 0; + + /* + * Use the standard pre-move hook so we share the same cleanup/invalidate + * path as migrations: drop any CPU vmap and schedule the necessary GPU + * unbind/rebind work. + * + * This must be called before ttm_bo_validate() frees the pages. + * May fail in no-wait contexts (fault/shrinker) or if the BO is + * pinned. Keep state unchanged on failure so we don't end up "PURGED" + * with stale mappings. + */ + ret = xe_bo_move_notify(bo, ctx); + if (ret) + return ret; + + ret = ttm_bo_validate(ttm_bo, &place, ctx); + if (ret) + return ret; + + /* Commit the state transition only once invalidation was queued */ + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_PURGED); + + return 0; +} + +static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_resource *new_mem, + struct ttm_place *hop) +{ + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct ttm_resource *old_mem = ttm_bo->resource; + u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM; + struct ttm_tt *ttm = ttm_bo->ttm; + struct xe_migrate *migrate = NULL; + struct dma_fence *fence; + bool move_lacks_source; + bool tt_has_data; + bool needs_clear; + bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) && + ttm && ttm_tt_is_populated(ttm)) ? true : false; + int ret = 0; + + /* + * Purge only non-shared BOs explicitly marked DONTNEED by userspace. + * The move_notify callback will handle invalidation asynchronously. + */ + if (evict && xe_bo_madv_is_dontneed(bo)) { + ret = xe_ttm_bo_purge(ttm_bo, ctx); + if (ret) + return ret; + + /* Free the unused eviction destination resource */ + ttm_resource_free(ttm_bo, &new_mem); + return 0; + } + + /* Bo creation path, moving to system or TT. */ + if ((!old_mem && ttm) && !handle_system_ccs) { + if (new_mem->mem_type == XE_PL_TT) + ret = xe_tt_map_sg(xe, ttm); + if (!ret) + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + if (ttm_bo->type == ttm_bo_type_sg) { + if (new_mem->mem_type == XE_PL_SYSTEM) + ret = xe_bo_move_notify(bo, ctx); + if (!ret) + ret = xe_bo_move_dmabuf(ttm_bo, new_mem); + return ret; + } + + tt_has_data = ttm && (ttm_tt_is_populated(ttm) || ttm_tt_is_swapped(ttm)); + + move_lacks_source = !old_mem || (handle_system_ccs ? (!bo->ccs_cleared) : + (!mem_type_is_vram(old_mem_type) && !tt_has_data)); + + needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) || + (!ttm && ttm_bo->type == ttm_bo_type_device); + + if (new_mem->mem_type == XE_PL_TT) { + ret = xe_tt_map_sg(xe, ttm); + if (ret) + goto out; + } + + if ((move_lacks_source && !needs_clear)) { + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + if (!move_lacks_source && (bo->flags & XE_BO_FLAG_CPU_ADDR_MIRROR) && + new_mem->mem_type == XE_PL_SYSTEM) { + ret = xe_svm_bo_evict(bo); + if (!ret) { + drm_dbg(&xe->drm, "Evict system allocator BO success\n"); + ttm_bo_move_null(ttm_bo, new_mem); + } else { + drm_dbg(&xe->drm, "Evict system allocator BO failed=%pe\n", + ERR_PTR(ret)); + } + + goto out; + } + + if (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT && !handle_system_ccs) { + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + /* + * Failed multi-hop where the old_mem is still marked as + * TTM_PL_FLAG_TEMPORARY, should just be a dummy move. + */ + if (old_mem_type == XE_PL_TT && + new_mem->mem_type == XE_PL_TT) { + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + if (!move_lacks_source && !xe_bo_is_pinned(bo)) { + ret = xe_bo_move_notify(bo, ctx); + if (ret) + goto out; + } + + if (old_mem_type == XE_PL_TT && + new_mem->mem_type == XE_PL_SYSTEM) { + long timeout = dma_resv_wait_timeout(ttm_bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, + false, + MAX_SCHEDULE_TIMEOUT); + if (timeout < 0) { + ret = timeout; + goto out; + } + + if (!handle_system_ccs) { + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + } + + if (!move_lacks_source && + ((old_mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) || + (mem_type_is_vram(old_mem_type) && + new_mem->mem_type == XE_PL_SYSTEM))) { + hop->fpfn = 0; + hop->lpfn = 0; + hop->mem_type = XE_PL_TT; + hop->flags = TTM_PL_FLAG_TEMPORARY; + ret = -EMULTIHOP; + goto out; + } + + if (bo->tile) + migrate = bo->tile->migrate; + else if (resource_is_vram(new_mem)) + migrate = mem_type_to_migrate(xe, new_mem->mem_type); + else if (mem_type_is_vram(old_mem_type)) + migrate = mem_type_to_migrate(xe, old_mem_type); + else + migrate = xe->tiles[0].migrate; + + xe_assert(xe, migrate); + trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source); + if (xe_rpm_reclaim_safe(xe)) { + /* + * We might be called through swapout in the validation path of + * another TTM device, so acquire rpm here. + */ + xe_pm_runtime_get(xe); + } else { + drm_WARN_ON(&xe->drm, handle_system_ccs); + xe_pm_runtime_get_noresume(xe); + } + + /* + * Attach CCS BBs before submitting the copy job below so a VF + * migration racing the copy sees valid, up to date attach state. + */ + if (IS_VF_CCS_READY(xe) && + ((move_lacks_source && new_mem->mem_type == XE_PL_TT) || + (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) && + handle_system_ccs) { + ret = xe_sriov_vf_ccs_attach_bo(bo, new_mem); + if (ret) { + xe_pm_runtime_put(xe); + goto out; + } + } + + if (move_lacks_source) { + u32 flags = 0; + + if (mem_type_is_vram(new_mem->mem_type)) + flags |= XE_MIGRATE_CLEAR_FLAG_FULL; + else if (handle_system_ccs) + flags |= XE_MIGRATE_CLEAR_FLAG_CCS_DATA; + + fence = xe_migrate_clear(migrate, bo, new_mem, flags); + } else { + fence = xe_migrate_copy(migrate, bo, bo, old_mem, new_mem, + handle_system_ccs); + } + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + xe_pm_runtime_put(xe); + goto out; + } + if (!move_lacks_source) { + ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict, true, + new_mem); + if (ret) { + dma_fence_wait(fence, false); + ttm_bo_move_null(ttm_bo, new_mem); + ret = 0; + } + } else { + /* + * ttm_bo_move_accel_cleanup() may blow up if + * bo->resource == NULL, so just attach the + * fence and set the new resource. + */ + dma_resv_add_fence(ttm_bo->base.resv, fence, + DMA_RESV_USAGE_KERNEL); + ttm_bo_move_null(ttm_bo, new_mem); + } + + /* + * Detach must wait for the copy above to complete: a VF migration + * racing an in-flight copy must still see valid CCS BBs, so don't + * tear them down until the copy fence has signaled. + */ + if (IS_VF_CCS_READY(xe) && old_mem_type == XE_PL_TT && + new_mem->mem_type == XE_PL_SYSTEM) { + dma_fence_wait(fence, false); + xe_sriov_vf_ccs_detach_bo(bo); + } + + dma_fence_put(fence); + xe_pm_runtime_put(xe); + +out: + if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) && + ttm_bo->ttm) { + long timeout = dma_resv_wait_timeout(ttm_bo->base.resv, + DMA_RESV_USAGE_KERNEL, + false, + MAX_SCHEDULE_TIMEOUT); + if (timeout < 0) + ret = timeout; + + if (IS_VF_CCS_READY(xe)) + xe_sriov_vf_ccs_detach_bo(bo); + + xe_tt_unmap_sg(xe, ttm_bo->ttm); + } + + return ret; +} + +static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx, + struct ttm_buffer_object *bo, + unsigned long *scanned) +{ + struct xe_device *xe = ttm_to_xe_device(bo->bdev); + struct ttm_tt *tt = bo->ttm; + long lret; + + /* Fake move to system, without copying data. */ + if (bo->resource->mem_type != XE_PL_SYSTEM) { + struct ttm_resource *new_resource; + + lret = ttm_bo_wait_ctx(bo, ctx); + if (lret) + return lret; + + lret = ttm_bo_mem_space(bo, &sys_placement, &new_resource, ctx); + if (lret) + return lret; + + xe_tt_unmap_sg(xe, bo->ttm); + ttm_bo_move_null(bo, new_resource); + } + + *scanned += bo->ttm->num_pages; + lret = ttm_bo_shrink(ctx, bo, (struct ttm_bo_shrink_flags) + {.purge = true, + .writeback = false, + .allow_move = false}); + + if (lret > 0) { + xe_ttm_tt_account_subtract(xe, bo->ttm); + update_global_total_pages(bo->bdev, -(long)tt->num_pages); + } + + return lret; +} + +static bool +xe_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) +{ + struct drm_gpuvm_bo *vm_bo; + + if (!ttm_bo_eviction_valuable(bo, place)) + return false; + + if (!xe_bo_is_xe_bo(bo)) + return true; + + drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) { + if (xe_vm_is_validating(gpuvm_to_vm(vm_bo->vm))) + return false; + } + + return true; +} + +/** + * xe_bo_shrink() - Try to shrink an xe bo. + * @ctx: The struct ttm_operation_ctx used for shrinking. + * @bo: The TTM buffer object whose pages to shrink. + * @flags: Flags governing the shrink behaviour. + * @scanned: Pointer to a counter of the number of pages + * attempted to shrink. + * + * Try to shrink- or purge a bo, and if it succeeds, unmap dma. + * Note that we need to be able to handle also non xe bos + * (ghost bos), but only if the struct ttm_tt is embedded in + * a struct xe_ttm_tt. When the function attempts to shrink + * the pages of a buffer object, The value pointed to by @scanned + * is updated. + * + * Return: The number of pages shrunken or purged, or negative error + * code on failure. + */ +long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, + const struct xe_bo_shrink_flags flags, + unsigned long *scanned) +{ + struct ttm_tt *tt = bo->ttm; + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + struct ttm_place place = {.mem_type = bo->resource->mem_type}; + struct xe_bo *xe_bo = ttm_to_xe_bo(bo); + struct xe_device *xe = ttm_to_xe_device(bo->bdev); + bool needs_rpm; + long lret = 0L; + + if (!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE) || + (flags.purge && !xe_tt->purgeable)) + return -EBUSY; + + if (!xe_bo_eviction_valuable(bo, &place)) + return -EBUSY; + + if (!xe_bo_is_xe_bo(bo) || !xe_bo_get_unless_zero(xe_bo)) + return xe_bo_shrink_purge(ctx, bo, scanned); + + if (xe_tt->purgeable) { + if (bo->resource->mem_type != XE_PL_SYSTEM) + lret = xe_bo_move_notify(xe_bo, ctx); + if (!lret) + lret = xe_bo_shrink_purge(ctx, bo, scanned); + if (lret > 0 && xe_bo_madv_is_dontneed(xe_bo)) + xe_bo_set_purgeable_state(xe_bo, + XE_MADV_PURGEABLE_PURGED); + goto out_unref; + } + + /* System CCS needs gpu copy when moving PL_TT -> PL_SYSTEM */ + needs_rpm = (!IS_DGFX(xe) && bo->resource->mem_type != XE_PL_SYSTEM && + xe_bo_needs_ccs_pages(xe_bo)); + if (needs_rpm && !xe_pm_runtime_get_if_active(xe)) + goto out_unref; + + *scanned += tt->num_pages; + lret = ttm_bo_shrink(ctx, bo, (struct ttm_bo_shrink_flags) + {.purge = false, + .writeback = flags.writeback, + .allow_move = true}); + if (needs_rpm) + xe_pm_runtime_put(xe); + + if (lret > 0) { + xe_ttm_tt_account_subtract(xe, tt); + update_global_total_pages(bo->bdev, -(long)tt->num_pages); + } + +out_unref: + xe_bo_put(xe_bo); + + return lret; +} + +/** + * xe_bo_notifier_prepare_pinned() - Prepare a pinned VRAM object to be backed + * up in system memory. + * @bo: The buffer object to prepare. + * + * On successful completion, the object backup pages are allocated. Expectation + * is that this is called from the PM notifier, prior to suspend/hibernation. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_bo_notifier_prepare_pinned(struct xe_bo *bo) +{ + struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_bo *backup; + int ret = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.exclusive = true}, ret) { + ret = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + xe_assert(xe, !ret); + xe_assert(xe, !bo->backup_obj); + + /* + * Since this is called from the PM notifier we might have raced with + * someone unpinning this after we dropped the pinned list lock and + * grabbing the above bo lock. + */ + if (!xe_bo_is_pinned(bo)) + break; + + if (!xe_bo_is_vram(bo)) + break; + + if (bo->flags & XE_BO_FLAG_PINNED_NORESTORE) + break; + + backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, xe_bo_size(bo), + DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS | + XE_BO_FLAG_PINNED, NULL, &exec); + if (IS_ERR(backup)) { + drm_exec_retry_on_contention(&exec); + ret = PTR_ERR(backup); + xe_validation_retry_on_oom(&ctx, &ret); + break; + } + + backup->parent_obj = xe_bo_get(bo); /* Released by bo_destroy */ + ttm_bo_pin(&backup->ttm); + bo->backup_obj = backup; + } + + return ret; +} + +/** + * xe_bo_notifier_unprepare_pinned() - Undo the previous prepare operation. + * @bo: The buffer object to undo the prepare for. + * + * Always returns 0. The backup object is removed, if still present. Expectation + * it that this called from the PM notifier when undoing the prepare step. + * + * Return: Always returns 0. + */ +int xe_bo_notifier_unprepare_pinned(struct xe_bo *bo) +{ + xe_bo_lock(bo, false); + if (bo->backup_obj) { + ttm_bo_unpin(&bo->backup_obj->ttm); + xe_bo_put(bo->backup_obj); + bo->backup_obj = NULL; + } + xe_bo_unlock(bo); + + return 0; +} + +static int xe_bo_evict_pinned_copy(struct xe_bo *bo, struct xe_bo *backup) +{ + struct xe_device *xe = xe_bo_device(bo); + bool unmap = false; + int ret = 0; + + if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) { + struct xe_migrate *migrate; + struct dma_fence *fence; + + if (bo->tile) + migrate = bo->tile->migrate; + else + migrate = mem_type_to_migrate(xe, bo->ttm.resource->mem_type); + + xe_assert(xe, bo->ttm.base.resv == backup->ttm.base.resv); + ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1); + if (ret) + goto out_backup; + + fence = xe_migrate_copy(migrate, bo, backup, bo->ttm.resource, + backup->ttm.resource, false); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto out_backup; + } + + dma_resv_add_fence(bo->ttm.base.resv, fence, + DMA_RESV_USAGE_KERNEL); + dma_fence_put(fence); + } else { + ret = xe_bo_vmap(backup); + if (ret) + goto out_backup; + + if (iosys_map_is_null(&bo->vmap)) { + ret = xe_bo_vmap(bo); + if (ret) + goto out_vunmap; + unmap = true; + } + + xe_map_memcpy_from(xe, backup->vmap.vaddr, &bo->vmap, 0, + xe_bo_size(bo)); + } + + if (!bo->backup_obj) + bo->backup_obj = backup; +out_vunmap: + xe_bo_vunmap(backup); +out_backup: + if (unmap) + xe_bo_vunmap(bo); + + return ret; +} + +/** + * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory + * @bo: The buffer object to move. + * + * On successful completion, the object memory will be moved to system memory. + * + * This is needed to for special handling of pinned VRAM object during + * suspend-resume. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_bo_evict_pinned(struct xe_bo *bo) +{ + struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_bo *backup = bo->backup_obj; + bool backup_created = false; + int ret = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.exclusive = true}, ret) { + ret = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + xe_assert(xe, !ret); + + if (WARN_ON(!bo->ttm.resource)) { + ret = -EINVAL; + break; + } + + if (WARN_ON(!xe_bo_is_pinned(bo))) { + ret = -EINVAL; + break; + } + + if (!xe_bo_is_vram(bo)) + break; + + if (bo->flags & XE_BO_FLAG_PINNED_NORESTORE) + break; + + if (!backup) { + backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, + xe_bo_size(bo), + DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS | + XE_BO_FLAG_PINNED, NULL, &exec); + if (IS_ERR(backup)) { + drm_exec_retry_on_contention(&exec); + ret = PTR_ERR(backup); + xe_validation_retry_on_oom(&ctx, &ret); + break; + } + backup->parent_obj = xe_bo_get(bo); /* Released by bo_destroy */ + backup_created = true; + } + + ret = xe_bo_evict_pinned_copy(bo, backup); + } + + if (ret && backup_created) + xe_bo_put(backup); + + return ret; +} + +/** + * xe_bo_restore_pinned() - Restore a pinned VRAM object + * @bo: The buffer object to move. + * + * On successful completion, the object memory will be moved back to VRAM. + * + * This is needed to for special handling of pinned VRAM object during + * suspend-resume. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_bo_restore_pinned(struct xe_bo *bo) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .gfp_retry_mayfail = false, + }; + struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); + struct xe_bo *backup = bo->backup_obj; + bool unmap = false; + int ret; + + if (!backup) + return 0; + + xe_bo_lock(bo, false); + + if (!xe_bo_is_pinned(backup)) { + ret = ttm_bo_validate(&backup->ttm, &backup->placement, &ctx); + if (ret) + goto out_unlock_bo; + } + + if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) { + struct xe_migrate *migrate; + struct dma_fence *fence; + + if (bo->tile) + migrate = bo->tile->migrate; + else + migrate = mem_type_to_migrate(xe, bo->ttm.resource->mem_type); + + ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1); + if (ret) + goto out_unlock_bo; + + fence = xe_migrate_copy(migrate, backup, bo, + backup->ttm.resource, bo->ttm.resource, + false); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto out_unlock_bo; + } + + dma_resv_add_fence(bo->ttm.base.resv, fence, + DMA_RESV_USAGE_KERNEL); + dma_fence_put(fence); + } else { + ret = xe_bo_vmap(backup); + if (ret) + goto out_unlock_bo; + + if (iosys_map_is_null(&bo->vmap)) { + ret = xe_bo_vmap(bo); + if (ret) + goto out_backup; + unmap = true; + } + + xe_map_memcpy_to(xe, &bo->vmap, 0, backup->vmap.vaddr, + xe_bo_size(bo)); + } + + bo->backup_obj = NULL; + +out_backup: + xe_bo_vunmap(backup); + if (!bo->backup_obj) { + if (xe_bo_is_pinned(backup)) + ttm_bo_unpin(&backup->ttm); + xe_bo_put(backup); + } +out_unlock_bo: + if (unmap) + xe_bo_vunmap(bo); + xe_bo_unlock(bo); + return ret; +} + +int xe_bo_dma_unmap_pinned(struct xe_bo *bo) +{ + struct ttm_buffer_object *ttm_bo = &bo->ttm; + struct ttm_tt *tt = ttm_bo->ttm; + + if (tt) { + struct xe_ttm_tt *xe_tt = container_of(tt, typeof(*xe_tt), ttm); + + if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) { + dma_buf_unmap_attachment(ttm_bo->base.import_attach, + ttm_bo->sg, + DMA_BIDIRECTIONAL); + ttm_bo->sg = NULL; + xe_tt->sg = NULL; + } else if (xe_tt->sg) { + dma_unmap_sgtable(ttm_to_xe_device(ttm_bo->bdev)->drm.dev, + xe_tt->sg, + DMA_BIDIRECTIONAL, 0); + sg_free_table(xe_tt->sg); + xe_tt->sg = NULL; + } + } + + return 0; +} + +static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo, + unsigned long page_offset) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct xe_res_cursor cursor; + struct xe_vram_region *vram; + + if (ttm_bo->resource->mem_type == XE_PL_STOLEN) + return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT; + + vram = xe_map_resource_to_region(ttm_bo->resource); + xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor); + return (vram->io_start + cursor.start) >> PAGE_SHIFT; +} + +static void __xe_bo_vunmap(struct xe_bo *bo); + +/* + * TODO: Move this function to TTM so we don't rely on how TTM does its + * locking, thereby abusing TTM internals. + */ +static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo) +{ + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + bool locked; + + xe_assert(xe, !kref_read(&ttm_bo->kref)); + + /* + * We can typically only race with TTM trylocking under the + * lru_lock, which will immediately be unlocked again since + * the ttm_bo refcount is zero at this point. So trylocking *should* + * always succeed here, as long as we hold the lru lock. + */ + spin_lock(&ttm_bo->bdev->lru_lock); + locked = dma_resv_trylock(&ttm_bo->base._resv); + spin_unlock(&ttm_bo->bdev->lru_lock); + xe_assert(xe, locked); + + return locked; +} + +static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + struct dma_fence *replacement = NULL; + struct xe_bo *bo; + + if (!xe_bo_is_xe_bo(ttm_bo)) + return; + + bo = ttm_to_xe_bo(ttm_bo); + xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount))); + + if (!xe_ttm_bo_lock_in_destructor(ttm_bo)) + return; + + /* + * Scrub the preempt fences if any. The unbind fence is already + * attached to the resv. + * TODO: Don't do this for external bos once we scrub them after + * unbind. + */ + dma_resv_for_each_fence(&cursor, &ttm_bo->base._resv, + DMA_RESV_USAGE_BOOKKEEP, fence) { + if (xe_fence_is_xe_preempt(fence) && + !dma_fence_is_signaled(fence)) { + if (!replacement) + replacement = dma_fence_get_stub(); + + dma_resv_replace_fences(&ttm_bo->base._resv, + fence->context, + replacement, + DMA_RESV_USAGE_BOOKKEEP); + } + } + dma_fence_put(replacement); + + dma_resv_unlock(&ttm_bo->base._resv); +} + +static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + + if (!xe_bo_is_xe_bo(ttm_bo)) + return; + + if (IS_VF_CCS_READY(ttm_to_xe_device(ttm_bo->bdev))) + xe_sriov_vf_ccs_detach_bo(bo); + + /* + * Object is idle and about to be destroyed. Release the + * dma-buf attachment. + */ + if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) { + struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, + struct xe_ttm_tt, ttm); + + dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg, + DMA_BIDIRECTIONAL); + ttm_bo->sg = NULL; + xe_tt->sg = NULL; + } +} + +static void xe_ttm_bo_swap_notify(struct ttm_buffer_object *ttm_bo) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .gfp_retry_mayfail = false, + }; + + if (ttm_bo->ttm) { + struct xe_ttm_tt *xe_tt = + container_of(ttm_bo->ttm, struct xe_ttm_tt, ttm); + + if (xe_tt->purgeable) + xe_ttm_bo_purge(ttm_bo, &ctx); + } +} + +static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo, + unsigned long offset, void *buf, int len, + int write) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + struct iosys_map vmap; + struct xe_res_cursor cursor; + struct xe_vram_region *vram; + int bytes_left = len; + int err = 0; + + xe_bo_assert_held(bo); + xe_device_assert_mem_access(xe); + + if (!mem_type_is_vram(ttm_bo->resource->mem_type)) + return -EIO; + + if (!xe_bo_is_visible_vram(bo) || len >= SZ_16K) { + struct xe_migrate *migrate = + mem_type_to_migrate(xe, ttm_bo->resource->mem_type); + + err = xe_migrate_access_memory(migrate, bo, offset, buf, len, + write); + goto out; + } + + vram = xe_map_resource_to_region(ttm_bo->resource); + xe_res_first(ttm_bo->resource, offset & PAGE_MASK, + xe_bo_size(bo) - (offset & PAGE_MASK), &cursor); + + do { + unsigned long page_offset = (offset & ~PAGE_MASK); + int byte_count = min((int)(PAGE_SIZE - page_offset), bytes_left); + + iosys_map_set_vaddr_iomem(&vmap, (u8 __iomem *)vram->mapping + + cursor.start); + if (write) + xe_map_memcpy_to(xe, &vmap, page_offset, buf, byte_count); + else + xe_map_memcpy_from(xe, buf, &vmap, page_offset, byte_count); + + buf += byte_count; + offset += byte_count; + bytes_left -= byte_count; + if (bytes_left) + xe_res_next(&cursor, PAGE_SIZE); + } while (bytes_left); + +out: + return err ?: len; +} + +const struct ttm_device_funcs xe_ttm_funcs = { + .ttm_tt_create = xe_ttm_tt_create, + .ttm_tt_populate = xe_ttm_tt_populate, + .ttm_tt_unpopulate = xe_ttm_tt_unpopulate, + .ttm_tt_destroy = xe_ttm_tt_destroy, + .evict_flags = xe_evict_flags, + .move = xe_bo_move, + .io_mem_reserve = xe_ttm_io_mem_reserve, + .io_mem_pfn = xe_ttm_io_mem_pfn, + .access_memory = xe_ttm_access_memory, + .release_notify = xe_ttm_bo_release_notify, + .eviction_valuable = xe_bo_eviction_valuable, + .delete_mem_notify = xe_ttm_bo_delete_mem_notify, + .swap_notify = xe_ttm_bo_swap_notify, +}; + +static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + struct xe_tile *tile; + u8 id; + + if (bo->ttm.base.import_attach) + drm_prime_gem_destroy(&bo->ttm.base, NULL); + if (bo->dma_buf) + dma_buf_put(bo->dma_buf); + drm_gem_object_release(&bo->ttm.base); + + xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list)); + + for_each_tile(tile, xe, id) + if (bo->ggtt_node[id]) + xe_ggtt_remove_bo(tile->mem.ggtt, bo); + +#ifdef CONFIG_PROC_FS + if (bo->client) + xe_drm_client_remove_bo(bo); +#endif + + if (bo->vm && xe_bo_is_user(bo)) + xe_vm_put(bo->vm); + + if (bo->parent_obj) + xe_bo_put(bo->parent_obj); + + mutex_lock(&xe->mem_access.vram_userfault.lock); + if (!list_empty(&bo->vram_userfault_link)) + list_del(&bo->vram_userfault_link); + mutex_unlock(&xe->mem_access.vram_userfault.lock); + + kfree(bo); +} + +static void xe_gem_object_free(struct drm_gem_object *obj) +{ + /* Our BO reference counting scheme works as follows: + * + * The gem object kref is typically used throughout the driver, + * and the gem object holds a ttm_buffer_object refcount, so + * that when the last gem object reference is put, which is when + * we end up in this function, we put also that ttm_buffer_object + * refcount. Anything using gem interfaces is then no longer + * allowed to access the object in a way that requires a gem + * refcount, including locking the object. + * + * driver ttm callbacks is allowed to use the ttm_buffer_object + * refcount directly if needed. + */ + __xe_bo_vunmap(gem_to_xe_bo(obj)); + ttm_bo_fini(container_of(obj, struct ttm_buffer_object, base)); +} + +static void xe_gem_object_close(struct drm_gem_object *obj, + struct drm_file *file_priv) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + + if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) { + xe_assert(xe_bo_device(bo), xe_bo_is_user(bo)); + + xe_bo_lock(bo, false); + ttm_bo_set_bulk_move(&bo->ttm, NULL); + xe_bo_unlock(bo); + } +} + +static bool should_migrate_to_smem(struct xe_bo *bo) +{ + /* + * NOTE: The following atomic checks are platform-specific. For example, + * if a device supports CXL atomics, these may not be necessary or + * may behave differently. + */ + + return bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL || + bo->attr.atomic_access == DRM_XE_ATOMIC_CPU; +} + +static int xe_bo_wait_usage_kernel(struct xe_bo *bo, struct ttm_operation_ctx *ctx) +{ + long lerr; + + if (ctx->no_wait_gpu) + return dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL) ? + 0 : -EBUSY; + + lerr = dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + ctx->interruptible, MAX_SCHEDULE_TIMEOUT); + if (lerr < 0) + return lerr; + if (lerr == 0) + return -EBUSY; + + return 0; +} + +/* Populate the bo if swapped out, or migrate if the access mode requires that. */ +static int xe_bo_fault_migrate(struct xe_bo *bo, struct ttm_operation_ctx *ctx, + struct drm_exec *exec) +{ + struct ttm_buffer_object *tbo = &bo->ttm; + int err = 0; + + if (ttm_manager_type(tbo->bdev, tbo->resource->mem_type)->use_tt) { + err = xe_bo_wait_usage_kernel(bo, ctx); + if (!err) + err = ttm_bo_populate(&bo->ttm, ctx); + } else if (should_migrate_to_smem(bo)) { + xe_assert(xe_bo_device(bo), bo->flags & XE_BO_FLAG_SYSTEM); + err = xe_bo_migrate(bo, XE_PL_TT, ctx, exec); + } + + return err; +} + +/* Call into TTM to populate PTEs, and register bo for PTE removal on runtime suspend. */ +static vm_fault_t __xe_bo_cpu_fault(struct vm_fault *vmf, struct xe_device *xe, struct xe_bo *bo) +{ + vm_fault_t ret; + + trace_xe_bo_cpu_fault(bo); + + ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, + TTM_BO_VM_NUM_PREFAULT); + /* + * When TTM is actually called to insert PTEs, ensure no blocking conditions + * remain, in which case TTM may drop locks and return VM_FAULT_RETRY. + */ + xe_assert(xe, ret != VM_FAULT_RETRY); + + if (ret == VM_FAULT_NOPAGE && + mem_type_is_vram(bo->ttm.resource->mem_type)) { + mutex_lock(&xe->mem_access.vram_userfault.lock); + if (list_empty(&bo->vram_userfault_link)) + list_add(&bo->vram_userfault_link, + &xe->mem_access.vram_userfault.list); + mutex_unlock(&xe->mem_access.vram_userfault.lock); + } + + return ret; +} + +static vm_fault_t xe_err_to_fault_t(int err) +{ + switch (err) { + case 0: + case -EINTR: + case -ERESTARTSYS: + case -EAGAIN: + return VM_FAULT_NOPAGE; + case -ENOMEM: + case -ENOSPC: + return VM_FAULT_OOM; + default: + break; + } + return VM_FAULT_SIGBUS; +} + +static bool xe_ttm_bo_is_imported(struct ttm_buffer_object *tbo) +{ + dma_resv_assert_held(tbo->base.resv); + + return tbo->ttm && + (tbo->ttm->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE)) == + TTM_TT_FLAG_EXTERNAL; +} + +static vm_fault_t xe_bo_cpu_fault_fastpath(struct vm_fault *vmf, struct xe_device *xe, + struct xe_bo *bo, bool needs_rpm) +{ + struct ttm_buffer_object *tbo = &bo->ttm; + vm_fault_t ret = VM_FAULT_RETRY; + struct xe_validation_ctx ctx; + struct ttm_operation_ctx tctx = { + .interruptible = true, + .no_wait_gpu = true, + .gfp_retry_mayfail = true, + + }; + int err; + + if (needs_rpm && !xe_pm_runtime_get_if_active(xe)) + return VM_FAULT_RETRY; + + err = xe_validation_ctx_init(&ctx, &xe->val, NULL, + (struct xe_val_flags) { + .interruptible = true, + .no_block = true + }); + if (err) + goto out_pm; + + if (!dma_resv_trylock(tbo->base.resv)) + goto out_validation; + + /* + * Reject CPU faults to purgeable BOs. DONTNEED BOs can be purged + * at any time, and purged BOs have no backing store. Either case + * is undefined behavior for CPU access. + */ + if (xe_bo_madv_is_dontneed(bo) || xe_bo_is_purged(bo)) { + ret = VM_FAULT_SIGBUS; + goto out_unlock; + } + + if (xe_ttm_bo_is_imported(tbo)) { + ret = VM_FAULT_SIGBUS; + drm_dbg(&xe->drm, "CPU trying to access an imported buffer object.\n"); + goto out_unlock; + } + + err = xe_bo_fault_migrate(bo, &tctx, NULL); + if (err) { + /* Return VM_FAULT_RETRY on these errors. */ + if (err != -ENOMEM && err != -ENOSPC && err != -EBUSY) + ret = xe_err_to_fault_t(err); + goto out_unlock; + } + + if (dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL)) + ret = __xe_bo_cpu_fault(vmf, xe, bo); + +out_unlock: + dma_resv_unlock(tbo->base.resv); +out_validation: + xe_validation_ctx_fini(&ctx); +out_pm: + if (needs_rpm) + xe_pm_runtime_put(xe); + + return ret; +} + +static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) +{ + struct ttm_buffer_object *tbo = vmf->vma->vm_private_data; + struct drm_device *ddev = tbo->base.dev; + struct xe_device *xe = to_xe_device(ddev); + struct xe_bo *bo = ttm_to_xe_bo(tbo); + bool needs_rpm = bo->flags & XE_BO_FLAG_VRAM_MASK; + bool retry_after_wait = false; + struct xe_validation_ctx ctx; + struct drm_exec exec; + vm_fault_t ret; + int err = 0; + int idx; + + if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx)) + return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); + + ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); + if (ret != VM_FAULT_RETRY) + goto out; + + if (fault_flag_allow_retry_first(vmf->flags)) { + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + goto out; + retry_after_wait = true; + xe_bo_get(bo); + mmap_read_unlock(vmf->vma->vm_mm); + } else { + ret = VM_FAULT_NOPAGE; + } + + /* + * The fastpath failed and we were not required to return and retry immediately. + * We're now running in one of two modes: + * + * 1) retry_after_wait == true: The mmap_read_lock() is dropped, and we're trying + * to resolve blocking waits. But we can't resolve the fault since the + * mmap_read_lock() is dropped. After retrying the fault, the aim is that the fastpath + * should succeed. But it may fail since we drop the bo lock. + * + * 2) retry_after_wait == false: The fastpath failed, typically even after + * a retry. Do whatever's necessary to resolve the fault. + * + * This construct is recommended to avoid excessive waits under the mmap_lock. + */ + + if (needs_rpm) + xe_pm_runtime_get(xe); + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, + err) { + struct ttm_operation_ctx tctx = { + .interruptible = true, + .no_wait_gpu = false, + .gfp_retry_mayfail = retry_after_wait, + }; + + err = drm_exec_lock_obj(&exec, &tbo->base); + drm_exec_retry_on_contention(&exec); + if (err) + break; + + /* + * Reject CPU faults to purgeable BOs. DONTNEED BOs can be + * purged at any time, and purged BOs have no backing store. + */ + if (xe_bo_madv_is_dontneed(bo) || xe_bo_is_purged(bo)) { + err = -EFAULT; + break; + } + + if (xe_ttm_bo_is_imported(tbo)) { + err = -EFAULT; + drm_dbg(&xe->drm, "CPU trying to access an imported buffer object.\n"); + break; + } + + err = xe_bo_fault_migrate(bo, &tctx, &exec); + if (err) { + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + + err = xe_bo_wait_usage_kernel(bo, &tctx); + if (err) + break; + + if (!retry_after_wait) + ret = __xe_bo_cpu_fault(vmf, xe, bo); + } + /* if retry_after_wait == true, we *must* return VM_FAULT_RETRY. */ + if (err && !retry_after_wait) + ret = xe_err_to_fault_t(err); + + if (needs_rpm) + xe_pm_runtime_put(xe); + + if (retry_after_wait) + xe_bo_put(bo); +out: + drm_dev_exit(idx); + + return ret; +} + +static int xe_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct ttm_buffer_object *ttm_bo = vma->vm_private_data; + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct xe_device *xe = xe_bo_device(bo); + + guard(xe_pm_runtime)(xe); + return ttm_bo_vm_access(vma, addr, buf, len, write); +} + +/** + * xe_bo_read() - Read from an xe_bo + * @bo: The buffer object to read from. + * @offset: The byte offset to start reading from. + * @dst: Location to store the read. + * @size: Size in bytes for the read. + * + * Read @size bytes from the @bo, starting from @offset, storing into @dst. + * + * Return: Zero on success, or negative error. + */ +int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size) +{ + int ret; + + ret = ttm_bo_access(&bo->ttm, offset, dst, size, 0); + if (ret >= 0 && ret != size) + ret = -EIO; + else if (ret == size) + ret = 0; + + return ret; +} + +static const struct vm_operations_struct xe_gem_vm_ops = { + .fault = xe_bo_cpu_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = xe_bo_vm_access, +}; + +static int xe_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + int err = 0; + + /* + * Reject mmap of purgeable BOs. DONTNEED BOs can be purged + * at any time, making CPU access undefined behavior. Purged BOs have + * no backing store and are permanently invalid. + */ + err = xe_bo_lock(bo, true); + if (err) + return err; + + if (xe_bo_madv_is_dontneed(bo)) + err = -EBUSY; + else if (xe_bo_is_purged(bo)) + err = -EINVAL; + xe_bo_unlock(bo); + if (err) + return err; + + return drm_gem_ttm_mmap(obj, vma); +} + +static const struct drm_gem_object_funcs xe_gem_object_funcs = { + .free = xe_gem_object_free, + .close = xe_gem_object_close, + .mmap = xe_gem_object_mmap, + .export = xe_gem_prime_export, + .vm_ops = &xe_gem_vm_ops, +}; + +/** + * xe_bo_alloc - Allocate storage for a struct xe_bo + * + * This function is intended to allocate storage to be used for input + * to __xe_bo_create_locked(), in the case a pointer to the bo to be + * created is needed before the call to __xe_bo_create_locked(). + * If __xe_bo_create_locked ends up never to be called, then the + * storage allocated with this function needs to be freed using + * xe_bo_free(). + * + * Return: A pointer to an uninitialized struct xe_bo on success, + * ERR_PTR(-ENOMEM) on error. + */ +struct xe_bo *xe_bo_alloc(void) +{ + struct xe_bo *bo = kzalloc_obj(*bo); + + if (!bo) + return ERR_PTR(-ENOMEM); + + return bo; +} + +/** + * xe_bo_free - Free storage allocated using xe_bo_alloc() + * @bo: The buffer object storage. + * + * Refer to xe_bo_alloc() documentation for valid use-cases. + */ +void xe_bo_free(struct xe_bo *bo) +{ + kfree(bo); +} + +/** + * xe_bo_init_locked() - Initialize or create an xe_bo. + * @xe: The xe device. + * @bo: An already allocated buffer object or NULL + * if the function should allocate a new one. + * @tile: The tile to select for migration of this bo, and the tile used for + * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos. + * @resv: Pointer to a locked shared reservation object to use for this bo, + * or NULL for the xe_bo to use its own. + * @bulk: The bulk move to use for LRU bumping, or NULL for external bos. + * @size: The storage size to use for the bo. + * @cpu_caching: The cpu caching used for system memory backing store. + * @type: The TTM buffer object type. + * @flags: XE_BO_FLAG_ flags. + * @dma_buf: The dma-buf to reference for the BO lifetime (imported BOs), + * or NULL. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Initialize or create an xe buffer object. On failure, any allocated buffer + * object passed in @bo will have been unreferenced. + * + * Return: The buffer object on success. Negative error pointer on failure. + */ +struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, + struct xe_tile *tile, struct dma_resv *resv, + struct ttm_lru_bulk_move *bulk, size_t size, + u16 cpu_caching, enum ttm_bo_type type, + u32 flags, struct dma_buf *dma_buf, + struct drm_exec *exec) +{ + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + .gfp_retry_mayfail = true, + }; + struct ttm_placement *placement; + uint32_t alignment; + size_t aligned_size; + int err; + + /* Only kernel objects should set GT */ + xe_assert(xe, !tile || type == ttm_bo_type_kernel); + + if (XE_WARN_ON(!size)) { + xe_bo_free(bo); + return ERR_PTR(-EINVAL); + } + + /* XE_BO_FLAG_GGTTx requires XE_BO_FLAG_GGTT also be set */ + if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT)) { + xe_bo_free(bo); + return ERR_PTR(-EINVAL); + } + + if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) && + !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) && + ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) || + (flags & (XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M | + XE_BO_FLAG_NEEDS_1G)))) { + size_t align; + + if (flags & XE_BO_FLAG_NEEDS_1G) + align = SZ_1G; + else if (flags & XE_BO_FLAG_NEEDS_2M) + align = SZ_2M; + else + align = SZ_64K; + + aligned_size = ALIGN(size, align); + if (type != ttm_bo_type_device) + size = ALIGN(size, align); + flags |= XE_BO_FLAG_INTERNAL_64K; + alignment = align >> PAGE_SHIFT; + } else { + aligned_size = ALIGN(size, SZ_4K); + flags &= ~XE_BO_FLAG_INTERNAL_64K; + alignment = SZ_4K >> PAGE_SHIFT; + } + + if (type == ttm_bo_type_device && aligned_size != size) { + xe_bo_free(bo); + return ERR_PTR(-EINVAL); + } + + if (!bo) { + bo = xe_bo_alloc(); + if (IS_ERR(bo)) + return bo; + } + + bo->ccs_cleared = false; + bo->tile = tile; + bo->flags = flags; + bo->cpu_caching = cpu_caching; + bo->ttm.base.funcs = &xe_gem_object_funcs; + bo->ttm.priority = XE_BO_PRIORITY_NORMAL; + INIT_LIST_HEAD(&bo->pinned_link); +#ifdef CONFIG_PROC_FS + INIT_LIST_HEAD(&bo->client_link); +#endif + INIT_LIST_HEAD(&bo->vram_userfault_link); + + /* Initialize purge advisory state */ + bo->purgeable.state = XE_MADV_PURGEABLE_WILLNEED; + + drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size); + + if (resv) { + ctx.allow_res_evict = !(flags & XE_BO_FLAG_NO_RESV_EVICT); + ctx.resv = resv; + } + + xe_validation_assert_exec(xe, exec, &bo->ttm.base); + if (!(flags & XE_BO_FLAG_FIXED_PLACEMENT)) { + err = __xe_bo_placement_for_flags(xe, bo, bo->flags, type); + if (WARN_ON(err)) { + xe_ttm_bo_destroy(&bo->ttm); + return ERR_PTR(err); + } + } + + /* Defer populating type_sg bos */ + placement = (type == ttm_bo_type_sg || + bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement : + &bo->placement; + + /* + * For imported BOs, keep the exporter dma-buf alive for the BO + * lifetime. Taken before ttm_bo_init_reserved() to also cover a + * creation failure there. Released in xe_ttm_bo_destroy(). + */ + if (dma_buf) { + get_dma_buf(dma_buf); + bo->dma_buf = dma_buf; + } + + err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type, + placement, alignment, + &ctx, NULL, resv, xe_ttm_bo_destroy); + if (err) + return ERR_PTR(err); + + /* + * The VRAM pages underneath are potentially still being accessed by the + * GPU, as per async GPU clearing and async evictions. However TTM makes + * sure to add any corresponding move/clear fences into the objects + * dma-resv using the DMA_RESV_USAGE_KERNEL slot. + * + * For KMD internal buffers we don't care about GPU clearing, however we + * still need to handle async evictions, where the VRAM is still being + * accessed by the GPU. Most internal callers are not expecting this, + * since they are missing the required synchronisation before accessing + * the memory. To keep things simple just sync wait any kernel fences + * here, if the buffer is designated KMD internal. + * + * For normal userspace objects we should already have the required + * pipelining or sync waiting elsewhere, since we already have to deal + * with things like async GPU clearing. + */ + if (type == ttm_bo_type_kernel) { + long timeout = dma_resv_wait_timeout(bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL, + ctx.interruptible, + MAX_SCHEDULE_TIMEOUT); + + if (timeout < 0) { + if (!resv) + dma_resv_unlock(bo->ttm.base.resv); + xe_bo_put(bo); + return ERR_PTR(timeout); + } + } + + bo->created = true; + if (bulk) + ttm_bo_set_bulk_move(&bo->ttm, bulk); + else + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + + return bo; +} + +static int __xe_bo_fixed_placement(struct xe_device *xe, + struct xe_bo *bo, enum ttm_bo_type type, + u32 flags, + u64 start, u64 end, u64 size) +{ + struct ttm_place *place = bo->placements; + u32 vram_flag, vram_stolen_flags; + + /* + * to allow fixed placement in GGTT of a VF, post-migration fixups would have to + * include selecting a new fixed offset and shifting the page ranges for it + */ + xe_assert(xe, !IS_SRIOV_VF(xe) || !(bo->flags & XE_BO_FLAG_GGTT)); + + if (flags & (XE_BO_FLAG_USER | XE_BO_FLAG_SYSTEM)) + return -EINVAL; + + vram_flag = flags & XE_BO_FLAG_VRAM_MASK; + vram_stolen_flags = (flags & (XE_BO_FLAG_STOLEN)) | vram_flag; + + /* check if more than one VRAM/STOLEN flag is set */ + if (hweight32(vram_stolen_flags) > 1) + return -EINVAL; + + place->flags = TTM_PL_FLAG_CONTIGUOUS; + place->fpfn = start >> PAGE_SHIFT; + place->lpfn = end >> PAGE_SHIFT; + + if (flags & XE_BO_FLAG_STOLEN) + place->mem_type = XE_PL_STOLEN; + else + place->mem_type = bo_vram_flags_to_vram_placement(xe, flags, vram_flag, type); + + bo->placement = (struct ttm_placement) { + .num_placement = 1, + .placement = place, + }; + + return 0; +} + +static struct xe_bo * +__xe_bo_create_locked(struct xe_device *xe, + struct xe_tile *tile, struct xe_vm *vm, + size_t size, u64 start, u64 end, + u16 cpu_caching, enum ttm_bo_type type, u32 flags, + u64 alignment, struct drm_exec *exec) +{ + struct xe_bo *bo = NULL; + int err; + + if (vm) + xe_vm_assert_held(vm); + + if (start || end != ~0ULL) { + bo = xe_bo_alloc(); + if (IS_ERR(bo)) + return bo; + + flags |= XE_BO_FLAG_FIXED_PLACEMENT; + err = __xe_bo_fixed_placement(xe, bo, type, flags, start, end, size); + if (err) { + xe_bo_free(bo); + return ERR_PTR(err); + } + } + + bo = xe_bo_init_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL, + vm && !xe_vm_in_fault_mode(vm) && + flags & XE_BO_FLAG_USER ? + &vm->lru_bulk_move : NULL, size, + cpu_caching, type, flags, NULL, exec); + if (IS_ERR(bo)) + return bo; + + bo->min_align = alignment; + + /* + * Note that instead of taking a reference no the drm_gpuvm_resv_bo(), + * to ensure the shared resv doesn't disappear under the bo, the bo + * will keep a reference to the vm, and avoid circular references + * by having all the vm's bo refereferences released at vm close + * time. + */ + if (vm && xe_bo_is_user(bo)) + xe_vm_get(vm); + bo->vm = vm; + + if (bo->flags & XE_BO_FLAG_GGTT) { + struct xe_tile *t; + u8 id; + + if (!(bo->flags & XE_BO_FLAG_GGTT_ALL)) { + if (!tile && flags & XE_BO_FLAG_STOLEN) + tile = xe_device_get_root_tile(xe); + + xe_assert(xe, tile); + } + + for_each_tile(t, xe, id) { + if (t != tile && !(bo->flags & XE_BO_FLAG_GGTTx(t))) + continue; + + if (flags & XE_BO_FLAG_FIXED_PLACEMENT) { + err = xe_ggtt_insert_bo_at(t->mem.ggtt, bo, + start + xe_bo_size(bo), U64_MAX, + exec); + } else { + err = xe_ggtt_insert_bo(t->mem.ggtt, bo, exec); + } + if (err) + goto err_unlock_put_bo; + } + } + + trace_xe_bo_create(bo); + return bo; + +err_unlock_put_bo: + __xe_bo_unset_bulk_move(bo); + xe_bo_unlock_vm_held(bo); + xe_bo_put(bo); + return ERR_PTR(err); +} + +/** + * xe_bo_create_locked() - Create a BO + * @xe: The xe device. + * @tile: The tile to select for migration of this bo, and the tile used for + * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos. + * @vm: The local vm or NULL for external objects. + * @size: The storage size to use for the bo. + * @type: The TTM buffer object type. + * @flags: XE_BO_FLAG_ flags. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Create a locked xe BO with no range- nor alignment restrictions. + * + * Return: The buffer object on success. Negative error pointer on failure. + */ +struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags, + struct drm_exec *exec) +{ + return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type, + flags, 0, exec); +} + +static struct xe_bo *xe_bo_create_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, u16 cpu_caching, + enum ttm_bo_type type, u32 flags, + u64 alignment, bool intr) +{ + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_bo *bo; + int ret = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = intr}, + ret) { + bo = __xe_bo_create_locked(xe, tile, NULL, size, 0, ~0ULL, + cpu_caching, type, flags, alignment, &exec); + drm_exec_retry_on_contention(&exec); + if (IS_ERR(bo)) { + ret = PTR_ERR(bo); + xe_validation_retry_on_oom(&ctx, &ret); + } else { + xe_bo_unlock(bo); + } + } + + return ret ? ERR_PTR(ret) : bo; +} + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +static void xe_bo_debug_mixed_mode_cur_index_advance(struct xe_device *xe, struct xe_bo *bo) +{ + if (!xe_debug_page_size_mode_is_mixed(xe)) + return; + + if (!(bo->flags & XE_BO_FLAG_VRAM_MASK) || + !(bo->flags & XE_BO_FLAG_USER)) + return; + + mutex_lock(&xe->page_size_alloc_ctrl.lock); + if (xe->page_size_alloc_ctrl.mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED) + xe->page_size_alloc_ctrl.cur_index++; + mutex_unlock(&xe->page_size_alloc_ctrl.lock); +} + +static bool xe_size_align_overflows(size_t size, size_t align) +{ + return size > SIZE_MAX - (align - 1); +} + +static u32 get_flag_from_cur_index_in_mixed_mode(struct xe_device *xe, size_t *align_size, + int *err) +{ + static const struct { + u32 flag; + size_t align; + } map[] = { + { 0, SZ_4K }, /* default: 4K, no flag */ + { XE_BO_FLAG_NEEDS_64K, SZ_64K }, + { XE_BO_FLAG_NEEDS_2M, SZ_2M }, + { XE_BO_FLAG_NEEDS_1G, SZ_1G }, + }; + u32 idx; + const typeof(*map) *entry; + + lockdep_assert_held(&xe->page_size_alloc_ctrl.lock); + + *err = 0; + idx = xe->page_size_alloc_ctrl.cur_index % ARRAY_SIZE(map); + + entry = &map[idx]; + + if (!entry->flag) + return 0; + + if (xe_size_align_overflows(*align_size, entry->align)) { + *err = -EINVAL; + return 0; + } + *align_size = ALIGN(*align_size, entry->align); + + return entry->flag; +} + +static int xe_bo_apply_debug_page_size_policy(struct xe_device *xe, + u32 *bo_flags, + size_t *size) +{ + enum xe_page_size_alloc_ctrl_mode mode; + u32 want = 0; + size_t align_size = *size; + int err = 0; + + /* + * The debug page-size policy is only meaningful for BOs placed in + * VRAM, where the downstream BO init path can + * actually honor the corresponding minimum page-size requirement. + */ + if (!(*bo_flags & XE_BO_FLAG_VRAM_MASK)) + return 0; + + /* + * Do not override existing page-size requirement flags, since they + * may reflect functional requirements for specific BO types. + */ + if (*bo_flags & (XE_BO_FLAG_NEEDS_64K | + XE_BO_FLAG_NEEDS_2M | + XE_BO_FLAG_NEEDS_1G)) + return 0; + + if (!READ_ONCE(xe->page_size_alloc_ctrl.mode)) + return 0; + + mutex_lock(&xe->page_size_alloc_ctrl.lock); + + mode = xe->page_size_alloc_ctrl.mode; + if (mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE) { + goto out_unlock; + } else if (mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M) { + if (xe_size_align_overflows(align_size, SZ_2M)) { + err = -EINVAL; + goto out_unlock; + } + want = XE_BO_FLAG_NEEDS_2M; + align_size = ALIGN(align_size, SZ_2M); + } else if (mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G) { + if (xe_size_align_overflows(align_size, SZ_1G)) { + err = -EINVAL; + goto out_unlock; + } + want = XE_BO_FLAG_NEEDS_1G; + align_size = ALIGN(align_size, SZ_1G); + } else if (mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED) { + want = get_flag_from_cur_index_in_mixed_mode(xe, &align_size, &err); + if (err) + goto out_unlock; + } else { + goto out_unlock; + } + + mutex_unlock(&xe->page_size_alloc_ctrl.lock); + + *bo_flags |= want; + /* + * Apply the debug page-size policy by rounding the user BO size up to + * the selected granularity. + */ + *size = align_size; + return err; + +out_unlock: + mutex_unlock(&xe->page_size_alloc_ctrl.lock); + return err; +} +#else +static int xe_bo_apply_debug_page_size_policy(struct xe_device *xe, + u32 *bo_flags, + size_t *size) +{ + return 0; +} + +static void xe_bo_debug_mixed_mode_cur_index_advance(struct xe_device *xe, + struct xe_bo *bo) +{ +} +#endif + +/** + * xe_bo_create_user() - Create a user BO + * @xe: The xe device. + * @vm: The local vm or NULL for external objects. + * @size: The storage size to use for the bo. + * @cpu_caching: The caching mode to be used for system backing store. + * @flags: XE_BO_FLAG_ flags. + * @exec: The drm_exec transaction to use for exhaustive eviction, or NULL + * if such a transaction should be initiated by the call. + * + * Create a bo on behalf of user-space. + * + * Return: The buffer object on success. Negative error pointer on failure. + */ +struct xe_bo *xe_bo_create_user(struct xe_device *xe, + struct xe_vm *vm, size_t size, + u16 cpu_caching, + u32 flags, struct drm_exec *exec) +{ + struct xe_bo *bo; + int err = 0; + + flags |= XE_BO_FLAG_USER; + + if (xe_debug_page_size_mode_not_none(xe)) { + err = xe_bo_apply_debug_page_size_policy(xe, &flags, &size); + if (err) + return ERR_PTR(err); + } + + if (vm || exec) { + xe_assert(xe, exec); + bo = __xe_bo_create_locked(xe, NULL, vm, size, 0, ~0ULL, + cpu_caching, ttm_bo_type_device, + flags, 0, exec); + if (!IS_ERR(bo)) + xe_bo_unlock_vm_held(bo); + } else { + bo = xe_bo_create_novm(xe, NULL, size, cpu_caching, + ttm_bo_type_device, flags, 0, true); + } + + return bo; +} + +/** + * xe_bo_create_pin_range_novm() - Create and pin a BO with range options. + * @xe: The xe device. + * @tile: The tile to select for migration of this bo, and the tile used for + * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos. + * @size: The storage size to use for the bo. + * @start: Start of fixed VRAM range or 0. + * @end: End of fixed VRAM range or ~0ULL. + * @type: The TTM buffer object type. + * @flags: XE_BO_FLAG_ flags. + * + * Create an Xe BO with range- and options. If @start and @end indicate + * a fixed VRAM range, this must be a ttm_bo_type_kernel bo with VRAM placement + * only. + * + * Return: The buffer object on success. Negative error pointer on failure. + */ +struct xe_bo *xe_bo_create_pin_range_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, u64 start, u64 end, + enum ttm_bo_type type, u32 flags) +{ + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_bo *bo; + int err = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { + bo = __xe_bo_create_locked(xe, tile, NULL, size, start, end, + 0, type, flags, 0, &exec); + if (IS_ERR(bo)) { + drm_exec_retry_on_contention(&exec); + err = PTR_ERR(bo); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + + err = xe_bo_pin(bo, &exec); + xe_bo_unlock(bo); + if (err) { + xe_bo_put(bo); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + } + + return err ? ERR_PTR(err) : bo; +} + +static struct xe_bo *xe_bo_create_pin_map_at_aligned(struct xe_device *xe, + struct xe_tile *tile, + struct xe_vm *vm, + size_t size, u64 offset, + enum ttm_bo_type type, u32 flags, + u64 alignment, struct drm_exec *exec) +{ + struct xe_bo *bo; + int err; + u64 start = offset == ~0ull ? 0 : offset; + u64 end = offset == ~0ull ? ~0ull : start + size; + + if (flags & XE_BO_FLAG_STOLEN && + xe_ttm_stolen_cpu_access_needs_ggtt(xe)) + flags |= XE_BO_FLAG_GGTT; + + bo = __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type, + flags | XE_BO_FLAG_NEEDS_CPU_ACCESS | XE_BO_FLAG_PINNED, + alignment, exec); + if (IS_ERR(bo)) + return bo; + + err = xe_bo_pin(bo, exec); + if (err) + goto err_put; + + err = xe_bo_vmap(bo); + if (err) + goto err_unpin; + + xe_bo_unlock_vm_held(bo); + + return bo; + +err_unpin: + xe_bo_unpin(bo); +err_put: + xe_bo_unlock_vm_held(bo); + xe_bo_put(bo); + return ERR_PTR(err); +} + +/** + * xe_bo_create_pin_map_at_novm() - Create pinned and mapped bo at optional VRAM offset + * @xe: The xe device. + * @tile: The tile to select for migration of this bo, and the tile used for + * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos. + * @size: The storage size to use for the bo. + * @offset: Optional VRAM offset or %~0ull for don't care. + * @type: The TTM buffer object type. + * @flags: XE_BO_FLAG_ flags. + * @alignment: GGTT alignment. + * @intr: Whether to execute any waits for backing store interruptible. + * + * Create a pinned and optionally mapped bo with VRAM offset and GGTT alignment + * options. The bo will be external and not associated with a VM. + * + * Return: The buffer object on success. Negative error pointer on failure. + * In particular, the function may return ERR_PTR(%-EINTR) if @intr was set + * to true on entry. + */ +struct xe_bo * +xe_bo_create_pin_map_at_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, u64 offset, enum ttm_bo_type type, u32 flags, + u64 alignment, bool intr) +{ + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_bo *bo; + int ret = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = intr}, + ret) { + bo = xe_bo_create_pin_map_at_aligned(xe, tile, NULL, size, offset, + type, flags, alignment, &exec); + if (IS_ERR(bo)) { + drm_exec_retry_on_contention(&exec); + ret = PTR_ERR(bo); + xe_validation_retry_on_oom(&ctx, &ret); + } + } + + return ret ? ERR_PTR(ret) : bo; +} + +/** + * xe_bo_create_pin_map() - Create pinned and mapped bo + * @xe: The xe device. + * @tile: The tile to select for migration of this bo, and the tile used for + * @vm: The vm to associate the buffer object with. The vm's resv must be locked + * with the transaction represented by @exec. + * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos. + * @size: The storage size to use for the bo. + * @type: The TTM buffer object type. + * @flags: XE_BO_FLAG_ flags. + * @exec: The drm_exec transaction to use for exhaustive eviction, and + * previously used for locking @vm's resv. + * + * Create a pinned and mapped bo. The bo will be external and not associated + * with a VM. + * + * Return: The buffer object on success. Negative error pointer on failure. + * In particular, the function may return ERR_PTR(%-EINTR) if @exec was + * configured for interruptible locking. + */ +struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags, + struct drm_exec *exec) +{ + return xe_bo_create_pin_map_at_aligned(xe, tile, vm, size, ~0ull, type, flags, + 0, exec); +} + +/** + * xe_bo_create_pin_map_novm() - Create pinned and mapped bo + * @xe: The xe device. + * @tile: The tile to select for migration of this bo, and the tile used for + * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos. + * @size: The storage size to use for the bo. + * @type: The TTM buffer object type. + * @flags: XE_BO_FLAG_ flags. + * @intr: Whether to execute any waits for backing store interruptible. + * + * Create a pinned and mapped bo. The bo will be external and not associated + * with a VM. + * + * Return: The buffer object on success. Negative error pointer on failure. + * In particular, the function may return ERR_PTR(%-EINTR) if @intr was set + * to true on entry. + */ +struct xe_bo *xe_bo_create_pin_map_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, enum ttm_bo_type type, u32 flags, + bool intr) +{ + return xe_bo_create_pin_map_at_novm(xe, tile, size, ~0ull, type, flags, 0, intr); +} + +static void __xe_bo_unpin_map_no_vm(void *arg) +{ + xe_bo_unpin_map_no_vm(arg); +} + +struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, + size_t size, u32 flags) +{ + struct xe_bo *bo; + int ret; + + KUNIT_STATIC_STUB_REDIRECT(xe_managed_bo_create_pin_map, xe, tile, size, flags); + bo = xe_bo_create_pin_map_novm(xe, tile, size, ttm_bo_type_kernel, flags, true); + if (IS_ERR(bo)) + return bo; + + ret = devm_add_action_or_reset(xe->drm.dev, __xe_bo_unpin_map_no_vm, bo); + if (ret) + return ERR_PTR(ret); + + return bo; +} + +void xe_managed_bo_unpin_map_no_vm(struct xe_bo *bo) +{ + devm_release_action(xe_bo_device(bo)->drm.dev, __xe_bo_unpin_map_no_vm, bo); +} + +struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, + const void *data, size_t size, u32 flags) +{ + struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags); + + if (IS_ERR(bo)) + return bo; + + xe_map_memcpy_to(xe, &bo->vmap, 0, data, size); + + return bo; +} + +/** + * xe_managed_bo_reinit_in_vram + * @xe: xe device + * @tile: Tile where the new buffer will be created + * @src: Managed buffer object allocated in system memory + * + * Replace a managed src buffer object allocated in system memory with a new + * one allocated in vram, copying the data between them. + * Buffer object in VRAM is not going to have the same GGTT address, the caller + * is responsible for making sure that any old references to it are updated. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src) +{ + struct xe_bo *bo; + u32 dst_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT; + + dst_flags |= (*src)->flags & (XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + + xe_assert(xe, IS_DGFX(xe)); + xe_assert(xe, !(*src)->vmap.is_iomem); + + bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr, + xe_bo_size(*src), dst_flags); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + devm_release_action(xe->drm.dev, __xe_bo_unpin_map_no_vm, *src); + *src = bo; + + return 0; +} + +/* + * XXX: This is in the VM bind data path, likely should calculate this once and + * store, with a recalculation if the BO is moved. + */ +uint64_t vram_region_gpu_offset(struct ttm_resource *res) +{ + struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); + + switch (res->mem_type) { + case XE_PL_STOLEN: + return xe_ttm_stolen_gpu_offset(xe); + case XE_PL_TT: + case XE_PL_SYSTEM: + return 0; + default: + return xe_map_resource_to_region(res)->dpa_base; + } + return 0; +} + +/** + * xe_bo_pin_external - pin an external BO + * @bo: buffer object to be pinned + * @in_place: Pin in current placement, don't attempt to migrate. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD) + * BO. Unique call compared to xe_bo_pin as this function has it own set of + * asserts and code to ensure evict / restore on suspend / resume. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec) +{ + struct xe_device *xe = xe_bo_device(bo); + int err; + + xe_assert(xe, !bo->vm); + xe_assert(xe, xe_bo_is_user(bo)); + + if (!xe_bo_is_pinned(bo)) { + if (!in_place) { + err = xe_bo_validate(bo, NULL, false, exec); + if (err) + return err; + } + + spin_lock(&xe->pinned.lock); + list_add_tail(&bo->pinned_link, &xe->pinned.late.external); + spin_unlock(&xe->pinned.lock); + } + + ttm_bo_pin(&bo->ttm); + if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) + xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); + + /* + * FIXME: If we always use the reserve / unreserve functions for locking + * we do not need this. + */ + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + + return 0; +} + +/** + * xe_bo_pin() - Pin a kernel bo after potentially migrating it + * @bo: The kernel bo to pin. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Attempts to migrate a bo to @bo->placement. If that succeeds, + * pins the bo. + * + * Return: %0 on success, negative error code on migration failure. + */ +int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec) +{ + struct ttm_place *place = &bo->placements[0]; + struct xe_device *xe = xe_bo_device(bo); + int err; + + /* We currently don't expect user BO to be pinned */ + xe_assert(xe, !xe_bo_is_user(bo)); + + /* Pinned object must be in GGTT or have pinned flag */ + xe_assert(xe, bo->flags & (XE_BO_FLAG_PINNED | + XE_BO_FLAG_GGTT)); + + /* + * No reason we can't support pinning imported dma-bufs we just don't + * expect to pin an imported dma-buf. + */ + xe_assert(xe, !bo->ttm.base.import_attach); + + /* We only expect at most 1 pin */ + xe_assert(xe, !xe_bo_is_pinned(bo)); + + err = xe_bo_validate(bo, NULL, false, exec); + if (err) + return err; + + if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) { + spin_lock(&xe->pinned.lock); + if (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE) + list_add_tail(&bo->pinned_link, &xe->pinned.late.kernel_bo_present); + else + list_add_tail(&bo->pinned_link, &xe->pinned.early.kernel_bo_present); + spin_unlock(&xe->pinned.lock); + } + + ttm_bo_pin(&bo->ttm); + if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) + xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); + + /* + * FIXME: If we always use the reserve / unreserve functions for locking + * we do not need this. + */ + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + + return 0; +} + +/** + * xe_bo_unpin_external - unpin an external BO + * @bo: buffer object to be unpinned + * + * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD) + * BO. Unique call compared to xe_bo_unpin as this function has it own set of + * asserts and code to ensure evict / restore on suspend / resume. + * + * Returns 0 for success, negative error code otherwise. + */ +void xe_bo_unpin_external(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + xe_assert(xe, !bo->vm); + xe_assert(xe, xe_bo_is_pinned(bo)); + xe_assert(xe, xe_bo_is_user(bo)); + + spin_lock(&xe->pinned.lock); + if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link)) + list_del_init(&bo->pinned_link); + spin_unlock(&xe->pinned.lock); + + ttm_bo_unpin(&bo->ttm); + if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) + xe_ttm_tt_account_add(xe, bo->ttm.ttm); + + /* + * FIXME: If we always use the reserve / unreserve functions for locking + * we do not need this. + */ + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); +} + +void xe_bo_unpin(struct xe_bo *bo) +{ + struct ttm_place *place = &bo->placements[0]; + struct xe_device *xe = xe_bo_device(bo); + + xe_assert(xe, !bo->ttm.base.import_attach); + xe_assert(xe, xe_bo_is_pinned(bo)); + + if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) { + spin_lock(&xe->pinned.lock); + xe_assert(xe, !list_empty(&bo->pinned_link)); + list_del_init(&bo->pinned_link); + spin_unlock(&xe->pinned.lock); + + if (bo->backup_obj) { + if (xe_bo_is_pinned(bo->backup_obj)) + ttm_bo_unpin(&bo->backup_obj->ttm); + xe_bo_put(bo->backup_obj); + bo->backup_obj = NULL; + } + } + ttm_bo_unpin(&bo->ttm); + if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) + xe_ttm_tt_account_add(xe, bo->ttm.ttm); +} + +/** + * xe_bo_validate() - Make sure the bo is in an allowed placement + * @bo: The bo, + * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or + * NULL. Used together with @allow_res_evict. + * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's + * reservation object. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Make sure the bo is in allowed placement, migrating it if necessary. If + * needed, other bos will be evicted. If bos selected for eviction shares + * the @vm's reservation object, they can be evicted iff @allow_res_evict is + * set to true, otherwise they will be bypassed. + * + * Return: 0 on success, negative error code on failure. May return + * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal. + */ +int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict, + struct drm_exec *exec) +{ + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + .gfp_retry_mayfail = true, + }; + int ret; + + if (xe_bo_is_pinned(bo)) + return 0; + + if (vm) { + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + ctx.allow_res_evict = allow_res_evict; + ctx.resv = xe_vm_resv(vm); + } + + xe_vm_set_validating(vm, allow_res_evict); + trace_xe_bo_validate(bo); + xe_validation_assert_exec(xe_bo_device(bo), exec, &bo->ttm.base); + ret = ttm_bo_validate(&bo->ttm, &bo->placement, &ctx); + xe_vm_clear_validating(vm, allow_res_evict); + + return ret; +} + +bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo) +{ + if (bo->destroy == &xe_ttm_bo_destroy) + return true; + + return false; +} + +/* + * Resolve a BO address. There is no assert to check if the proper lock is held + * so it should only be used in cases where it is not fatal to get the wrong + * address, such as printing debug information, but not in cases where memory is + * written based on this result. + */ +dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_res_cursor cur; + u64 page; + + xe_assert(xe, page_size <= PAGE_SIZE); + page = offset >> PAGE_SHIFT; + offset &= (PAGE_SIZE - 1); + + if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) { + xe_assert(xe, bo->ttm.ttm); + + xe_res_first_sg(xe_bo_sg(bo), page << PAGE_SHIFT, + page_size, &cur); + return xe_res_dma(&cur) + offset; + } else { + struct xe_res_cursor cur; + + xe_res_first(bo->ttm.resource, page << PAGE_SHIFT, + page_size, &cur); + return cur.start + offset + vram_region_gpu_offset(bo->ttm.resource); + } +} + +dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size) +{ + if (!READ_ONCE(bo->ttm.pin_count)) + xe_bo_assert_held(bo); + return __xe_bo_addr(bo, offset, page_size); +} + +int xe_bo_vmap(struct xe_bo *bo) +{ + struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); + void *virtual; + bool is_iomem; + int ret; + + xe_bo_assert_held(bo); + + if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) || + !force_contiguous(bo->flags))) + return -EINVAL; + + if (!iosys_map_is_null(&bo->vmap)) + return 0; + + /* + * We use this more or less deprecated interface for now since + * ttm_bo_vmap() doesn't offer the optimization of kmapping + * single page bos, which is done here. + * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap + * to use struct iosys_map. + */ + ret = ttm_bo_kmap(&bo->ttm, 0, xe_bo_size(bo) >> PAGE_SHIFT, &bo->kmap); + if (ret) + return ret; + + virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); + if (is_iomem) + iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual); + else + iosys_map_set_vaddr(&bo->vmap, virtual); + + return 0; +} + +static void __xe_bo_vunmap(struct xe_bo *bo) +{ + if (!iosys_map_is_null(&bo->vmap)) { + iosys_map_clear(&bo->vmap); + ttm_bo_kunmap(&bo->kmap); + } +} + +void xe_bo_vunmap(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + __xe_bo_vunmap(bo); +} + +static int gem_create_set_pxp_type(struct xe_device *xe, struct xe_bo *bo, u64 value) +{ + if (value == DRM_XE_PXP_TYPE_NONE) + return 0; + + /* we only support DRM_XE_PXP_TYPE_HWDRM for now */ + if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) + return -EINVAL; + + return xe_pxp_key_assign(xe->pxp, bo); +} + +typedef int (*xe_gem_create_set_property_fn)(struct xe_device *xe, + struct xe_bo *bo, + u64 value); + +static const xe_gem_create_set_property_fn gem_create_set_property_funcs[] = { + [DRM_XE_GEM_CREATE_SET_PROPERTY_PXP_TYPE] = gem_create_set_pxp_type, +}; + +static int gem_create_user_ext_set_property(struct xe_device *xe, + struct xe_bo *bo, + u64 extension) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.property >= + ARRAY_SIZE(gem_create_set_property_funcs)) || + XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.property != DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(gem_create_set_property_funcs)); + if (!gem_create_set_property_funcs[idx]) + return -EINVAL; + + return gem_create_set_property_funcs[idx](xe, bo, ext.value); +} + +typedef int (*xe_gem_create_user_extension_fn)(struct xe_device *xe, + struct xe_bo *bo, + u64 extension); + +static const xe_gem_create_user_extension_fn gem_create_user_extension_funcs[] = { + [DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY] = gem_create_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 16 +static int gem_create_user_extensions(struct xe_device *xe, struct xe_bo *bo, + u64 extensions, int ext_number) +{ + u64 __user *address = u64_to_user_ptr(extensions); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(gem_create_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, + ARRAY_SIZE(gem_create_user_extension_funcs)); + err = gem_create_user_extension_funcs[idx](xe, bo, extensions); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (ext.next_extension) + return gem_create_user_extensions(xe, bo, ext.next_extension, + ++ext_number); + + return 0; +} + +int xe_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_gem_create *args = data; + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_vm *vm = NULL; + struct xe_bo *bo; + unsigned int bo_flags; + u32 handle; + int err; + + if (XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + /* at least one valid memory placement must be specified */ + if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) || + !args->placement)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->flags & + ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING | + DRM_XE_GEM_CREATE_FLAG_SCANOUT | + DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM | + DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->handle)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, !args->size)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK)) + return -EINVAL; + + bo_flags = 0; + if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING) + bo_flags |= XE_BO_FLAG_DEFER_BACKING; + + /* + * Display scanout is always non-coherent with the CPU cache. + */ + if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) + bo_flags |= XE_BO_FLAG_FORCE_WC; + + if (args->flags & DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION) { + if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20)) + return -EOPNOTSUPP; + bo_flags |= XE_BO_FLAG_NO_COMPRESSION; + } + + bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1); + + /* CCS formats need physical placement at a 64K alignment in VRAM. */ + if ((bo_flags & XE_BO_FLAG_VRAM_MASK) && + (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) && + !(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) && + IS_ALIGNED(args->size, SZ_64K)) + bo_flags |= XE_BO_FLAG_NEEDS_64K; + + if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) { + if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_FLAG_VRAM_MASK))) + return -EINVAL; + + bo_flags |= XE_BO_FLAG_NEEDS_CPU_ACCESS; + } + + if (XE_IOCTL_DBG(xe, !args->cpu_caching || + args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_VRAM_MASK && + args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_FORCE_WC && + args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) + return -EINVAL; + + if (args->vm_id) { + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -ENOENT; + } + + err = 0; + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, + err) { + if (vm) { + err = xe_vm_drm_exec_lock(vm, &exec); + drm_exec_retry_on_contention(&exec); + if (err) + break; + } + bo = xe_bo_create_user(xe, vm, args->size, args->cpu_caching, + bo_flags, &exec); + drm_exec_retry_on_contention(&exec); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + } + if (err) + goto out_vm; + + if (args->extensions) { + err = gem_create_user_extensions(xe, bo, args->extensions, 0); + if (err) + goto out_bulk; + } + + err = drm_gem_handle_create(file, &bo->ttm.base, &handle); + if (err) + goto out_bulk; + + xe_bo_debug_mixed_mode_cur_index_advance(xe, bo); + + args->handle = handle; + goto out_put; + +out_bulk: + if (vm && !xe_vm_in_fault_mode(vm)) { + xe_vm_lock(vm, false); + __xe_bo_unset_bulk_move(bo); + xe_vm_unlock(vm); + } +out_put: + xe_bo_put(bo); +out_vm: + if (vm) + xe_vm_put(vm); + + return err; +} + +int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct drm_xe_gem_mmap_offset *args = data; + struct drm_gem_object *gem_obj; + + if (XE_IOCTL_DBG(xe, args->extensions) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->flags & + ~DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER)) + return -EINVAL; + + if (args->flags & DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER) { + if (XE_IOCTL_DBG(xe, !IS_DGFX(xe))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->handle)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, PAGE_SIZE > SZ_4K)) + return -EINVAL; + + BUILD_BUG_ON(((XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT) + + SZ_4K) >= DRM_FILE_PAGE_OFFSET_START); + args->offset = XE_PCI_BARRIER_MMAP_OFFSET; + return 0; + } + + gem_obj = drm_gem_object_lookup(file, args->handle); + if (XE_IOCTL_DBG(xe, !gem_obj)) + return -ENOENT; + + /* The mmap offset was set up at BO allocation time. */ + args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); + + xe_bo_put(gem_to_xe_bo(gem_obj)); + return 0; +} + +/** + * xe_bo_decompress - schedule in-place decompress and install fence + * @bo: buffer object (caller should hold drm_exec reservations for VM+BO) + * + * Schedules an in-place resolve via the migrate layer and installs the + * returned dma_fence into the BO kernel reservation slot (DMA_RESV_USAGE_KERNEL). + * In preempt fence mode, this operation interrupts hardware execution + * which is expensive. Page fault mode is recommended for better performance. + * + * The resolve path only runs for VRAM-backed buffers (currently dGPU-only); + * iGPU/system-memory objects fail the resource check and bypass the resolve. + * + * Returns 0 on success, negative errno on error. + */ +int xe_bo_decompress(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct dma_fence *decomp_fence = NULL; + struct ttm_operation_ctx op_ctx = { + .interruptible = true, + .no_wait_gpu = false, + .gfp_retry_mayfail = false, + }; + int err = 0; + + /* Silently skip decompression for non-VRAM buffers */ + if (!bo->ttm.resource || !mem_type_is_vram(bo->ttm.resource->mem_type)) + return 0; + + /* Notify before scheduling resolve */ + err = xe_bo_move_notify(bo, &op_ctx); + if (err) + return err; + + /* Reserve fence slot before scheduling */ + err = dma_resv_reserve_fences(bo->ttm.base.resv, 1); + if (err) + return err; + + /* Schedule the in-place decompression */ + decomp_fence = xe_migrate_resolve(tile->migrate, + bo, + bo->ttm.resource); + + if (IS_ERR(decomp_fence)) + return PTR_ERR(decomp_fence); + + /* Install kernel-usage fence */ + dma_resv_add_fence(bo->ttm.base.resv, decomp_fence, DMA_RESV_USAGE_KERNEL); + dma_fence_put(decomp_fence); + + return 0; +} + +/** + * xe_bo_lock() - Lock the buffer object's dma_resv object + * @bo: The struct xe_bo whose lock is to be taken + * @intr: Whether to perform any wait interruptible + * + * Locks the buffer object's dma_resv object. If the buffer object is + * pointing to a shared dma_resv object, that shared lock is locked. + * + * Return: 0 on success, -EINTR if @intr is true and the wait for a + * contended lock was interrupted. If @intr is set to false, the + * function always returns 0. + */ +int xe_bo_lock(struct xe_bo *bo, bool intr) +{ + if (intr) + return dma_resv_lock_interruptible(bo->ttm.base.resv, NULL); + + dma_resv_lock(bo->ttm.base.resv, NULL); + + return 0; +} + +/** + * xe_bo_unlock() - Unlock the buffer object's dma_resv object + * @bo: The struct xe_bo whose lock is to be released. + * + * Unlock a buffer object lock that was locked by xe_bo_lock(). + */ +void xe_bo_unlock(struct xe_bo *bo) +{ + dma_resv_unlock(bo->ttm.base.resv); +} + +/** + * xe_bo_can_migrate - Whether a buffer object likely can be migrated + * @bo: The buffer object to migrate + * @mem_type: The TTM memory type intended to migrate to + * + * Check whether the buffer object supports migration to the + * given memory type. Note that pinning may affect the ability to migrate as + * returned by this function. + * + * This function is primarily intended as a helper for checking the + * possibility to migrate buffer objects and can be called without + * the object lock held. + * + * Return: true if migration is possible, false otherwise. + */ +bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type) +{ + unsigned int cur_place; + + if (bo->ttm.type == ttm_bo_type_kernel) + return true; + + if (bo->ttm.type == ttm_bo_type_sg) + return false; + + for (cur_place = 0; cur_place < bo->placement.num_placement; + cur_place++) { + if (bo->placements[cur_place].mem_type == mem_type) + return true; + } + + return false; +} + +static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place) +{ + memset(place, 0, sizeof(*place)); + place->mem_type = mem_type; +} + +/** + * xe_bo_migrate - Migrate an object to the desired region id + * @bo: The buffer object to migrate. + * @mem_type: The TTM region type to migrate to. + * @tctx: A pointer to a struct ttm_operation_ctx or NULL if + * a default interruptibe ctx is to be used. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Attempt to migrate the buffer object to the desired memory region. The + * buffer object may not be pinned, and must be locked. + * On successful completion, the object memory type will be updated, + * but an async migration task may not have completed yet, and to + * accomplish that, the object's kernel fences must be signaled with + * the object lock held. + * + * Return: 0 on success. Negative error code on failure. In particular may + * return -EINTR or -ERESTARTSYS if signal pending. + */ +int xe_bo_migrate(struct xe_bo *bo, u32 mem_type, struct ttm_operation_ctx *tctx, + struct drm_exec *exec) +{ + struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + .gfp_retry_mayfail = true, + }; + struct ttm_placement placement; + struct ttm_place requested; + + xe_bo_assert_held(bo); + tctx = tctx ? tctx : &ctx; + + if (bo->ttm.resource->mem_type == mem_type) + return 0; + + if (xe_bo_is_pinned(bo)) + return -EBUSY; + + if (!xe_bo_can_migrate(bo, mem_type)) + return -EINVAL; + + xe_place_from_ttm_type(mem_type, &requested); + placement.num_placement = 1; + placement.placement = &requested; + + /* + * Stolen needs to be handled like below VRAM handling if we ever need + * to support it. + */ + drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN); + + if (mem_type_is_vram(mem_type)) { + u32 c = 0; + + add_vram(xe, bo, &requested, bo->flags, mem_type, &c); + } + + if (!tctx->no_wait_gpu) + xe_validation_assert_exec(xe_bo_device(bo), exec, &bo->ttm.base); + return ttm_bo_validate(&bo->ttm, &placement, tctx); +} + +/** + * xe_bo_evict - Evict an object to evict placement + * @bo: The buffer object to migrate. + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * On successful completion, the object memory will be moved to evict + * placement. This function blocks until the object has been fully moved. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_bo_evict(struct xe_bo *bo, struct drm_exec *exec) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + .gfp_retry_mayfail = true, + }; + struct ttm_placement placement; + int ret; + + xe_evict_flags(&bo->ttm, &placement); + ret = ttm_bo_validate(&bo->ttm, &placement, &ctx); + if (ret) + return ret; + + dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + + return 0; +} + +/** + * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when + * placed in system memory. + * @bo: The xe_bo + * + * Return: true if extra pages need to be allocated, false otherwise. + */ +bool xe_bo_needs_ccs_pages(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) + return false; + + if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device) + return false; + + /* On discrete GPUs, if the GPU can access this buffer from + * system memory (i.e., it allows XE_PL_TT placement), FlatCCS + * can't be used since there's no CCS storage associated with + * non-VRAM addresses. + */ + if (IS_DGFX(xe) && (bo->flags & XE_BO_FLAG_SYSTEM)) + return false; + + /* Check if userspace explicitly requested no compression */ + if (bo->flags & XE_BO_FLAG_NO_COMPRESSION) + return false; + + /* + * For WB (Write-Back) CPU caching mode, check if the device + * supports WB compression with coherency. + */ + if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB && + xe->pat.idx[XE_CACHE_WB_COMPRESSION] == XE_PAT_INVALID_IDX) + return false; + + return true; +} + +/** + * __xe_bo_release_dummy() - Dummy kref release function + * @kref: The embedded struct kref. + * + * Dummy release function for xe_bo_put_deferred(). Keep off. + */ +void __xe_bo_release_dummy(struct kref *kref) +{ +} + +/** + * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred(). + * @deferred: The lockless list used for the call to xe_bo_put_deferred(). + * + * Puts all bos whose put was deferred by xe_bo_put_deferred(). + * The @deferred list can be either an onstack local list or a global + * shared list used by a workqueue. + */ +void xe_bo_put_commit(struct llist_head *deferred) +{ + struct llist_node *freed; + struct xe_bo *bo, *next; + + if (!deferred) + return; + + freed = llist_del_all(deferred); + if (!freed) + return; + + llist_for_each_entry_safe(bo, next, freed, freed) + drm_gem_object_free(&bo->ttm.base.refcount); +} + +static void xe_bo_dev_work_func(struct work_struct *work) +{ + struct xe_bo_dev *bo_dev = container_of(work, typeof(*bo_dev), async_free); + + xe_bo_put_commit(&bo_dev->async_list); +} + +/** + * xe_bo_dev_init() - Initialize BO dev to manage async BO freeing + * @bo_dev: The BO dev structure + */ +void xe_bo_dev_init(struct xe_bo_dev *bo_dev) +{ + INIT_WORK(&bo_dev->async_free, xe_bo_dev_work_func); +} + +/** + * xe_bo_dev_fini() - Finalize BO dev managing async BO freeing + * @bo_dev: The BO dev structure + */ +void xe_bo_dev_fini(struct xe_bo_dev *bo_dev) +{ + flush_work(&bo_dev->async_free); +} + +void xe_bo_put(struct xe_bo *bo) +{ + struct xe_tile *tile; + u8 id; + + might_sleep(); + if (bo) { +#ifdef CONFIG_PROC_FS + if (bo->client) + might_lock(&bo->client->bos_lock); +#endif + for_each_tile(tile, xe_bo_device(bo), id) + if (bo->ggtt_node[id]) + xe_ggtt_might_lock(tile->mem.ggtt); + drm_gem_object_put(&bo->ttm.base); + } +} + +/** + * xe_bo_dumb_create - Create a dumb bo as backing for a fb + * @file_priv: ... + * @dev: ... + * @args: ... + * + * See dumb_create() hook in include/drm/drm_drv.h + * + * Return: ... + */ +int xe_bo_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_bo *bo; + uint32_t handle; + int err; + u32 page_size = max_t(u32, PAGE_SIZE, + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K); + + err = drm_mode_size_dumb(dev, args, SZ_64, page_size); + if (err) + return err; + + bo = xe_bo_create_user(xe, NULL, args->size, + DRM_XE_GEM_CPU_CACHING_WC, + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_NEEDS_CPU_ACCESS, NULL); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle); + /* drop reference from allocate - handle holds it now */ + drm_gem_object_put(&bo->ttm.base); + if (!err) + args->handle = handle; + return err; +} + +void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo) +{ + struct ttm_buffer_object *tbo = &bo->ttm; + struct ttm_device *bdev = tbo->bdev; + + drm_vma_node_unmap(&tbo->base.vma_node, bdev->dev_mapping); + + list_del_init(&bo->vram_userfault_link); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_bo.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h new file mode 100644 index 000000000..e8081af5b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -0,0 +1,617 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_BO_H_ +#define _XE_BO_H_ + +#include +#include + +#include "xe_bo_types.h" +#include "xe_ggtt.h" +#include "xe_macros.h" +#include "xe_validation.h" +#include "xe_vm_types.h" +#include "xe_vm.h" +#include "xe_vram_types.h" + +#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */ + +#define XE_BO_FLAG_USER BIT(0) +/* The bits below need to be contiguous, or things break */ +#define XE_BO_FLAG_SYSTEM BIT(1) +#define XE_BO_FLAG_VRAM0 BIT(2) +#define XE_BO_FLAG_VRAM1 BIT(3) +#define XE_BO_FLAG_VRAM_MASK (XE_BO_FLAG_VRAM0 | XE_BO_FLAG_VRAM1) +/* -- */ +#define XE_BO_FLAG_STOLEN BIT(4) +#define XE_BO_FLAG_VRAM(vram) (XE_BO_FLAG_VRAM0 << ((vram)->id)) +#define XE_BO_FLAG_VRAM_IF_DGFX(tile) (IS_DGFX(tile_to_xe(tile)) ? \ + XE_BO_FLAG_VRAM((tile)->mem.vram) : \ + XE_BO_FLAG_SYSTEM) +#define XE_BO_FLAG_GGTT BIT(5) +#define XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE BIT(6) +#define XE_BO_FLAG_PINNED BIT(7) +#define XE_BO_FLAG_NO_RESV_EVICT BIT(8) +#define XE_BO_FLAG_DEFER_BACKING BIT(9) +#define XE_BO_FLAG_FORCE_WC BIT(10) +#define XE_BO_FLAG_FIXED_PLACEMENT BIT(11) +#define XE_BO_FLAG_PAGETABLE BIT(12) +#define XE_BO_FLAG_NEEDS_CPU_ACCESS BIT(13) +#define XE_BO_FLAG_NEEDS_UC BIT(14) +#define XE_BO_FLAG_NEEDS_64K BIT(15) +#define XE_BO_FLAG_NEEDS_2M BIT(16) +#define XE_BO_FLAG_GGTT_INVALIDATE BIT(17) +#define XE_BO_FLAG_PINNED_NORESTORE BIT(18) +#define XE_BO_FLAG_PINNED_LATE_RESTORE BIT(19) +#define XE_BO_FLAG_GGTT0 BIT(20) +#define XE_BO_FLAG_GGTT1 BIT(21) +#define XE_BO_FLAG_GGTT2 BIT(22) +#define XE_BO_FLAG_GGTT3 BIT(23) +#define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24) +#define XE_BO_FLAG_FORCE_USER_VRAM BIT(25) +#define XE_BO_FLAG_NO_COMPRESSION BIT(26) +#define XE_BO_FLAG_NEEDS_1G BIT(27) + +/* this one is trigger internally only */ +#define XE_BO_FLAG_INTERNAL_TEST BIT(30) +#define XE_BO_FLAG_INTERNAL_64K BIT(31) + +#define XE_BO_FLAG_GGTT_ALL (XE_BO_FLAG_GGTT0 | \ + XE_BO_FLAG_GGTT1 | \ + XE_BO_FLAG_GGTT2 | \ + XE_BO_FLAG_GGTT3) + +#define XE_BO_FLAG_GGTTx(tile) \ + (XE_BO_FLAG_GGTT0 << (tile)->id) + +#define XE_PTE_SHIFT 12 +#define XE_PAGE_SIZE (1 << XE_PTE_SHIFT) +#define XE_PTE_MASK (XE_PAGE_SIZE - 1) +#define XE_PDE_SHIFT (XE_PTE_SHIFT - 3) +#define XE_PDES (1 << XE_PDE_SHIFT) +#define XE_PDE_MASK (XE_PDES - 1) + +#define XE_64K_PTE_SHIFT 16 +#define XE_64K_PAGE_SIZE (1 << XE_64K_PTE_SHIFT) +#define XE_64K_PTE_MASK (XE_64K_PAGE_SIZE - 1) +#define XE_64K_PDE_MASK (XE_PDE_MASK >> 4) + +#define XE_PL_SYSTEM TTM_PL_SYSTEM +#define XE_PL_TT TTM_PL_TT +#define XE_PL_VRAM0 TTM_PL_VRAM +#define XE_PL_VRAM1 (XE_PL_VRAM0 + 1) +#define XE_PL_STOLEN (TTM_NUM_MEM_TYPES - 1) + +#define XE_BO_PROPS_INVALID (-1) + +#define XE_PCI_BARRIER_MMAP_OFFSET (0x50 << XE_PTE_SHIFT) + +/** + * enum xe_madv_purgeable_state - Buffer object purgeable state enumeration + * + * This enum defines the possible purgeable states for a buffer object, + * allowing userspace to provide memory usage hints to the kernel for + * better memory management under pressure. + * + * @XE_MADV_PURGEABLE_WILLNEED: The buffer object is needed and should not be purged. + * This is the default state. + * @XE_MADV_PURGEABLE_DONTNEED: The buffer object is not currently needed and can be + * purged by the kernel under memory pressure. + * @XE_MADV_PURGEABLE_PURGED: The buffer object has been purged by the kernel. + * + * Accessing a purged buffer will result in an error. Per i915 semantics, + * once purged, a BO remains permanently invalid and must be destroyed and recreated. + */ +enum xe_madv_purgeable_state { + XE_MADV_PURGEABLE_WILLNEED, + XE_MADV_PURGEABLE_DONTNEED, + XE_MADV_PURGEABLE_PURGED, +}; + +struct sg_table; + +struct xe_bo *xe_bo_alloc(void); +void xe_bo_free(struct xe_bo *bo); + +struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, + struct xe_tile *tile, struct dma_resv *resv, + struct ttm_lru_bulk_move *bulk, size_t size, + u16 cpu_caching, enum ttm_bo_type type, + u32 flags, struct dma_buf *dma_buf, + struct drm_exec *exec); +struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags, + struct drm_exec *exec); +struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_vm *vm, size_t size, + u16 cpu_caching, u32 flags, struct drm_exec *exec); +struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags, + struct drm_exec *exec); +struct xe_bo *xe_bo_create_pin_map_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, enum ttm_bo_type type, u32 flags, + bool intr); +struct xe_bo *xe_bo_create_pin_range_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, u64 start, u64 end, + enum ttm_bo_type type, u32 flags); +struct xe_bo * +xe_bo_create_pin_map_at_novm(struct xe_device *xe, struct xe_tile *tile, + size_t size, u64 offset, enum ttm_bo_type type, + u32 flags, u64 alignment, bool intr); +struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, + size_t size, u32 flags); +void xe_managed_bo_unpin_map_no_vm(struct xe_bo *bo); +struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, + const void *data, size_t size, u32 flags); +int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src); + +int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags, enum ttm_bo_type type); + +static inline struct xe_bo *ttm_to_xe_bo(const struct ttm_buffer_object *bo) +{ + return container_of(bo, struct xe_bo, ttm); +} + +static inline struct xe_bo *gem_to_xe_bo(const struct drm_gem_object *obj) +{ + return container_of(obj, struct xe_bo, ttm.base); +} + +#define xe_bo_device(bo) ttm_to_xe_device((bo)->ttm.bdev) + +static inline struct xe_bo *xe_bo_get(struct xe_bo *bo) +{ + if (bo) + drm_gem_object_get(&bo->ttm.base); + + return bo; +} + +void xe_bo_put(struct xe_bo *bo); + +/* + * xe_bo_get_unless_zero() - Conditionally obtain a GEM object refcount on an + * xe bo + * @bo: The bo for which we want to obtain a refcount. + * + * There is a short window between where the bo's GEM object refcount reaches + * zero and where we put the final ttm_bo reference. Code in the eviction- and + * shrinking path should therefore attempt to grab a gem object reference before + * trying to use members outside of the base class ttm object. This function is + * intended for that purpose. On successful return, this function must be paired + * with an xe_bo_put(). + * + * Return: @bo on success, NULL on failure. + */ +static inline __must_check struct xe_bo *xe_bo_get_unless_zero(struct xe_bo *bo) +{ + if (!bo || !kref_get_unless_zero(&bo->ttm.base.refcount)) + return NULL; + + return bo; +} + +static inline void __xe_bo_unset_bulk_move(struct xe_bo *bo) +{ + if (bo) + ttm_bo_set_bulk_move(&bo->ttm, NULL); +} + +static inline void xe_bo_assert_held(struct xe_bo *bo) +{ + if (bo) + dma_resv_assert_held((bo)->ttm.base.resv); +} + +int xe_bo_lock(struct xe_bo *bo, bool intr); + +void xe_bo_unlock(struct xe_bo *bo); + +static inline void xe_bo_unlock_vm_held(struct xe_bo *bo) +{ + if (bo) { + XE_WARN_ON(bo->vm && bo->ttm.base.resv != xe_vm_resv(bo->vm)); + if (bo->vm) + xe_vm_assert_held(bo->vm); + else + dma_resv_unlock(bo->ttm.base.resv); + } +} + +int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec); +int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec); +void xe_bo_unpin_external(struct xe_bo *bo); +void xe_bo_unpin(struct xe_bo *bo); +int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict, + struct drm_exec *exec); + +static inline bool xe_bo_is_pinned(struct xe_bo *bo) +{ + return bo->ttm.pin_count; +} + +static inline bool xe_bo_is_protected(const struct xe_bo *bo) +{ + return bo->pxp_key_instance; +} + +/** + * xe_bo_is_purged() - Check if buffer object has been purged + * @bo: The buffer object to check + * + * Checks if the buffer object's backing store has been discarded by the + * kernel due to memory pressure after being marked as purgeable (DONTNEED). + * Once purged, the BO cannot be restored and any attempt to use it will fail. + * + * Context: Caller must hold the BO's dma-resv lock + * Return: true if the BO has been purged, false otherwise + */ +static inline bool xe_bo_is_purged(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + return bo->purgeable.state == XE_MADV_PURGEABLE_PURGED; +} + +/** + * xe_bo_madv_is_dontneed() - Check if BO is marked as DONTNEED + * @bo: The buffer object to check + * + * Checks if userspace has marked this BO as DONTNEED (i.e., its contents + * are not currently needed and can be discarded under memory pressure). + * This is used internally to decide whether a BO is eligible for purging. + * + * Context: Caller must hold the BO's dma-resv lock + * Return: true if the BO is marked DONTNEED, false otherwise + */ +static inline bool xe_bo_madv_is_dontneed(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + return bo->purgeable.state == XE_MADV_PURGEABLE_DONTNEED; +} + +void xe_bo_set_purgeable_state(struct xe_bo *bo, enum xe_madv_purgeable_state new_state); + +/** + * xe_bo_willneed_get_locked() - Acquire a WILLNEED holder on a BO + * @bo: Buffer object + * + * Increments willneed_count and, on a 0->1 transition, promotes the BO + * from DONTNEED to WILLNEED. PURGED is terminal and is never modified. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_willneed_get_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + /* Imported BOs are owned externally; do not track purgeability. */ + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + if (bo->purgeable.willneed_count++ == 0 && xe_bo_madv_is_dontneed(bo)) + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_WILLNEED); +} + +/** + * xe_bo_willneed_put_locked() - Release a WILLNEED holder on a BO + * @bo: Buffer object + * + * Decrements willneed_count and, on a 1->0 transition, marks the BO + * DONTNEED only if it still has VMAs (implying all active VMAs are + * DONTNEED). If the last VMA is being removed, preserve the current BO + * state to match the previous VMA-walk semantics. + * + * PURGED is terminal and the BO state is never modified. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_willneed_put_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + xe_assert(xe_bo_device(bo), bo->purgeable.willneed_count > 0); + if (--bo->purgeable.willneed_count == 0 && bo->purgeable.vma_count > 0 && + !xe_bo_is_purged(bo)) + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED); +} + +/** + * xe_bo_vma_count_inc_locked() - Account a new VMA on a BO + * @bo: Buffer object + * + * Increments vma_count. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_vma_count_inc_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + bo->purgeable.vma_count++; +} + +/** + * xe_bo_vma_count_dec_locked() - Account a VMA removal on a BO + * @bo: Buffer object + * + * Decrements vma_count. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_vma_count_dec_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + xe_assert(xe_bo_device(bo), bo->purgeable.vma_count > 0); + bo->purgeable.vma_count--; +} + +static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) +{ + if (likely(bo)) { + xe_bo_lock(bo, false); + xe_bo_unpin(bo); + xe_bo_unlock(bo); + + xe_bo_put(bo); + } +} + +bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo); +dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size); +dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size); + +static inline dma_addr_t +xe_bo_main_addr(struct xe_bo *bo, size_t page_size) +{ + return xe_bo_addr(bo, 0, page_size); +} + +/** + * xe_bo_size() - Xe BO size + * @bo: The bo object. + * + * Simple helper to return Xe BO's size. + * + * Return: Xe BO's size + */ +static inline size_t xe_bo_size(struct xe_bo *bo) +{ + return bo->ttm.base.size; +} + +static inline u32 +__xe_bo_ggtt_addr(struct xe_bo *bo, u8 tile_id) +{ + struct xe_ggtt_node *ggtt_node = bo->ggtt_node[tile_id]; + u64 offset; + + if (XE_WARN_ON(!ggtt_node)) + return 0; + + offset = xe_ggtt_node_addr(ggtt_node); + XE_WARN_ON(offset + xe_bo_size(bo) > (1ull << 32)); + return offset; +} + +static inline u32 +xe_bo_ggtt_addr(struct xe_bo *bo) +{ + xe_assert(xe_bo_device(bo), bo->tile); + + return __xe_bo_ggtt_addr(bo, bo->tile->id); +} + +int xe_bo_vmap(struct xe_bo *bo); +void xe_bo_vunmap(struct xe_bo *bo); +int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size); + +bool mem_type_is_vram(u32 mem_type); +bool xe_bo_is_vram(struct xe_bo *bo); +bool xe_bo_is_visible_vram(struct xe_bo *bo); +bool xe_bo_is_stolen(struct xe_bo *bo); +bool xe_bo_is_stolen_devmem(struct xe_bo *bo); +bool xe_bo_is_vm_bound(struct xe_bo *bo); +bool xe_bo_has_single_placement(struct xe_bo *bo); +uint64_t vram_region_gpu_offset(struct ttm_resource *res); + +bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type); + +int xe_bo_migrate(struct xe_bo *bo, u32 mem_type, struct ttm_operation_ctx *ctc, + struct drm_exec *exec); +int xe_bo_evict(struct xe_bo *bo, struct drm_exec *exec); + +int xe_bo_evict_pinned(struct xe_bo *bo); +int xe_bo_notifier_prepare_pinned(struct xe_bo *bo); +int xe_bo_notifier_unprepare_pinned(struct xe_bo *bo); +int xe_bo_restore_pinned(struct xe_bo *bo); + +int xe_bo_dma_unmap_pinned(struct xe_bo *bo); + +extern const struct ttm_device_funcs xe_ttm_funcs; +extern const char *const xe_mem_type_to_name[]; + +int xe_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo); + +int xe_bo_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args); + +bool xe_bo_needs_ccs_pages(struct xe_bo *bo); + +int xe_bo_decompress(struct xe_bo *bo); + +static inline size_t xe_bo_ccs_pages_start(struct xe_bo *bo) +{ + return PAGE_ALIGN(xe_bo_size(bo)); +} + +/** + * xe_bo_has_valid_ccs_bb - Check if CCS's BBs were setup for the BO. + * @bo: the &xe_bo to check + * + * The CCS's BBs should only be setup by the driver VF, but it is safe + * to call this function also by non-VF driver. + * + * Return: true iff the CCS's BBs are setup, false otherwise. + */ +static inline bool xe_bo_has_valid_ccs_bb(struct xe_bo *bo) +{ + return bo->bb_ccs[XE_SRIOV_VF_CCS_READ_CTX] && + bo->bb_ccs[XE_SRIOV_VF_CCS_WRITE_CTX]; +} + +static inline bool xe_bo_has_pages(struct xe_bo *bo) +{ + if ((bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) || + xe_bo_is_vram(bo)) + return true; + + return false; +} + +void __xe_bo_release_dummy(struct kref *kref); + +/** + * xe_bo_put_deferred() - Put a buffer object with delayed final freeing + * @bo: The bo to put. + * @deferred: List to which to add the buffer object if we cannot put, or + * NULL if the function is to put unconditionally. + * + * Since the final freeing of an object includes both sleeping and (!) + * memory allocation in the dma_resv individualization, it's not ok + * to put an object from atomic context nor from within a held lock + * tainted by reclaim. In such situations we want to defer the final + * freeing until we've exited the restricting context, or in the worst + * case to a workqueue. + * This function either puts the object if possible without the refcount + * reaching zero, or adds it to the @deferred list if that was not possible. + * The caller needs to follow up with a call to xe_bo_put_commit() to actually + * put the bo iff this function returns true. It's safe to always + * follow up with a call to xe_bo_put_commit(). + * TODO: It's TTM that is the villain here. Perhaps TTM should add an + * interface like this. + * + * Return: true if @bo was the first object put on the @freed list, + * false otherwise. + */ +static inline bool +xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred) +{ + if (!deferred) { + xe_bo_put(bo); + return false; + } + + if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy)) + return false; + + return llist_add(&bo->freed, deferred); +} + +void xe_bo_put_commit(struct llist_head *deferred); + +/** + * xe_bo_put_async() - Put BO async + * @bo: The bo to put. + * + * Put BO async, the final put is deferred to a worker to exit an IRQ context. + */ +static inline void +xe_bo_put_async(struct xe_bo *bo) +{ + struct xe_bo_dev *bo_device = &xe_bo_device(bo)->bo_device; + + if (xe_bo_put_deferred(bo, &bo_device->async_list)) + schedule_work(&bo_device->async_free); +} + +void xe_bo_dev_init(struct xe_bo_dev *bo_device); + +void xe_bo_dev_fini(struct xe_bo_dev *bo_device); + +struct sg_table *xe_bo_sg(struct xe_bo *bo); + +/** + * xe_bo_sg_is_contiguous() - Check if a BO's DMA address space is contiguous. + * @bo: the BO to check (must have a valid sg table, i.e. !xe_bo_is_vram()) + * @len: required contiguous length in bytes + * + * Returns true if the first @len bytes of the BO are mapped to a contiguous + * DMA address range. + */ +static inline bool xe_bo_sg_is_contiguous(struct xe_bo *bo, size_t len) +{ + return drm_prime_get_contiguous_size(xe_bo_sg(bo)) >= len; +} + +/* + * xe_sg_segment_size() - Provides upper limit for sg segment size. + * @dev: device pointer + * + * Returns the maximum segment size for the 'struct scatterlist' + * elements. + */ +static inline unsigned int xe_sg_segment_size(struct device *dev) +{ + struct scatterlist __maybe_unused sg; + size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1; + + max = min_t(size_t, max, dma_max_mapping_size(dev)); + + /* + * The iommu_dma_map_sg() function ensures iova allocation doesn't + * cross dma segment boundary. It does so by padding some sg elements. + * This can cause overflow, ending up with sg->length being set to 0. + * Avoid this by ensuring maximum segment size is half of 'max' + * rounded down to PAGE_SIZE. + */ + return round_down(max / 2, PAGE_SIZE); +} + +/** + * struct xe_bo_shrink_flags - flags governing the shrink behaviour. + * @purge: Only purging allowed. Don't shrink if bo not purgeable. + * @writeback: Attempt to immediately move content to swap. + */ +struct xe_bo_shrink_flags { + u32 purge : 1; + u32 writeback : 1; +}; + +long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, + const struct xe_bo_shrink_flags flags, + unsigned long *scanned); + +/** + * xe_bo_is_mem_type - Whether the bo currently resides in the given + * TTM memory type + * @bo: The bo to check. + * @mem_type: The TTM memory type. + * + * Return: true iff the bo resides in @mem_type, false otherwise. + */ +static inline bool xe_bo_is_mem_type(struct xe_bo *bo, u32 mem_type) +{ + xe_bo_assert_held(bo); + return bo->ttm.resource->mem_type == mem_type; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_bo_doc.h b/drivers/gpu/drm/xe/xe_bo_doc.h new file mode 100644 index 000000000..401e7dd26 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_doc.h @@ -0,0 +1,179 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BO_DOC_H_ +#define _XE_BO_DOC_H_ + +/** + * DOC: Buffer Objects (BO) + * + * BO management + * ============= + * + * TTM manages (placement, eviction, etc...) all BOs in Xe. + * + * BO creation + * =========== + * + * Create a chunk of memory which can be used by the GPU. Placement rules + * (sysmem or vram region) passed in upon creation. TTM handles placement of BO + * and can trigger eviction of other BOs to make space for the new BO. + * + * Kernel BOs + * ---------- + * + * A kernel BO is created as part of driver load (e.g. uC firmware images, GuC + * ADS, etc...) or a BO created as part of a user operation which requires + * a kernel BO (e.g. engine state, memory for page tables, etc...). These BOs + * are typically mapped in the GGTT (any kernel BOs aside memory for page tables + * are in the GGTT), are pinned (can't move or be evicted at runtime), have a + * vmap (Xe can access the memory via xe_map layer) and have contiguous physical + * memory. + * + * More details of why kernel BOs are pinned and contiguous below. + * + * User BOs + * -------- + * + * A user BO is created via the DRM_IOCTL_XE_GEM_CREATE IOCTL. Once it is + * created the BO can be mmap'd (via DRM_IOCTL_XE_GEM_MMAP_OFFSET) for user + * access and it can be bound for GPU access (via DRM_IOCTL_XE_VM_BIND). All + * user BOs are evictable and user BOs are never pinned by Xe. The allocation of + * the backing store can be deferred from creation time until first use which is + * either mmap, bind, or pagefault. + * + * Private BOs + * ~~~~~~~~~~~ + * + * A private BO is a user BO created with a valid VM argument passed into the + * create IOCTL. If a BO is private it cannot be exported via prime FD and + * mappings can only be created for the BO within the VM it is tied to. Lastly, + * the BO dma-resv slots / lock point to the VM's dma-resv slots / lock (all + * private BOs to a VM share common dma-resv slots / lock). + * + * External BOs + * ~~~~~~~~~~~~ + * + * An external BO is a user BO created with a NULL VM argument passed into the + * create IOCTL. An external BO can be shared with different UMDs / devices via + * prime FD and the BO can be mapped into multiple VMs. An external BO has its + * own unique dma-resv slots / lock. An external BO will be in an array of all + * VMs which has a mapping of the BO. This allows VMs to lookup and lock all + * external BOs mapped in the VM as needed. + * + * BO placement + * ~~~~~~~~~~~~ + * + * When a user BO is created, a mask of valid placements is passed indicating + * which memory regions are considered valid. + * + * The memory region information is available via query uAPI (TODO: add link). + * + * BO validation + * ============= + * + * BO validation (ttm_bo_validate) refers to ensuring a BO has a valid + * placement. If a BO was swapped to temporary storage, a validation call will + * trigger a move back to a valid (location where GPU can access BO) placement. + * Validation of a BO may evict other BOs to make room for the BO being + * validated. + * + * BO eviction / moving + * ==================== + * + * All eviction (or in other words, moving a BO from one memory location to + * another) is routed through TTM with a callback into Xe. + * + * Runtime eviction + * ---------------- + * + * Runtime evictions refers to during normal operations where TTM decides it + * needs to move a BO. Typically this is because TTM needs to make room for + * another BO and the evicted BO is first BO on LRU list that is not locked. + * + * An example of this is a new BO which can only be placed in VRAM but there is + * not space in VRAM. There could be multiple BOs which have sysmem and VRAM + * placement rules which currently reside in VRAM, TTM trigger a will move of + * one (or multiple) of these BO(s) until there is room in VRAM to place the new + * BO. The evicted BO(s) are valid but still need new bindings before the BO + * used again (exec or compute mode rebind worker). + * + * Another example would be, TTM can't find a BO to evict which has another + * valid placement. In this case TTM will evict one (or multiple) unlocked BO(s) + * to a temporary unreachable (invalid) placement. The evicted BO(s) are invalid + * and before next use need to be moved to a valid placement and rebound. + * + * In both cases, moves of these BOs are scheduled behind the fences in the BO's + * dma-resv slots. + * + * WW locking tries to ensures if 2 VMs use 51% of the memory forward progress + * is made on both VMs. + * + * Runtime eviction uses per a GT migration engine (TODO: link to migration + * engine doc) to do a GPU memcpy from one location to another. + * + * Rebinds after runtime eviction + * ------------------------------ + * + * When BOs are moved, every mapping (VMA) of the BO needs to rebound before + * the BO is used again. Every VMA is added to an evicted list of its VM when + * the BO is moved. This is safe because of the VM locking structure (TODO: link + * to VM locking doc). On the next use of a VM (exec or compute mode rebind + * worker) the evicted VMA list is checked and rebinds are triggered. In the + * case of faulting VM, the rebind is done in the page fault handler. + * + * Suspend / resume eviction of VRAM + * --------------------------------- + * + * During device suspend / resume VRAM may lose power which means the contents + * of VRAM's memory is blown away. Thus BOs present in VRAM at the time of + * suspend must be moved to sysmem in order for their contents to be saved. + * + * A simple TTM call (ttm_resource_manager_evict_all) can move all non-pinned + * (user) BOs to sysmem. External BOs that are pinned need to be manually + * evicted with a simple loop + xe_bo_evict call. It gets a little trickier + * with kernel BOs. + * + * Some kernel BOs are used by the GT migration engine to do moves, thus we + * can't move all of the BOs via the GT migration engine. For simplity, use a + * TTM memcpy (CPU) to move any kernel (pinned) BO on either suspend or resume. + * + * Some kernel BOs need to be restored to the exact same physical location. TTM + * makes this rather easy but the caveat is the memory must be contiguous. Again + * for simplity, we enforce that all kernel (pinned) BOs are contiguous and + * restored to the same physical location. + * + * Pinned external BOs in VRAM are restored on resume via the GPU. + * + * Rebinds after suspend / resume + * ------------------------------ + * + * Most kernel BOs have GGTT mappings which must be restored during the resume + * process. All user BOs are rebound after validation on their next use. + * + * Future work + * =========== + * + * Trim the list of BOs which is saved / restored via TTM memcpy on suspend / + * resume. All we really need to save / restore via TTM memcpy is the memory + * required for the GuC to load and the memory for the GT migrate engine to + * operate. + * + * Do not require kernel BOs to be contiguous in physical memory / restored to + * the same physical address on resume. In all likelihood the only memory that + * needs to be restored to the same physical address is memory used for page + * tables. All of that memory is allocated 1 page at time so the contiguous + * requirement isn't needed. Some work on the vmap code would need to be done if + * kernel BOs are not contiguous too. + * + * Make some kernel BO evictable rather than pinned. An example of this would be + * engine state, in all likelihood if the dma-slots of these BOs where properly + * used rather than pinning we could safely evict + rebind these BOs as needed. + * + * Some kernel BOs do not need to be restored on resume (e.g. GuC ADS as that is + * repopulated on resume), add flag to mark such objects as no save / restore. + */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_bo_evict.c b/drivers/gpu/drm/xe/xe_bo_evict.c new file mode 100644 index 000000000..7661fca7f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_evict.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bo_evict.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_ggtt.h" +#include "xe_tile.h" + +typedef int (*xe_pinned_fn)(struct xe_bo *bo); + +static int xe_bo_apply_to_pinned(struct xe_device *xe, + struct list_head *pinned_list, + struct list_head *new_list, + const xe_pinned_fn pinned_fn) +{ + LIST_HEAD(still_in_list); + struct xe_bo *bo; + int ret = 0; + + spin_lock(&xe->pinned.lock); + while (!ret) { + bo = list_first_entry_or_null(pinned_list, typeof(*bo), + pinned_link); + if (!bo) + break; + xe_bo_get(bo); + list_move_tail(&bo->pinned_link, &still_in_list); + spin_unlock(&xe->pinned.lock); + + ret = pinned_fn(bo); + if (ret && pinned_list != new_list) { + spin_lock(&xe->pinned.lock); + /* + * We might no longer be pinned, since PM notifier can + * call this. If the pinned link is now empty, keep it + * that way. + */ + if (!list_empty(&bo->pinned_link)) + list_move(&bo->pinned_link, pinned_list); + spin_unlock(&xe->pinned.lock); + } + xe_bo_put(bo); + spin_lock(&xe->pinned.lock); + } + list_splice_tail(&still_in_list, new_list); + spin_unlock(&xe->pinned.lock); + + return ret; +} + +/** + * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all + * pinned VRAM objects which need to be saved. + * @xe: xe device + * + * Should be called from PM notifier when preparing for s3/s4. + * + * Return: 0 on success, negative error code on error. + */ +int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe) +{ + int ret; + + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, + &xe->pinned.early.kernel_bo_present, + xe_bo_notifier_prepare_pinned); + if (!ret) + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, + &xe->pinned.late.kernel_bo_present, + xe_bo_notifier_prepare_pinned); + + if (!ret) + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, + &xe->pinned.late.external, + xe_bo_notifier_prepare_pinned); + + return ret; +} + +/** + * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all + * pinned VRAM objects which have been restored. + * @xe: xe device + * + * Should be called from PM notifier after exiting s3/s4 (either on success or + * failure). + */ +void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe) +{ + (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, + &xe->pinned.early.kernel_bo_present, + xe_bo_notifier_unprepare_pinned); + + (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, + &xe->pinned.late.kernel_bo_present, + xe_bo_notifier_unprepare_pinned); + + (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, + &xe->pinned.late.external, + xe_bo_notifier_unprepare_pinned); +} + +/** + * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM + * @xe: xe device + * + * Evict non-pinned user BOs (via GPU). + * + * Evict == move VRAM BOs to temporary (typically system) memory. + */ +int xe_bo_evict_all_user(struct xe_device *xe) +{ + struct ttm_device *bdev = &xe->ttm; + u32 mem_type; + int ret; + + /* User memory */ + for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) { + struct ttm_resource_manager *man = + ttm_manager_type(bdev, mem_type); + + /* + * On igpu platforms with flat CCS we need to ensure we save and restore any CCS + * state since this state lives inside graphics stolen memory which doesn't survive + * hibernation. + * + * This can be further improved by only evicting objects that we know have actually + * used a compression enabled PAT index. + */ + if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe))) + continue; + + if (man) { + ret = ttm_resource_manager_evict_all(bdev, man); + if (ret) + return ret; + } + } + + return 0; +} + +/** + * xe_bo_evict_all - evict all BOs from VRAM + * @xe: xe device + * + * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next + * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU. + * All eviction magic done via TTM calls. + * + * Evict == move VRAM BOs to temporary (typically system) memory. + * + * This function should be called before the device goes into a suspend state + * where the VRAM loses power. + */ +int xe_bo_evict_all(struct xe_device *xe) +{ + struct xe_tile *tile; + u8 id; + int ret; + + ret = xe_bo_evict_all_user(xe); + if (ret) + return ret; + + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, + &xe->pinned.late.external, xe_bo_evict_pinned); + + if (!ret) + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, + &xe->pinned.late.evicted, xe_bo_evict_pinned); + + /* + * Wait for all user BO to be evicted as those evictions depend on the + * memory moved below. + */ + for_each_tile(tile, xe, id) + xe_tile_migrate_wait(tile); + + if (ret) + return ret; + + return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, + &xe->pinned.early.evicted, + xe_bo_evict_pinned); +} + +static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo) +{ + int ret; + + ret = xe_bo_restore_pinned(bo); + if (ret) + return ret; + + if (bo->flags & XE_BO_FLAG_GGTT) { + struct xe_tile *tile; + u8 id; + + for_each_tile(tile, xe_bo_device(bo), id) { + if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile))) + continue; + + xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo); + } + } + + return 0; +} + +/** + * xe_bo_restore_early - restore early phase kernel BOs to VRAM + * + * @xe: xe device + * + * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All + * moves done via TTM calls. + * + * This function should be called early, before trying to init the GT, on device + * resume. + */ +int xe_bo_restore_early(struct xe_device *xe) +{ + return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted, + &xe->pinned.early.kernel_bo_present, + xe_bo_restore_and_map_ggtt); +} + +/** + * xe_bo_restore_late - restore pinned late phase BOs + * + * @xe: xe device + * + * Move pinned user and kernel BOs which can use blitter from temporary + * (typically system) memory to VRAM. All moves done via TTM calls. + * + * This function should be called late, after GT init, on device resume. + */ +int xe_bo_restore_late(struct xe_device *xe) +{ + struct xe_tile *tile; + int ret, id; + + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted, + &xe->pinned.late.kernel_bo_present, + xe_bo_restore_and_map_ggtt); + + for_each_tile(tile, xe, id) + xe_tile_migrate_wait(tile); + + if (ret) + return ret; + + if (!IS_DGFX(xe)) + return 0; + + /* Pinned user memory in VRAM should be validated on resume */ + ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, + &xe->pinned.late.external, + xe_bo_restore_pinned); + + /* Wait for restore to complete */ + for_each_tile(tile, xe, id) + xe_tile_migrate_wait(tile); + + return ret; +} + +static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe) +{ + struct xe_tile *tile; + unsigned int id; + + (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, + &xe->pinned.late.external, + xe_bo_dma_unmap_pinned); + for_each_tile(tile, xe, id) + xe_tile_migrate_wait(tile); +} + +/** + * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed + * @xe: The xe device. + * + * On pci_device removal we need to drop all dma mappings and move + * the data of exported bos out to system. This includes SVM bos and + * exported dma-buf bos. This is done by evicting all bos, but + * the evict placement in xe_evict_flags() is chosen such that all + * bos except those mentioned are purged, and thus their memory + * is released. + * + * For pinned bos, we're unmapping dma. + */ +void xe_bo_pci_dev_remove_all(struct xe_device *xe) +{ + unsigned int mem_type; + + /* + * Move pagemap bos and exported dma-buf to system, and + * purge everything else. + */ + for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) { + struct ttm_resource_manager *man = + ttm_manager_type(&xe->ttm, mem_type); + + if (man) { + int ret = ttm_resource_manager_evict_all(&xe->ttm, man); + + drm_WARN_ON(&xe->drm, ret); + } + } + + xe_bo_pci_dev_remove_pinned(xe); +} + +static void xe_bo_pinned_fini(void *arg) +{ + struct xe_device *xe = arg; + + (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, + &xe->pinned.late.kernel_bo_present, + xe_bo_dma_unmap_pinned); + (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, + &xe->pinned.early.kernel_bo_present, + xe_bo_dma_unmap_pinned); +} + +/** + * xe_bo_pinned_init() - Initialize pinned bo tracking + * @xe: The xe device. + * + * Initializes the lists and locks required for pinned bo + * tracking and registers a callback to dma-unmap + * any remaining pinned bos on pci device removal. + * + * Return: %0 on success, negative error code on error. + */ +int xe_bo_pinned_init(struct xe_device *xe) +{ + spin_lock_init(&xe->pinned.lock); + INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present); + INIT_LIST_HEAD(&xe->pinned.early.evicted); + INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present); + INIT_LIST_HEAD(&xe->pinned.late.evicted); + INIT_LIST_HEAD(&xe->pinned.late.external); + + return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe); +} diff --git a/drivers/gpu/drm/xe/xe_bo_evict.h b/drivers/gpu/drm/xe/xe_bo_evict.h new file mode 100644 index 000000000..e8385cb7f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_evict.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BO_EVICT_H_ +#define _XE_BO_EVICT_H_ + +struct xe_device; + +int xe_bo_evict_all(struct xe_device *xe); +int xe_bo_evict_all_user(struct xe_device *xe); +int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe); +void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe); +int xe_bo_restore_early(struct xe_device *xe); +int xe_bo_restore_late(struct xe_device *xe); + +void xe_bo_pci_dev_remove_all(struct xe_device *xe); + +int xe_bo_pinned_init(struct xe_device *xe); +#endif diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h new file mode 100644 index 000000000..e45f24301 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BO_TYPES_H_ +#define _XE_BO_TYPES_H_ + +#include + +#include +#include +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_ggtt_types.h" + +struct xe_device; +struct xe_mem_pool_node; +struct xe_vm; + +#define XE_BO_MAX_PLACEMENTS 3 + +/* TODO: To be selected with VM_MADVISE */ +#define XE_BO_PRIORITY_NORMAL 1 + +/** + * struct xe_bo - Xe buffer object + */ +struct xe_bo { + /** @ttm: TTM base buffer object */ + struct ttm_buffer_object ttm; + /** @backup_obj: The backup object when pinned and suspended (vram only) */ + struct xe_bo *backup_obj; + /** @parent_obj: Ref to parent bo if this a backup_obj */ + struct xe_bo *parent_obj; + /** @dma_buf: Imported dma-buf ref to keep its resv alive. */ + struct dma_buf *dma_buf; + /** @flags: flags for this buffer object */ + u32 flags; + /** @vm: VM this BO is attached to, for extobj this will be NULL */ + struct xe_vm *vm; + /** @tile: Tile this BO is attached to (kernel BO only) */ + struct xe_tile *tile; + /** @placements: valid placements for this BO */ + struct ttm_place placements[XE_BO_MAX_PLACEMENTS]; + /** @placement: current placement for this BO */ + struct ttm_placement placement; + /** @ggtt_node: Array of GGTT nodes if this BO is mapped in the GGTTs */ + struct xe_ggtt_node *ggtt_node[XE_MAX_TILES_PER_DEVICE]; + /** @vmap: iosys map of this buffer */ + struct iosys_map vmap; + /** @kmap: TTM bo kmap object for internal use only. Keep off. */ + struct ttm_bo_kmap_obj kmap; + /** @pinned_link: link to present / evicted list of pinned BO */ + struct list_head pinned_link; +#ifdef CONFIG_PROC_FS + /** + * @client: @xe_drm_client which created the bo + */ + struct xe_drm_client *client; + /** + * @client_link: Link into @xe_drm_client.objects_list + */ + struct list_head client_link; +#endif + /** @attr: User controlled attributes for bo */ + struct { + /** + * @attr.atomic_access: type of atomic access bo needs + * protected by bo dma-resv lock + */ + u32 atomic_access; + } attr; + /** + * @pxp_key_instance: PXP key instance this BO was created against. A + * 0 in this variable indicates that the BO does not use PXP encryption. + */ + u32 pxp_key_instance; + + /** @freed: List node for delayed put. */ + struct llist_node freed; + /** @update_index: Update index if PT BO */ + int update_index; + /** @created: Whether the bo has passed initial creation */ + bool created; + + /** @ccs_cleared: true means that CCS region of BO is already cleared */ + bool ccs_cleared; + + /** @bb_ccs: BB instructions of CCS read/write. Valid only for VF */ + struct xe_mem_pool_node *bb_ccs[XE_SRIOV_VF_CCS_CTX_COUNT]; + + /** + * @cpu_caching: CPU caching mode. Currently only used for userspace + * objects. Exceptions are system memory on DGFX, which is always + * WB. + */ + u16 cpu_caching; + + /** @devmem_allocation: SVM device memory allocation */ + struct drm_pagemap_devmem devmem_allocation; + + /** @vram_userfault_link: Link into @mem_access.vram_userfault.list */ + struct list_head vram_userfault_link; + + /** + * @min_align: minimum alignment needed for this BO if different + * from default + */ + u64 min_align; + + /** + * @purgeable: Purgeability state and accounting. + * + * All fields are protected by the BO's dma-resv lock. + */ + struct { + /** + * @purgeable.state: BO purgeability state + * (WILLNEED/DONTNEED/PURGED). + */ + u32 state; + + /** + * @purgeable.vma_count: Number of VMAs currently mapping this BO. + */ + u32 vma_count; + + /** + * @purgeable.willneed_count: Number of active WILLNEED holders. + * + * Counts WILLNEED VMAs plus active dma-buf exports for + * non-imported BOs. The BO flips to DONTNEED on a 1->0 + * transition only when VMAs still exist; if the last VMA is + * removed, the previous BO state is preserved. + */ + u32 willneed_count; + } purgeable; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c new file mode 100644 index 000000000..052cce962 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -0,0 +1,1418 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "instructions/xe_mi_commands.h" +#include "xe_configfs.h" +#include "xe_defaults.h" +#include "xe_gt_types.h" +#include "xe_hw_engine_types.h" +#include "xe_module.h" +#include "xe_pci_types.h" +#include "xe_sriov_types.h" + +/** + * DOC: Xe Configfs + * + * Overview + * ======== + * + * Configfs is a filesystem-based manager of kernel objects. Xe KMD registers a + * configfs subsystem called ``xe`` that creates a directory in the mounted + * configfs directory. The user can create devices under this directory and + * configure them as necessary. See Documentation/filesystems/configfs.rst for + * more information about how configfs works. + * + * Create devices + * ============== + * + * To create a device, the ``xe`` module should already be loaded, but some + * attributes can only be set before binding the device. It can be accomplished + * by blocking the driver autoprobe:: + * + * # echo 0 > /sys/bus/pci/drivers_autoprobe + * # modprobe xe + * + * In order to create a device, the user has to create a directory inside ``xe``:: + * + * # mkdir /sys/kernel/config/xe/0000:03:00.0/ + * + * Every device created is populated by the driver with entries that can be + * used to configure it:: + * + * /sys/kernel/config/xe/ + * ├── 0000:00:02.0 + * │   └── ... + * ├── 0000:00:02.1 + * │   └── ... + * : + * └── 0000:03:00.0 + * ├── survivability_mode + * ├── gt_types_allowed + * ├── engines_allowed + * └── enable_psmi + * + * After configuring the attributes as per next section, the device can be + * probed with:: + * + * # echo 0000:03:00.0 > /sys/bus/pci/drivers/xe/bind + * # # or + * # echo 0000:03:00.0 > /sys/bus/pci/drivers_probe + * + * Configure Attributes + * ==================== + * + * Survivability mode: + * ------------------- + * + * Enable survivability mode on supported cards. This setting only takes + * effect when probing the device. Example to enable it:: + * + * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/survivability_mode + * + * This attribute can only be set before binding to the device. + * + * Allowed GT types: + * ----------------- + * + * Allow only specific types of GTs to be detected and initialized by the + * driver. Any combination of GT types can be enabled/disabled, although + * some settings will cause the device to fail to probe. + * + * Writes support both comma- and newline-separated input format. Reads + * will always return one GT type per line. "primary" and "media" are the + * GT type names supported by this interface. + * + * This attribute can only be set before binding to the device. + * + * Examples: + * + * Allow both primary and media GTs to be initialized and used. This matches + * the driver's default behavior:: + * + * # echo 'primary,media' > /sys/kernel/config/xe/0000:03:00.0/gt_types_allowed + * + * Allow only the primary GT of each tile to be initialized and used, + * effectively disabling the media GT if it exists on the platform:: + * + * # echo 'primary' > /sys/kernel/config/xe/0000:03:00.0/gt_types_allowed + * + * Allow only the media GT of each tile to be initialized and used, + * effectively disabling the primary GT. **This configuration will cause + * device probe failure on all current platforms, but may be allowed on + * igpu platforms in the future**:: + * + * # echo 'media' > /sys/kernel/config/xe/0000:03:00.0/gt_types_allowed + * + * Disable all GTs. Only other GPU IP (such as display) is potentially usable. + * **This configuration will cause device probe failure on all current + * platforms, but may be allowed on igpu platforms in the future**:: + * + * # echo '' > /sys/kernel/config/xe/0000:03:00.0/gt_types_allowed + * + * Allowed engines: + * ---------------- + * + * Allow only a set of engine(s) to be available, disabling the other engines + * even if they are available in hardware. This is applied after HW fuses are + * considered on each tile. Examples: + * + * Allow only one render and one copy engines, nothing else:: + * + * # echo 'rcs0,bcs0' > /sys/kernel/config/xe/0000:03:00.0/engines_allowed + * + * Allow only compute engines and first copy engine:: + * + * # echo 'ccs*,bcs0' > /sys/kernel/config/xe/0000:03:00.0/engines_allowed + * + * Note that the engine names are the per-GT hardware names. On multi-tile + * platforms, writing ``rcs0,bcs0`` to this file would allow the first render + * and copy engines on each tile. + * + * The requested configuration may not be supported by the platform and driver + * may fail to probe. For example: if at least one copy engine is expected to be + * available for migrations, but it's disabled. This is intended for debugging + * purposes only. + * + * This attribute can only be set before binding to the device. + * + * PSMI + * ---- + * + * Enable extra debugging capabilities to trace engine execution. Only useful + * during early platform enabling and requires additional hardware connected. + * Once it's enabled, additionals WAs are added and runtime configuration is + * done via debugfs. Example to enable it:: + * + * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/enable_psmi + * + * This attribute can only be set before binding to the device. + * + * Context restore BB + * ------------------ + * + * Allow to execute a batch buffer during any context switches. When the + * GPU is restoring the context, it executes additional commands. It's useful + * for testing additional workarounds and validating certain HW behaviors: it's + * not intended for normal execution and will taint the kernel with TAINT_TEST + * when used. + * + * The syntax allows to pass straight instructions to be executed by the engine + * in a batch buffer or set specific registers. + * + * #. Generic instruction:: + * + * cmd [[dword0] [dword1] [...]] + * + * #. Simple register setting:: + * + * reg
+ * + * Commands are saved per engine class: all instances of that class will execute + * those commands during context switch. The instruction, dword arguments, + * addresses and values are in hex format like in the examples below. + * + * #. Execute a LRI command to write 0xDEADBEEF to register 0x4f10 after the + * normal context restore:: + * + * # echo 'rcs cmd 11000001 4F100 DEADBEEF' \ + * > /sys/kernel/config/xe/0000:03:00.0/ctx_restore_post_bb + * + * #. Execute a LRI command to write 0xDEADBEEF to register 0x4f10 at the + * beginning of the context restore:: + * + * # echo 'rcs cmd 11000001 4F100 DEADBEEF' \ + * > /sys/kernel/config/xe/0000:03:00.0/ctx_restore_mid_bb + + * #. Load certain values in a couple of registers (it can be used as a simpler + * alternative to the `cmd`) action:: + * + * # cat > /sys/kernel/config/xe/0000:03:00.0/ctx_restore_post_bb < /sys/kernel/config/xe/0000:00:02.0/sriov/max_vfs + * # echo 0000:00:02.0 > /sys/bus/pci/drivers/xe/bind + * + * How to enable PF with support up to 3 VFs:: + * + * # echo 3 > /sys/kernel/config/xe/0000:00:02.0/sriov/max_vfs + * # echo 0000:00:02.0 > /sys/bus/pci/drivers/xe/bind + * + * How to disable PF mode and always run as native:: + * + * # echo 0 > /sys/kernel/config/xe/0000:00:02.0/sriov/max_vfs + * # echo 0000:00:02.0 > /sys/bus/pci/drivers/xe/bind + * + * This setting only takes effect when probing the device. + * + * Enable multi-queue + * ------------------ + * + * Multi-queue support on the device is enabled by default where the + * hardware supports it. Writing 0 force-disables multi-queue support: + * multi-queue exec-queue group creation via ioctl is refused, and the + * GuC feature is disabled:: + * + * # echo 0 > /sys/kernel/config/xe/0000:03:00.0/enable_multi_queue + * + * This attribute can only be set before binding to the device. + * + * Remove devices + * ============== + * + * The created device directories can be removed using ``rmdir``:: + * + * # rmdir /sys/kernel/config/xe/0000:03:00.0/ + */ + +/* Similar to struct xe_bb, but not tied to HW (yet) */ +struct wa_bb { + u32 *cs; + u32 len; /* in dwords */ +}; + +struct xe_config_group_device { + struct config_group group; + struct config_group sriov; + + struct xe_config_device { + u64 gt_types_allowed; + u64 engines_allowed; + struct wa_bb ctx_restore_post_bb[XE_ENGINE_CLASS_MAX]; + struct wa_bb ctx_restore_mid_bb[XE_ENGINE_CLASS_MAX]; + bool survivability_mode; + bool enable_psmi; + bool enable_multi_queue; + struct { + unsigned int max_vfs; + bool admin_only_pf; + } sriov; + } config; + + /* protects attributes */ + struct mutex lock; + /* matching descriptor */ + const struct xe_device_desc *desc; + /* tentative SR-IOV mode */ + enum xe_sriov_mode mode; +}; + +static const struct xe_config_device device_defaults = { + .gt_types_allowed = U64_MAX, + .engines_allowed = U64_MAX, + .survivability_mode = false, + .enable_psmi = false, + .enable_multi_queue = true, + .sriov = { + .max_vfs = XE_DEFAULT_MAX_VFS, + .admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF, + }, +}; + +static void set_device_defaults(struct xe_config_device *config) +{ + *config = device_defaults; +#ifdef CONFIG_PCI_IOV + config->sriov.max_vfs = xe_modparam.max_vfs; +#endif +} + +struct engine_info { + const char *cls; + u64 mask; + enum xe_engine_class engine_class; +}; + +/* Some helpful macros to aid on the sizing of buffer allocation when parsing */ +#define MAX_ENGINE_CLASS_CHARS 5 +#define MAX_ENGINE_INSTANCE_CHARS 2 + +static const struct engine_info engine_info[] = { + { .cls = "rcs", .mask = XE_HW_ENGINE_RCS_MASK, .engine_class = XE_ENGINE_CLASS_RENDER }, + { .cls = "bcs", .mask = XE_HW_ENGINE_BCS_MASK, .engine_class = XE_ENGINE_CLASS_COPY }, + { .cls = "vcs", .mask = XE_HW_ENGINE_VCS_MASK, .engine_class = XE_ENGINE_CLASS_VIDEO_DECODE }, + { .cls = "vecs", .mask = XE_HW_ENGINE_VECS_MASK, .engine_class = XE_ENGINE_CLASS_VIDEO_ENHANCE }, + { .cls = "ccs", .mask = XE_HW_ENGINE_CCS_MASK, .engine_class = XE_ENGINE_CLASS_COMPUTE }, + { .cls = "gsccs", .mask = XE_HW_ENGINE_GSCCS_MASK, .engine_class = XE_ENGINE_CLASS_OTHER }, +}; + +static const struct { + const char *name; + enum xe_gt_type type; +} gt_types[] = { + { .name = "primary", .type = XE_GT_TYPE_MAIN }, + { .name = "media", .type = XE_GT_TYPE_MEDIA }, +}; + +static struct xe_config_group_device *to_xe_config_group_device(struct config_item *item) +{ + return container_of(to_config_group(item), struct xe_config_group_device, group); +} + +static struct xe_config_device *to_xe_config_device(struct config_item *item) +{ + return &to_xe_config_group_device(item)->config; +} + +static bool is_bound(struct xe_config_group_device *dev) +{ + unsigned int domain, bus, slot, function; + struct pci_dev *pdev; + const char *name; + bool ret; + + lockdep_assert_held(&dev->lock); + + name = dev->group.cg_item.ci_name; + if (sscanf(name, "%x:%x:%x.%x", &domain, &bus, &slot, &function) != 4) + return false; + + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); + if (!pdev) + return false; + + ret = pci_get_drvdata(pdev); + if (ret) + pci_dbg(pdev, "Already bound to driver\n"); + + pci_dev_put(pdev); + return ret; +} + +static ssize_t survivability_mode_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + + return sprintf(page, "%d\n", dev->survivability_mode); +} + +static ssize_t survivability_mode_store(struct config_item *item, const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + bool survivability_mode; + int ret; + + ret = kstrtobool(page, &survivability_mode); + if (ret) + return ret; + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.survivability_mode = survivability_mode; + + return len; +} + +static ssize_t gt_types_allowed_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + char *p = page; + + for (size_t i = 0; i < ARRAY_SIZE(gt_types); i++) + if (dev->gt_types_allowed & BIT_ULL(gt_types[i].type)) + p += sprintf(p, "%s\n", gt_types[i].name); + + return p - page; +} + +static ssize_t gt_types_allowed_store(struct config_item *item, const char *page, + size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + char *buf __free(kfree) = kstrdup(page, GFP_KERNEL); + char *p = buf; + u64 typemask = 0; + + if (!buf) + return -ENOMEM; + + while (p) { + char *typename = strsep(&p, ",\n"); + bool matched = false; + + if (typename[0] == '\0') + continue; + + for (size_t i = 0; i < ARRAY_SIZE(gt_types); i++) { + if (strcmp(typename, gt_types[i].name) == 0) { + typemask |= BIT(gt_types[i].type); + matched = true; + break; + } + } + + if (!matched) + return -EINVAL; + } + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.gt_types_allowed = typemask; + + return len; +} + +static ssize_t engines_allowed_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + char *p = page; + + for (size_t i = 0; i < ARRAY_SIZE(engine_info); i++) { + u64 mask = engine_info[i].mask; + + if ((dev->engines_allowed & mask) == mask) { + p += sprintf(p, "%s*\n", engine_info[i].cls); + } else if (mask & dev->engines_allowed) { + u16 bit0 = __ffs64(mask), bit; + + mask &= dev->engines_allowed; + + for_each_set_bit(bit, (const unsigned long *)&mask, 64) + p += sprintf(p, "%s%u\n", engine_info[i].cls, + bit - bit0); + } + } + + return p - page; +} + +/* + * Lookup engine_info. If @mask is not NULL, reduce the mask according to the + * instance in @pattern. + * + * Examples of inputs: + * - lookup_engine_info("rcs0", &mask): return "rcs" entry from @engine_info and + * mask == BIT_ULL(XE_HW_ENGINE_RCS0) + * - lookup_engine_info("rcs*", &mask): return "rcs" entry from @engine_info and + * mask == XE_HW_ENGINE_RCS_MASK + * - lookup_engine_info("rcs", NULL): return "rcs" entry from @engine_info + */ +static const struct engine_info *lookup_engine_info(const char *pattern, u64 *mask) +{ + for (size_t i = 0; i < ARRAY_SIZE(engine_info); i++) { + u8 instance; + u16 bit; + + if (!str_has_prefix(pattern, engine_info[i].cls)) + continue; + + pattern += strlen(engine_info[i].cls); + if (!mask) + return *pattern ? NULL : &engine_info[i]; + + if (!strcmp(pattern, "*")) { + *mask = engine_info[i].mask; + return &engine_info[i]; + } + + if (kstrtou8(pattern, 10, &instance)) + return NULL; + + bit = __ffs64(engine_info[i].mask) + instance; + if (bit >= fls64(engine_info[i].mask)) + return NULL; + + *mask = BIT_ULL(bit); + return &engine_info[i]; + } + + return NULL; +} + +static int parse_engine(const char *s, const char *end_chars, u64 *mask, + const struct engine_info **pinfo) +{ + char buf[MAX_ENGINE_CLASS_CHARS + MAX_ENGINE_INSTANCE_CHARS + 1]; + const struct engine_info *info; + size_t len; + + len = strcspn(s, end_chars); + if (len >= sizeof(buf)) + return -EINVAL; + + memcpy(buf, s, len); + buf[len] = '\0'; + + info = lookup_engine_info(buf, mask); + if (!info) + return -ENOENT; + + if (pinfo) + *pinfo = info; + + return len; +} + +static ssize_t engines_allowed_store(struct config_item *item, const char *page, + size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + ssize_t patternlen, p; + u64 mask, val = 0; + + for (p = 0; p < len; p += patternlen + 1) { + patternlen = parse_engine(page + p, ",\n", &mask, NULL); + if (patternlen < 0) + return -EINVAL; + + val |= mask; + } + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.engines_allowed = val; + + return len; +} + +static ssize_t enable_psmi_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + + return sprintf(page, "%d\n", dev->enable_psmi); +} + +static ssize_t enable_psmi_store(struct config_item *item, const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + bool val; + int ret; + + ret = kstrtobool(page, &val); + if (ret) + return ret; + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.enable_psmi = val; + + return len; +} + +static ssize_t enable_multi_queue_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + + return sprintf(page, "%d\n", dev->enable_multi_queue); +} + +static ssize_t enable_multi_queue_store(struct config_item *item, const char *page, + size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + bool val; + int ret; + + ret = kstrtobool(page, &val); + if (ret) + return ret; + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.enable_multi_queue = val; + + return len; +} + +static bool wa_bb_read_advance(bool dereference, char **p, + const char *append, size_t len, + size_t *max_size) +{ + if (dereference) { + if (len >= *max_size) + return false; + *max_size -= len; + if (append) + memcpy(*p, append, len); + } + + *p += len; + + return true; +} + +static ssize_t wa_bb_show(struct xe_config_group_device *dev, + struct wa_bb wa_bb[static XE_ENGINE_CLASS_MAX], + char *data, size_t sz) +{ + char *p = data; + + guard(mutex)(&dev->lock); + + for (size_t i = 0; i < ARRAY_SIZE(engine_info); i++) { + enum xe_engine_class ec = engine_info[i].engine_class; + size_t len; + + if (!wa_bb[ec].len) + continue; + + len = snprintf(p, sz, "%s:", engine_info[i].cls); + if (!wa_bb_read_advance(data, &p, NULL, len, &sz)) + return -ENOBUFS; + + for (size_t j = 0; j < wa_bb[ec].len; j++) { + len = snprintf(p, sz, " %08x", wa_bb[ec].cs[j]); + if (!wa_bb_read_advance(data, &p, NULL, len, &sz)) + return -ENOBUFS; + } + + if (!wa_bb_read_advance(data, &p, "\n", 1, &sz)) + return -ENOBUFS; + } + + if (!wa_bb_read_advance(data, &p, "", 1, &sz)) + return -ENOBUFS; + + /* Reserve one more to match check for '\0' */ + if (!data) + p++; + + return p - data; +} + +static ssize_t ctx_restore_mid_bb_show(struct config_item *item, char *page) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + + return wa_bb_show(dev, dev->config.ctx_restore_mid_bb, page, SZ_4K); +} + +static ssize_t ctx_restore_post_bb_show(struct config_item *item, char *page) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + + return wa_bb_show(dev, dev->config.ctx_restore_post_bb, page, SZ_4K); +} + +static void wa_bb_append(struct wa_bb *wa_bb, u32 val) +{ + if (wa_bb->cs) + wa_bb->cs[wa_bb->len] = val; + + wa_bb->len++; +} + +static ssize_t parse_hex(const char *line, u32 *pval) +{ + char numstr[12]; + const char *p; + ssize_t numlen; + + p = line + strspn(line, " \t"); + if (!*p || *p == '\n') + return 0; + + numlen = strcspn(p, " \t\n"); + if (!numlen || numlen >= sizeof(numstr) - 1) + return -EINVAL; + + memcpy(numstr, p, numlen); + numstr[numlen] = '\0'; + p += numlen; + + if (kstrtou32(numstr, 16, pval)) + return -EINVAL; + + return p - line; +} + +/* + * Parse lines with the format + * + * cmd + * reg + * + * and optionally save them in @wa_bb[i].cs is non-NULL. + * + * Return the number of dwords parsed. + */ +static ssize_t parse_wa_bb_lines(const char *lines, + struct wa_bb wa_bb[static XE_ENGINE_CLASS_MAX]) +{ + ssize_t dwords = 0, ret; + const char *p; + + for (p = lines; *p; p++) { + const struct engine_info *info = NULL; + u32 val, val2; + + /* Also allow empty lines */ + p += strspn(p, " \t\n"); + if (!*p) + break; + + ret = parse_engine(p, " \t\n", NULL, &info); + if (ret < 0) + return ret; + + p += ret; + p += strspn(p, " \t"); + + if (str_has_prefix(p, "cmd")) { + for (p += strlen("cmd"); *p;) { + ret = parse_hex(p, &val); + if (ret < 0) + return -EINVAL; + if (!ret) + break; + + p += ret; + dwords++; + wa_bb_append(&wa_bb[info->engine_class], val); + } + } else if (str_has_prefix(p, "reg")) { + p += strlen("reg"); + ret = parse_hex(p, &val); + if (ret <= 0) + return -EINVAL; + + p += ret; + ret = parse_hex(p, &val2); + if (ret <= 0) + return -EINVAL; + + p += ret; + dwords += 3; + wa_bb_append(&wa_bb[info->engine_class], + MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1)); + wa_bb_append(&wa_bb[info->engine_class], val); + wa_bb_append(&wa_bb[info->engine_class], val2); + } else { + return -EINVAL; + } + } + + return dwords; +} + +static ssize_t wa_bb_store(struct wa_bb wa_bb[static XE_ENGINE_CLASS_MAX], + struct xe_config_group_device *dev, + const char *page, size_t len) +{ + /* tmp_wa_bb must match wa_bb's size */ + struct wa_bb tmp_wa_bb[XE_ENGINE_CLASS_MAX] = { }; + ssize_t count, class; + u32 *tmp; + + /* 1. Count dwords - wa_bb[i].cs is NULL for all classes */ + count = parse_wa_bb_lines(page, tmp_wa_bb); + if (count < 0) + return count; + + guard(mutex)(&dev->lock); + + if (is_bound(dev)) + return -EBUSY; + + /* + * 2. Allocate a u32 array and set the pointers to the right positions + * according to the length of each class' wa_bb + */ + tmp = krealloc(wa_bb[0].cs, count * sizeof(u32), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + if (!count) { + memset(wa_bb, 0, sizeof(tmp_wa_bb)); + return len; + } + + for (class = 0, count = 0; class < XE_ENGINE_CLASS_MAX; ++class) { + tmp_wa_bb[class].cs = tmp + count; + count += tmp_wa_bb[class].len; + tmp_wa_bb[class].len = 0; + } + + /* 3. Parse wa_bb lines again, this time saving the values */ + count = parse_wa_bb_lines(page, tmp_wa_bb); + if (count < 0) + return count; + + memcpy(wa_bb, tmp_wa_bb, sizeof(tmp_wa_bb)); + + return len; +} + +static ssize_t ctx_restore_mid_bb_store(struct config_item *item, + const char *data, size_t sz) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + + return wa_bb_store(dev->config.ctx_restore_mid_bb, dev, data, sz); +} + +static ssize_t ctx_restore_post_bb_store(struct config_item *item, + const char *data, size_t sz) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + + return wa_bb_store(dev->config.ctx_restore_post_bb, dev, data, sz); +} + +CONFIGFS_ATTR(, ctx_restore_mid_bb); +CONFIGFS_ATTR(, ctx_restore_post_bb); +CONFIGFS_ATTR(, enable_multi_queue); +CONFIGFS_ATTR(, enable_psmi); +CONFIGFS_ATTR(, engines_allowed); +CONFIGFS_ATTR(, gt_types_allowed); +CONFIGFS_ATTR(, survivability_mode); + +static struct configfs_attribute *xe_config_device_attrs[] = { + &attr_ctx_restore_mid_bb, + &attr_ctx_restore_post_bb, + &attr_enable_multi_queue, + &attr_enable_psmi, + &attr_engines_allowed, + &attr_gt_types_allowed, + &attr_survivability_mode, + NULL, +}; + +static void xe_config_device_release(struct config_item *item) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + + mutex_destroy(&dev->lock); + + kfree(dev->config.ctx_restore_mid_bb[0].cs); + kfree(dev->config.ctx_restore_post_bb[0].cs); + kfree(dev); +} + +static struct configfs_item_operations xe_config_device_ops = { + .release = xe_config_device_release, +}; + +static bool xe_config_device_is_visible(struct config_item *item, + struct configfs_attribute *attr, int n) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + + if (attr == &attr_survivability_mode) { + if (!dev->desc->is_dgfx || dev->desc->platform < XE_BATTLEMAGE) + return false; + } + + return true; +} + +static struct configfs_group_operations xe_config_device_group_ops = { + .is_visible = xe_config_device_is_visible, +}; + +static const struct config_item_type xe_config_device_type = { + .ct_item_ops = &xe_config_device_ops, + .ct_group_ops = &xe_config_device_group_ops, + .ct_attrs = xe_config_device_attrs, + .ct_owner = THIS_MODULE, +}; + +static ssize_t sriov_max_vfs_show(struct config_item *item, char *page) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent); + + guard(mutex)(&dev->lock); + + if (dev->config.sriov.max_vfs == UINT_MAX) + return sprintf(page, "%s\n", "unlimited"); + else + return sprintf(page, "%u\n", dev->config.sriov.max_vfs); +} + +static ssize_t sriov_max_vfs_store(struct config_item *item, const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent); + unsigned int max_vfs; + int ret; + + guard(mutex)(&dev->lock); + + if (is_bound(dev)) + return -EBUSY; + + ret = kstrtouint(page, 0, &max_vfs); + if (ret) { + if (!sysfs_streq(page, "unlimited")) + return ret; + max_vfs = UINT_MAX; + } + + dev->config.sriov.max_vfs = max_vfs; + return len; +} + +static ssize_t sriov_admin_only_pf_show(struct config_item *item, char *page) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent); + + guard(mutex)(&dev->lock); + + return sprintf(page, "%s\n", str_yes_no(dev->config.sriov.admin_only_pf)); +} + +static ssize_t sriov_admin_only_pf_store(struct config_item *item, const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent); + bool admin_only_pf; + int ret; + + guard(mutex)(&dev->lock); + + if (is_bound(dev)) + return -EBUSY; + + ret = kstrtobool(page, &admin_only_pf); + if (ret) + return ret; + + dev->config.sriov.admin_only_pf = admin_only_pf; + return len; +} + +CONFIGFS_ATTR(sriov_, max_vfs); +CONFIGFS_ATTR(sriov_, admin_only_pf); + +static struct configfs_attribute *xe_config_sriov_attrs[] = { + &sriov_attr_max_vfs, + &sriov_attr_admin_only_pf, + NULL, +}; + +static bool xe_config_sriov_is_visible(struct config_item *item, + struct configfs_attribute *attr, int n) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent); + + if (attr == &sriov_attr_max_vfs && dev->mode != XE_SRIOV_MODE_PF) + return false; + if (attr == &sriov_attr_admin_only_pf && dev->mode != XE_SRIOV_MODE_PF) + return false; + + return true; +} + +static struct configfs_group_operations xe_config_sriov_group_ops = { + .is_visible = xe_config_sriov_is_visible, +}; + +static const struct config_item_type xe_config_sriov_type = { + .ct_owner = THIS_MODULE, + .ct_group_ops = &xe_config_sriov_group_ops, + .ct_attrs = xe_config_sriov_attrs, +}; + +static const struct xe_device_desc *xe_match_desc(struct pci_dev *pdev) +{ + struct device_driver *driver = driver_find("xe", &pci_bus_type); + struct pci_driver *drv = to_pci_driver(driver); + const struct pci_device_id *ids = drv ? drv->id_table : NULL; + const struct pci_device_id *found = pci_match_id(ids, pdev); + + return found ? (const void *)found->driver_data : NULL; +} + +static struct pci_dev *get_physfn_instead(struct pci_dev *virtfn) +{ + struct pci_dev *physfn = pci_physfn(virtfn); + + pci_dev_get(physfn); + pci_dev_put(virtfn); + return physfn; +} + +static struct config_group *xe_config_make_device_group(struct config_group *group, + const char *name) +{ + unsigned int domain, bus, slot, function; + struct xe_config_group_device *dev; + const struct xe_device_desc *match; + enum xe_sriov_mode mode; + struct pci_dev *pdev; + char canonical[16]; + int vfnumber = 0; + int ret; + + ret = sscanf(name, "%x:%x:%x.%x", &domain, &bus, &slot, &function); + if (ret != 4) + return ERR_PTR(-EINVAL); + + ret = scnprintf(canonical, sizeof(canonical), "%04x:%02x:%02x.%d", domain, bus, + PCI_SLOT(PCI_DEVFN(slot, function)), + PCI_FUNC(PCI_DEVFN(slot, function))); + if (ret != 12 || strcmp(name, canonical)) + return ERR_PTR(-EINVAL); + + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); + mode = pdev ? dev_is_pf(&pdev->dev) ? + XE_SRIOV_MODE_PF : XE_SRIOV_MODE_NONE : XE_SRIOV_MODE_VF; + + if (!pdev && function) + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, 0)); + if (!pdev && slot) + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(0, 0)); + if (!pdev) + return ERR_PTR(-ENODEV); + + if (PCI_DEVFN(slot, function) != pdev->devfn) { + pdev = get_physfn_instead(pdev); + vfnumber = PCI_DEVFN(slot, function) - pdev->devfn; + if (!dev_is_pf(&pdev->dev) || vfnumber > pci_sriov_get_totalvfs(pdev)) { + pci_dev_put(pdev); + return ERR_PTR(-ENODEV); + } + } + + match = xe_match_desc(pdev); + if (match && vfnumber && !match->has_sriov) { + pci_info(pdev, "xe driver does not support VFs on this device\n"); + match = NULL; + } else if (!match) { + pci_info(pdev, "xe driver does not support configuration of this device\n"); + } + + pci_dev_put(pdev); + + if (!match) + return ERR_PTR(-ENOENT); + + dev = kzalloc_obj(*dev); + if (!dev) + return ERR_PTR(-ENOMEM); + + dev->desc = match; + dev->mode = match->has_sriov ? mode : XE_SRIOV_MODE_NONE; + + set_device_defaults(&dev->config); + + config_group_init_type_name(&dev->group, name, &xe_config_device_type); + if (dev->mode != XE_SRIOV_MODE_NONE) { + config_group_init_type_name(&dev->sriov, "sriov", &xe_config_sriov_type); + configfs_add_default_group(&dev->sriov, &dev->group); + } + + mutex_init(&dev->lock); + + return &dev->group; +} + +static struct configfs_group_operations xe_config_group_ops = { + .make_group = xe_config_make_device_group, +}; + +static const struct config_item_type xe_configfs_type = { + .ct_group_ops = &xe_config_group_ops, + .ct_owner = THIS_MODULE, +}; + +static struct configfs_subsystem xe_configfs = { + .su_group = { + .cg_item = { + .ci_namebuf = "xe", + .ci_type = &xe_configfs_type, + }, + }, +}; + +static struct xe_config_group_device *find_xe_config_group_device(struct pci_dev *pdev) +{ + struct config_item *item; + + mutex_lock(&xe_configfs.su_mutex); + item = config_group_find_item(&xe_configfs.su_group, pci_name(pdev)); + mutex_unlock(&xe_configfs.su_mutex); + + if (!item) + return NULL; + + return to_xe_config_group_device(item); +} + +static void dump_custom_dev_config(struct pci_dev *pdev, + struct xe_config_group_device *dev) +{ +#define PRI_CUSTOM_ATTR(fmt_, attr_) do { \ + if (dev->config.attr_ != device_defaults.attr_) \ + pci_info(pdev, "configfs: " __stringify(attr_) " = " fmt_ "\n", \ + dev->config.attr_); \ + } while (0) + + PRI_CUSTOM_ATTR("%llx", gt_types_allowed); + PRI_CUSTOM_ATTR("%llx", engines_allowed); + PRI_CUSTOM_ATTR("%d", enable_multi_queue); + PRI_CUSTOM_ATTR("%d", enable_psmi); + PRI_CUSTOM_ATTR("%d", survivability_mode); + PRI_CUSTOM_ATTR("%u", sriov.admin_only_pf); + +#undef PRI_CUSTOM_ATTR +} + +/** + * xe_configfs_check_device() - Test if device was configured by configfs + * @pdev: the &pci_dev device to test + * + * Try to find the configfs group that belongs to the specified pci device + * and print a diagnostic message if different than the default value. + */ +void xe_configfs_check_device(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + + if (!dev) + return; + + /* memcmp here is safe as both are zero-initialized */ + if (memcmp(&dev->config, &device_defaults, sizeof(dev->config))) { + pci_info(pdev, "Found custom settings in configfs\n"); + dump_custom_dev_config(pdev, dev); + } + + config_group_put(&dev->group); +} + +/** + * xe_configfs_get_survivability_mode - get configfs survivability mode attribute + * @pdev: pci device + * + * Return: survivability_mode attribute in configfs + */ +bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + bool mode; + + if (!dev) + return device_defaults.survivability_mode; + + mode = dev->config.survivability_mode; + config_group_put(&dev->group); + + return mode; +} + +static u64 get_gt_types_allowed(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + u64 mask; + + if (!dev) + return device_defaults.gt_types_allowed; + + mask = dev->config.gt_types_allowed; + config_group_put(&dev->group); + + return mask; +} + +/** + * xe_configfs_primary_gt_allowed - determine whether primary GTs are supported + * @pdev: pci device + * + * Return: True if primary GTs are enabled, false if they have been disabled via + * configfs. + */ +bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev) +{ + return get_gt_types_allowed(pdev) & BIT_ULL(XE_GT_TYPE_MAIN); +} + +/** + * xe_configfs_media_gt_allowed - determine whether media GTs are supported + * @pdev: pci device + * + * Return: True if the media GTs are enabled, false if they have been disabled + * via configfs. + */ +bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) +{ + return get_gt_types_allowed(pdev) & BIT_ULL(XE_GT_TYPE_MEDIA); +} + +/** + * xe_configfs_get_engines_allowed - get engine allowed mask from configfs + * @pdev: pci device + * + * Return: engine mask with allowed engines set in configfs + */ +u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + u64 engines_allowed; + + if (!dev) + return device_defaults.engines_allowed; + + engines_allowed = dev->config.engines_allowed; + config_group_put(&dev->group); + + return engines_allowed; +} + +/** + * xe_configfs_get_psmi_enabled - get configfs enable_psmi setting + * @pdev: pci device + * + * Return: enable_psmi setting in configfs + */ +bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + bool ret; + + if (!dev) + return false; + + ret = dev->config.enable_psmi; + config_group_put(&dev->group); + + return ret; +} + +/** + * xe_configfs_get_enable_multi_queue - get configfs enable_multi_queue setting + * @pdev: pci device + * + * Return: true if multi-queue is enabled for this device (the default), + * false if it has been force-disabled via configfs. + */ +bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + bool ret; + + if (!dev) + return true; + + ret = dev->config.enable_multi_queue; + config_group_put(&dev->group); + + return ret; +} + +/** + * xe_configfs_get_ctx_restore_mid_bb - get configfs ctx_restore_mid_bb setting + * @pdev: pci device + * @class: hw engine class + * @cs: pointer to the bb to use - only valid during probe + * + * Return: Number of dwords used in the mid_ctx_restore setting in configfs + */ +u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, + const u32 **cs) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + u32 len; + + if (!dev) + return 0; + + if (cs) + *cs = dev->config.ctx_restore_mid_bb[class].cs; + + len = dev->config.ctx_restore_mid_bb[class].len; + config_group_put(&dev->group); + + return len; +} + +/** + * xe_configfs_get_ctx_restore_post_bb - get configfs ctx_restore_post_bb setting + * @pdev: pci device + * @class: hw engine class + * @cs: pointer to the bb to use - only valid during probe + * + * Return: Number of dwords used in the post_ctx_restore setting in configfs + */ +u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, + const u32 **cs) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + u32 len; + + if (!dev) + return 0; + + *cs = dev->config.ctx_restore_post_bb[class].cs; + len = dev->config.ctx_restore_post_bb[class].len; + config_group_put(&dev->group); + + return len; +} + +#ifdef CONFIG_PCI_IOV +/** + * xe_configfs_admin_only_pf() - Get PF's operational mode. + * @pdev: the &pci_dev device + * + * Find the configfs group that belongs to the PCI device and return a flag + * whether the PF driver should be dedicated for VFs management only. + * + * If configfs group is not present, use driver's default value. + * + * Return: true if PF driver is dedicated for VFs administration only. + */ +bool xe_configfs_admin_only_pf(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + bool admin_only_pf; + + if (!dev) + return XE_DEFAULT_ADMIN_ONLY_PF; + + scoped_guard(mutex, &dev->lock) + admin_only_pf = dev->config.sriov.admin_only_pf; + + config_group_put(&dev->group); + + return admin_only_pf; +} +/** + * xe_configfs_get_max_vfs() - Get number of VFs that could be managed + * @pdev: the &pci_dev device + * + * Find the configfs group that belongs to the PCI device and return maximum + * number of Virtual Functions (VFs) that could be managed by this device. + * If configfs group is not present, use value of max_vfs module parameter. + * + * Return: maximum number of VFs that could be managed. + */ +unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + unsigned int max_vfs; + + if (!dev) + return xe_modparam.max_vfs; + + scoped_guard(mutex, &dev->lock) + max_vfs = dev->config.sriov.max_vfs; + + config_group_put(&dev->group); + + return max_vfs; +} +#endif + +int __init xe_configfs_init(void) +{ + int ret; + + config_group_init(&xe_configfs.su_group); + mutex_init(&xe_configfs.su_mutex); + ret = configfs_register_subsystem(&xe_configfs); + if (ret) { + mutex_destroy(&xe_configfs.su_mutex); + return ret; + } + + return 0; +} + +void xe_configfs_exit(void) +{ + configfs_unregister_subsystem(&xe_configfs); + mutex_destroy(&xe_configfs.su_mutex); +} diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h new file mode 100644 index 000000000..4fbbeafba --- /dev/null +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ +#ifndef _XE_CONFIGFS_H_ +#define _XE_CONFIGFS_H_ + +#include +#include + +#include "xe_defaults.h" +#include "xe_hw_engine_types.h" +#include "xe_module.h" + +struct pci_dev; + +#if IS_ENABLED(CONFIG_CONFIGFS_FS) +int xe_configfs_init(void); +void xe_configfs_exit(void); +void xe_configfs_check_device(struct pci_dev *pdev); +bool xe_configfs_get_survivability_mode(struct pci_dev *pdev); +bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev); +bool xe_configfs_media_gt_allowed(struct pci_dev *pdev); +u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); +bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); +bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev); +u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, + const u32 **cs); +u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, + const u32 **cs); +#ifdef CONFIG_PCI_IOV +unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev); +bool xe_configfs_admin_only_pf(struct pci_dev *pdev); +#endif +#else +static inline int xe_configfs_init(void) { return 0; } +static inline void xe_configfs_exit(void) { } +static inline void xe_configfs_check_device(struct pci_dev *pdev) { } +static inline bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { return false; } +static inline bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev) { return true; } +static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return true; } +static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } +static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } +static inline bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) { return true; } +static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, + const u32 **cs) { return 0; } +static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, + const u32 **cs) { return 0; } +#ifdef CONFIG_PCI_IOV +static inline unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev) +{ + return xe_modparam.max_vfs; +} +static inline bool xe_configfs_admin_only_pf(struct pci_dev *pdev) +{ + return XE_DEFAULT_ADMIN_ONLY_PF; +} +#endif +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c new file mode 100644 index 000000000..8de78cd0a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -0,0 +1,777 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_debugfs.h" + +#include +#include +#include +#include + +#include + +#include "regs/xe_pmt.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_debugfs.h" +#include "xe_gt_printk.h" +#include "xe_guc_ads.h" +#include "xe_hw_engine.h" +#include "xe_mmio.h" +#include "xe_pcode.h" +#include "xe_pm.h" +#include "xe_psmi.h" +#include "xe_pxp_debugfs.h" +#include "xe_sriov.h" +#include "xe_sriov_pf_debugfs.h" +#include "xe_sriov_vf.h" +#include "xe_step.h" +#include "xe_tile_debugfs.h" +#include "xe_vsec.h" +#include "xe_wa.h" + +#ifdef CONFIG_DRM_XE_DEBUG +#include "xe_bo_evict.h" +#include "xe_migrate.h" +#include "xe_vm.h" +#endif + +DECLARE_FAULT_ATTR(gt_reset_failure); +DECLARE_FAULT_ATTR(inject_csc_hw_error); + +static bool csc_hw_error_available(struct xe_device *xe) +{ + return !IS_SRIOV_VF(xe) && xe->info.platform == XE_BATTLEMAGE; +} + +/* + * Fault injection table. Each entry registers a debugfs attribute; add a + * matching FAULT_ACTION() below for every entry added here. + */ +static struct { + const char *name; + struct fault_attr *attr; + bool (*is_visible)(struct xe_device *xe); +} xe_fault_inject_entry[] = { + { .name = "fail_gt_reset", + .attr = >_reset_failure }, + { .name = "inject_csc_hw_error", + .attr = &inject_csc_hw_error, + .is_visible = csc_hw_error_available }, +}; + +/* + * FAULT_ACTION(name, fault_attr) - generate xe_fault_() accessor. + * Add one entry per row in xe_fault_inject_entry[]. + */ +#define FAULT_ACTION(name, fault_attr) \ +bool xe_fault_##name(void) \ +{ \ + return should_fail(&(fault_attr), 1); \ +} + +FAULT_ACTION(gt_reset, gt_reset_failure) +FAULT_ACTION(csc_hw_error, inject_csc_hw_error) + +static void xe_fault_inject_debugfs_register(struct xe_device *xe, + struct dentry *root) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(xe_fault_inject_entry); i++) { + if (xe_fault_inject_entry[i].is_visible && + !xe_fault_inject_entry[i].is_visible(xe)) + continue; + + fault_create_debugfs_attr(xe_fault_inject_entry[i].name, root, + xe_fault_inject_entry[i].attr); + } +} + +static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio, + u32 offset, const char *name, struct drm_printer *p) +{ + u64 residency = 0; + int ret; + + ret = xe_pmt_telem_read(xe->drm.dev, + xe_mmio_read32(mmio, PUNIT_TELEMETRY_GUID), + &residency, offset, sizeof(residency)); + if (ret != sizeof(residency)) { + drm_warn(&xe->drm, "%s counter failed to read, ret %d\n", name, ret); + return; + } + + drm_printf(p, "%s : %llu\n", name, residency); +} + +static struct xe_device *node_to_xe(struct drm_info_node *node) +{ + return to_xe_device(node->minor->dev); +} + +static void print_engine_class_mask(struct drm_printer *p, u16 mask) +{ + if (!mask) { + drm_printf(p, " none\n"); + return; + } + + for (enum xe_engine_class ec = 0; ec < XE_ENGINE_CLASS_MAX; ec++) { + if (mask & BIT(ec)) + drm_printf(p, " %s", xe_hw_engine_class_to_str(ec)); + } + drm_printf(p, "\n"); +} + +static void print_engine_mask(struct drm_printer *p, struct xe_gt *gt, u64 mask) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + if (!mask) { + drm_printf(p, " none\n"); + return; + } + + for_each_hw_engine(hwe, gt, id) { + if (mask & BIT_ULL(id)) + drm_printf(p, " %s", hwe->name); + } + drm_printf(p, "\n"); +} + +static int info(struct seq_file *m, void *data) +{ + struct xe_device *xe = node_to_xe(m->private); + struct drm_printer p = drm_seq_file_printer(m); + struct xe_gt *gt; + u8 id; + + guard(xe_pm_runtime)(xe); + + drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100); + drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100); + drm_printf(&p, "stepping G:%s M:%s B:%s\n", + xe_step_name(xe->info.step.graphics), + xe_step_name(xe->info.step.media), + xe_step_name(xe->info.step.basedie)); + drm_printf(&p, "is_dgfx %s\n", str_yes_no(xe->info.is_dgfx)); + drm_printf(&p, "platform %d\n", xe->info.platform); + drm_printf(&p, "subplatform %d\n", + xe->info.subplatform > XE_SUBPLATFORM_NONE ? xe->info.subplatform : 0); + drm_printf(&p, "devid 0x%x\n", xe->info.devid); + drm_printf(&p, "revid %d\n", xe->info.revid); + drm_printf(&p, "tile_count %d\n", xe->info.tile_count); + drm_printf(&p, "vm_max_level %d\n", xe->info.vm_max_level); + drm_printf(&p, "has_flat_ccs %s\n", str_yes_no(xe->info.has_flat_ccs)); + drm_printf(&p, "has_usm %s\n", str_yes_no(xe->info.has_usm)); + drm_printf(&p, "skip_guc_pc %s\n", str_yes_no(xe->info.skip_guc_pc)); + drm_printf(&p, "multi_lrc_engine_classes"); + print_engine_class_mask(&p, xe->info.multi_lrc_mask); + for_each_gt(gt, xe, id) { + drm_printf(&p, "gt%d force wake %d\n", id, + xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT)); + drm_printf(&p, "gt%d engines", id); + print_engine_mask(&p, gt, gt->info.engine_mask); + drm_printf(&p, "gt%d multi_queue_engine_classes", id); + print_engine_class_mask(&p, gt->info.multi_queue_engine_class_mask); + } + + return 0; +} + +static int sriov_info(struct seq_file *m, void *data) +{ + struct xe_device *xe = node_to_xe(m->private); + struct drm_printer p = drm_seq_file_printer(m); + + xe_sriov_print_info(xe, &p); + return 0; +} + +static int workarounds(struct xe_device *xe, struct drm_printer *p) +{ + guard(xe_pm_runtime)(xe); + xe_wa_device_dump(xe, p); + + return 0; +} + +static int workaround_info(struct seq_file *m, void *data) +{ + struct xe_device *xe = node_to_xe(m->private); + struct drm_printer p = drm_seq_file_printer(m); + + workarounds(xe, &p); + return 0; +} + +static int pcode_info(struct seq_file *m, void *data) +{ + struct xe_device *xe = node_to_xe(m->private); + struct drm_printer p = drm_seq_file_printer(m); + struct xe_pcode_version version; + int ret = 0; + + ret = xe_get_pcode_version(xe, &version); + if (ret) + return ret; + + drm_printf(&p, "pcode version: %u.%u.%u\n", version.major, + version.minor, version.engg); + return 0; +} + +static int dgfx_pkg_residencies_show(struct seq_file *m, void *data) +{ + struct xe_device *xe; + struct xe_mmio *mmio; + struct drm_printer p; + + xe = node_to_xe(m->private); + p = drm_seq_file_printer(m); + guard(xe_pm_runtime)(xe); + mmio = xe_root_tile_mmio(xe); + static const struct { + u32 offset; + const char *name; + } residencies[] = { + {BMG_G2_RESIDENCY_OFFSET, "Package G2"}, + {BMG_G6_RESIDENCY_OFFSET, "Package G6"}, + {BMG_G7_RESIDENCY_OFFSET, "Package G7"}, + {BMG_G8_RESIDENCY_OFFSET, "Package G8"}, + {BMG_G10_RESIDENCY_OFFSET, "Package G10"}, + {BMG_MODS_RESIDENCY_OFFSET, "Package ModS"} + }; + + for (int i = 0; i < ARRAY_SIZE(residencies); i++) + read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p); + + return 0; +} + +static int dgfx_pcie_link_residencies_show(struct seq_file *m, void *data) +{ + struct xe_device *xe; + struct xe_mmio *mmio; + struct drm_printer p; + + xe = node_to_xe(m->private); + p = drm_seq_file_printer(m); + guard(xe_pm_runtime)(xe); + mmio = xe_root_tile_mmio(xe); + + static const struct { + u32 offset; + const char *name; + } residencies[] = { + {BMG_PCIE_LINK_L0_RESIDENCY_OFFSET, "PCIE LINK L0 RESIDENCY"}, + {BMG_PCIE_LINK_L1_RESIDENCY_OFFSET, "PCIE LINK L1 RESIDENCY"}, + {BMG_PCIE_LINK_L1_2_RESIDENCY_OFFSET, "PCIE LINK L1.2 RESIDENCY"} + }; + + for (int i = 0; i < ARRAY_SIZE(residencies); i++) + read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p); + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"info", info, 0}, + { .name = "sriov_info", .show = sriov_info, }, + { .name = "workarounds", .show = workaround_info, }, +}; + +static const struct drm_info_list pcode_info_debugfs[] = { + { .name = "pcode_info", .show = pcode_info, }, +}; + +static const struct drm_info_list debugfs_residencies[] = { + { .name = "dgfx_pkg_residencies", .show = dgfx_pkg_residencies_show, }, + { .name = "dgfx_pcie_link_residencies", .show = dgfx_pcie_link_residencies_show, }, +}; + +static int forcewake_open(struct inode *inode, struct file *file) +{ + struct xe_device *xe = inode->i_private; + struct xe_gt *gt; + u8 id, last_gt; + unsigned int fw_ref; + + xe_pm_runtime_get(xe); + for_each_gt(gt, xe, id) { + last_gt = id; + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) + goto err_fw_get; + } + + return 0; + +err_fw_get: + for_each_gt(gt, xe, id) { + if (id < last_gt) + xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); + else if (id == last_gt) + xe_force_wake_put(gt_to_fw(gt), fw_ref); + else + break; + } + + xe_pm_runtime_put(xe); + return -ETIMEDOUT; +} + +static int forcewake_release(struct inode *inode, struct file *file) +{ + struct xe_device *xe = inode->i_private; + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) + xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); + xe_pm_runtime_put(xe); + + return 0; +} + +static const struct file_operations forcewake_all_fops = { + .owner = THIS_MODULE, + .open = forcewake_open, + .release = forcewake_release, +}; + +static ssize_t wedged_mode_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len = 0; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->wedged.mode); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static int __wedged_mode_set_reset_policy(struct xe_gt *gt, enum xe_wedged_mode mode) +{ + bool enable_engine_reset; + int ret; + + enable_engine_reset = (mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET); + ret = xe_guc_ads_scheduler_policy_toggle_reset(>->uc.guc.ads, + enable_engine_reset); + if (ret) + xe_gt_err(gt, "Failed to update GuC ADS scheduler policy (%pe)\n", ERR_PTR(ret)); + + return ret; +} + +static int wedged_mode_set_reset_policy(struct xe_device *xe, enum xe_wedged_mode mode) +{ + struct xe_gt *gt; + int ret; + u8 id; + + guard(xe_pm_runtime)(xe); + for_each_gt(gt, xe, id) { + ret = __wedged_mode_set_reset_policy(gt, mode); + if (ret) { + if (id > 0) { + xe->wedged.inconsistent_reset = true; + drm_err(&xe->drm, "Inconsistent reset policy state between GTs\n"); + } + return ret; + } + } + + xe->wedged.inconsistent_reset = false; + + return 0; +} + +static bool wedged_mode_needs_policy_update(struct xe_device *xe, enum xe_wedged_mode mode) +{ + if (xe->wedged.inconsistent_reset) + return true; + + if (xe->wedged.mode == mode) + return false; + + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET || + mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) + return true; + + return false; +} + +static ssize_t wedged_mode_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + u32 wedged_mode; + ssize_t ret; + + ret = kstrtouint_from_user(ubuf, size, 0, &wedged_mode); + if (ret) + return ret; + + ret = xe_device_validate_wedged_mode(xe, wedged_mode); + if (ret) + return ret; + + if (wedged_mode_needs_policy_update(xe, wedged_mode)) { + ret = wedged_mode_set_reset_policy(xe, wedged_mode); + if (ret) + return ret; + } + + xe->wedged.mode = wedged_mode; + + return size; +} + +static const struct file_operations wedged_mode_fops = { + .owner = THIS_MODULE, + .read = wedged_mode_show, + .write = wedged_mode_set, +}; + +static ssize_t page_reclaim_hw_assist_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[8]; + int len; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->info.has_page_reclaim_hw_assist); + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t page_reclaim_hw_assist_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + bool val; + ssize_t ret; + + ret = kstrtobool_from_user(ubuf, size, &val); + if (ret) + return ret; + + xe->info.has_page_reclaim_hw_assist = val; + + return size; +} + +static const struct file_operations page_reclaim_hw_assist_fops = { + .owner = THIS_MODULE, + .read = page_reclaim_hw_assist_show, + .write = page_reclaim_hw_assist_set, +}; + +static ssize_t atomic_svm_timeslice_ms_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len = 0; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->atomic_svm_timeslice_ms); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t atomic_svm_timeslice_ms_set(struct file *f, + const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + u32 atomic_svm_timeslice_ms; + ssize_t ret; + + ret = kstrtouint_from_user(ubuf, size, 0, &atomic_svm_timeslice_ms); + if (ret) + return ret; + + xe->atomic_svm_timeslice_ms = atomic_svm_timeslice_ms; + + return size; +} + +static const struct file_operations atomic_svm_timeslice_ms_fops = { + .owner = THIS_MODULE, + .read = atomic_svm_timeslice_ms_show, + .write = atomic_svm_timeslice_ms_set, +}; + +static ssize_t min_run_period_lr_ms_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len = 0; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_lr_ms); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t min_run_period_lr_ms_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + u32 min_run_period_lr_ms; + ssize_t ret; + + ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_lr_ms); + if (ret) + return ret; + + xe->min_run_period_lr_ms = min_run_period_lr_ms; + + return size; +} + +static const struct file_operations min_run_period_lr_ms_fops = { + .owner = THIS_MODULE, + .read = min_run_period_lr_ms_show, + .write = min_run_period_lr_ms_set, +}; + +static ssize_t min_run_period_pf_ms_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len = 0; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_pf_ms); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t min_run_period_pf_ms_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + u32 min_run_period_pf_ms; + ssize_t ret; + + ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_pf_ms); + if (ret) + return ret; + + xe->min_run_period_pf_ms = min_run_period_pf_ms; + + return size; +} + +static const struct file_operations min_run_period_pf_ms_fops = { + .owner = THIS_MODULE, + .read = min_run_period_pf_ms_show, + .write = min_run_period_pf_ms_set, +}; + +static ssize_t disable_late_binding_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + struct xe_late_bind *late_bind = &xe->late_bind; + char buf[32]; + int len; + + len = scnprintf(buf, sizeof(buf), "%d\n", late_bind->disable); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t disable_late_binding_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + struct xe_late_bind *late_bind = &xe->late_bind; + bool val; + int ret; + + ret = kstrtobool_from_user(ubuf, size, &val); + if (ret) + return ret; + + late_bind->disable = val; + return size; +} + +static const struct file_operations disable_late_binding_fops = { + .owner = THIS_MODULE, + .read = disable_late_binding_show, + .write = disable_late_binding_set, +}; + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +static const char * const page_size_alloc_mode_names[] = { + [XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE] = "none", + [XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M] = "only_2m", + [XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G] = "only_1g", + [XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED] = "mixed", +}; + +static ssize_t page_size_alloc_mode_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len; + enum xe_page_size_alloc_ctrl_mode mode; + + mode = READ_ONCE(xe->page_size_alloc_ctrl.mode); + if (mode >= ARRAY_SIZE(page_size_alloc_mode_names) || + !page_size_alloc_mode_names[mode]) + len = scnprintf(buf, sizeof(buf), "unknown\n"); + else + len = scnprintf(buf, sizeof(buf), "%s\n", + page_size_alloc_mode_names[mode]); + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t page_size_alloc_mode_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + int ret; + char buf[32]; + int mode; + + if (*pos) + return -ESPIPE; + + if (size > sizeof(buf) - 1) + return -EINVAL; + + ret = simple_write_to_buffer(buf, sizeof(buf) - 1, pos, ubuf, size); + if (ret < 0) + return ret; + buf[ret] = '\0'; + + mode = sysfs_match_string(page_size_alloc_mode_names, buf); + if (mode < 0) + return mode; + + mutex_lock(&xe->page_size_alloc_ctrl.lock); + if (mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED) + xe->page_size_alloc_ctrl.cur_index = 0; + WRITE_ONCE(xe->page_size_alloc_ctrl.mode, + (enum xe_page_size_alloc_ctrl_mode)mode); + mutex_unlock(&xe->page_size_alloc_ctrl.lock); + + return size; +} + +static const struct file_operations page_size_alloc_mode_fops = { + .owner = THIS_MODULE, + .read = page_size_alloc_mode_show, + .write = page_size_alloc_mode_set, +}; +#endif + +void xe_debugfs_register(struct xe_device *xe) +{ + struct ttm_device *bdev = &xe->ttm; + struct drm_minor *minor = xe->drm.primary; + struct dentry *root = minor->debugfs_root; + struct ttm_resource_manager *man; + struct xe_tile *tile; + struct xe_gt *gt; + u8 tile_id; + u8 id; + + drm_debugfs_create_files(debugfs_list, + ARRAY_SIZE(debugfs_list), + root, minor); + + if (xe->info.platform == XE_BATTLEMAGE && !IS_SRIOV_VF(xe)) { + drm_debugfs_create_files(debugfs_residencies, + ARRAY_SIZE(debugfs_residencies), + root, minor); + } + + /* + * Pcode version read from PMT is currently only supported on CRI and BMG platforms in PF + * mode, as both platforms support the necessary telemetry read mechanism and have a fixed + * PUNIT_VERSION_OFFSET. + * Attempting this access on other platforms must be verified before enabling support. + */ + if (!IS_SRIOV_VF(xe) && + (xe->info.platform == XE_CRESCENTISLAND || xe->info.platform == XE_BATTLEMAGE)) + drm_debugfs_create_files(pcode_info_debugfs, + ARRAY_SIZE(pcode_info_debugfs), + root, minor); + + debugfs_create_file("forcewake_all", 0400, root, xe, + &forcewake_all_fops); + + debugfs_create_file("wedged_mode", 0600, root, xe, + &wedged_mode_fops); + + debugfs_create_file("atomic_svm_timeslice_ms", 0600, root, xe, + &atomic_svm_timeslice_ms_fops); + + debugfs_create_file("min_run_period_lr_ms", 0600, root, xe, + &min_run_period_lr_ms_fops); + + debugfs_create_file("min_run_period_pf_ms", 0600, root, xe, + &min_run_period_pf_ms_fops); + + debugfs_create_file("disable_late_binding", 0600, root, xe, + &disable_late_binding_fops); + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE + /* + * Expose a debugfs knob to control user BO page-size allocation: + * "none" - default behavior + * "only_2m" - force 2M page allocations + * "only_1g" - force 1G page allocations + * "mixed" - select 4K, 64K, 2M, and 1G in round-robin order + */ + if (xe_debug_page_size_supported(xe)) + debugfs_create_file("page_size_alloc_mode", 0600, root, xe, + &page_size_alloc_mode_fops); +#endif + /* + * Don't expose page reclaim configuration file if not supported by the + * hardware initially. + */ + if (xe->info.has_page_reclaim_hw_assist) + debugfs_create_file("page_reclaim_hw_assist", 0600, root, xe, + &page_reclaim_hw_assist_fops); + + man = ttm_manager_type(bdev, XE_PL_TT); + ttm_resource_manager_create_debugfs(man, root, "gtt_mm"); + + man = ttm_manager_type(bdev, XE_PL_STOLEN); + if (man) + ttm_resource_manager_create_debugfs(man, root, "stolen_mm"); + + for_each_tile(tile, xe, tile_id) + xe_tile_debugfs_register(tile); + + for_each_gt(gt, xe, id) + xe_gt_debugfs_register(gt); + + xe_pxp_debugfs_register(xe->pxp); + + xe_psmi_debugfs_register(xe); + + xe_fault_inject_debugfs_register(xe, root); + + if (IS_SRIOV_PF(xe)) + xe_sriov_pf_debugfs_register(xe, root); + else if (IS_SRIOV_VF(xe)) + xe_sriov_vf_debugfs_register(xe, root); +} diff --git a/drivers/gpu/drm/xe/xe_debugfs.h b/drivers/gpu/drm/xe/xe_debugfs.h new file mode 100644 index 000000000..cd56f7442 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_debugfs.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_DEBUGFS_H_ +#define _XE_DEBUGFS_H_ + +#include + +struct xe_device; + +#ifdef CONFIG_DEBUG_FS +bool xe_fault_gt_reset(void); +bool xe_fault_csc_hw_error(void); +void xe_debugfs_register(struct xe_device *xe); +#else +static inline bool xe_fault_gt_reset(void) { return false; } +static inline bool xe_fault_csc_hw_error(void) { return false; } +static inline void xe_debugfs_register(struct xe_device *xe) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_defaults.h b/drivers/gpu/drm/xe/xe_defaults.h new file mode 100644 index 000000000..c8ae1d5f3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_defaults.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ +#ifndef _XE_DEFAULTS_H_ +#define _XE_DEFAULTS_H_ + +#include "xe_device_types.h" + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define XE_DEFAULT_GUC_LOG_LEVEL 3 +#else +#define XE_DEFAULT_GUC_LOG_LEVEL 1 +#endif + +#define XE_DEFAULT_PROBE_DISPLAY IS_ENABLED(CONFIG_DRM_XE_DISPLAY) +#define XE_DEFAULT_VRAM_BAR_SIZE 0 +#define XE_DEFAULT_FORCE_PROBE CONFIG_DRM_XE_FORCE_PROBE +#define XE_DEFAULT_MAX_VFS ~0 +#define XE_DEFAULT_MAX_VFS_STR "unlimited" +#define XE_DEFAULT_ADMIN_ONLY_PF false +#define XE_DEFAULT_WEDGED_MODE XE_WEDGED_MODE_UPON_CRITICAL_ERROR +#define XE_DEFAULT_WEDGED_MODE_STR "upon-critical-error" +#define XE_DEFAULT_SVM_NOTIFIER_SIZE 512 + +#endif diff --git a/drivers/gpu/drm/xe/xe_dep_job_types.h b/drivers/gpu/drm/xe/xe_dep_job_types.h new file mode 100644 index 000000000..c6a484f24 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dep_job_types.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_DEP_JOB_TYPES_H_ +#define _XE_DEP_JOB_TYPES_H_ + +#include + +struct xe_dep_job; + +/** struct xe_dep_job_ops - Generic Xe dependency job operations */ +struct xe_dep_job_ops { + /** @run_job: Run generic Xe dependency job */ + struct dma_fence *(*run_job)(struct xe_dep_job *job); + /** @free_job: Free generic Xe dependency job */ + void (*free_job)(struct xe_dep_job *job); +}; + +/** struct xe_dep_job - Generic dependency Xe job */ +struct xe_dep_job { + /** @drm: base DRM scheduler job */ + struct drm_sched_job drm; + /** @ops: dependency job operations */ + const struct xe_dep_job_ops *ops; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c new file mode 100644 index 000000000..51d99fee9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include + +#include "xe_dep_job_types.h" +#include "xe_dep_scheduler.h" +#include "xe_device_types.h" + +/** + * DOC: Xe Dependency Scheduler + * + * The Xe dependency scheduler is a simple wrapper built around the DRM + * scheduler to execute jobs once their dependencies are resolved (i.e., all + * input fences specified as dependencies are signaled). The jobs that are + * executed contain virtual functions to run (execute) and free the job, + * allowing a single dependency scheduler to handle jobs performing different + * operations. + * + * Example use cases include deferred resource freeing, TLB invalidations after + * bind jobs, etc. + */ + +/** struct xe_dep_scheduler - Generic Xe dependency scheduler */ +struct xe_dep_scheduler { + /** @sched: DRM GPU scheduler */ + struct drm_gpu_scheduler sched; + /** @entity: DRM scheduler entity */ + struct drm_sched_entity entity; + /** @rcu: For safe freeing of exported dma fences */ + struct rcu_head rcu; +}; + +static struct dma_fence *xe_dep_scheduler_run_job(struct drm_sched_job *drm_job) +{ + struct xe_dep_job *dep_job = + container_of(drm_job, typeof(*dep_job), drm); + + return dep_job->ops->run_job(dep_job); +} + +static void xe_dep_scheduler_free_job(struct drm_sched_job *drm_job) +{ + struct xe_dep_job *dep_job = + container_of(drm_job, typeof(*dep_job), drm); + + dep_job->ops->free_job(dep_job); +} + +static const struct drm_sched_backend_ops sched_ops = { + .run_job = xe_dep_scheduler_run_job, + .free_job = xe_dep_scheduler_free_job, +}; + +/** + * xe_dep_scheduler_create() - Generic Xe dependency scheduler create + * @xe: Xe device + * @submit_wq: Submit workqueue struct (can be NULL) + * @name: Name of dependency scheduler + * @job_limit: Max dependency jobs that can be scheduled + * + * Create a generic Xe dependency scheduler and initialize internal DRM + * scheduler objects. + * + * Return: Generic Xe dependency scheduler object on success, ERR_PTR failure + */ +struct xe_dep_scheduler * +xe_dep_scheduler_create(struct xe_device *xe, + struct workqueue_struct *submit_wq, + const char *name, u32 job_limit) +{ + struct xe_dep_scheduler *dep_scheduler; + struct drm_gpu_scheduler *sched; + const struct drm_sched_init_args args = { + .ops = &sched_ops, + .submit_wq = submit_wq, + .num_rqs = 1, + .credit_limit = job_limit, + .timeout = MAX_SCHEDULE_TIMEOUT, + .name = name, + .dev = xe->drm.dev, + }; + int err; + + dep_scheduler = kzalloc_obj(*dep_scheduler); + if (!dep_scheduler) + return ERR_PTR(-ENOMEM); + + err = drm_sched_init(&dep_scheduler->sched, &args); + if (err) + goto err_free; + + sched = &dep_scheduler->sched; + err = drm_sched_entity_init(&dep_scheduler->entity, 0, &sched, 1, NULL); + if (err) + goto err_sched; + + init_rcu_head(&dep_scheduler->rcu); + + return dep_scheduler; + +err_sched: + drm_sched_fini(&dep_scheduler->sched); +err_free: + kfree(dep_scheduler); + + return ERR_PTR(err); +} + +/** + * xe_dep_scheduler_fini() - Generic Xe dependency scheduler finalize + * @dep_scheduler: Generic Xe dependency scheduler object + * + * Finalize internal DRM scheduler objects and free generic Xe dependency + * scheduler object + */ +void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler) +{ + drm_sched_entity_fini(&dep_scheduler->entity); + drm_sched_fini(&dep_scheduler->sched); + /* + * RCU free due sched being exported via DRM scheduler fences + * (timeline name). + */ + kfree_rcu(dep_scheduler, rcu); +} + +/** + * xe_dep_scheduler_entity() - Retrieve a generic Xe dependency scheduler + * DRM scheduler entity + * @dep_scheduler: Generic Xe dependency scheduler object + * + * Return: The generic Xe dependency scheduler's DRM scheduler entity + */ +struct drm_sched_entity * +xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler) +{ + return &dep_scheduler->entity; +} diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.h b/drivers/gpu/drm/xe/xe_dep_scheduler.h new file mode 100644 index 000000000..f314fb5d8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_DEP_SCHEDULER_H_ +#define _XE_DEP_SCHEDULER_H_ + +#include + +struct drm_sched_entity; +struct workqueue_struct; +struct xe_dep_scheduler; +struct xe_device; + +struct xe_dep_scheduler * +xe_dep_scheduler_create(struct xe_device *xe, + struct workqueue_struct *submit_wq, + const char *name, u32 job_limit); + +void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler); + +struct drm_sched_entity * +xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler); + +#endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c new file mode 100644 index 000000000..5f2b90b18 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_devcoredump.h" +#include "xe_devcoredump_types.h" + +#include +#include +#include + +#include + +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_guc_capture.h" +#include "xe_guc_ct.h" +#include "xe_guc_log.h" +#include "xe_guc_submit.h" +#include "xe_hw_engine.h" +#include "xe_pm.h" +#include "xe_sched_job.h" +#include "xe_vm.h" + +/** + * DOC: Xe device coredump + * + * Xe uses dev_coredump infrastructure for exposing the crash errors in a + * standardized way. Once a crash occurs, devcoredump exposes a temporary + * node under ``/sys/class/devcoredump/devcd/``. The same node is also + * accessible in ``/sys/class/drm/card/device/devcoredump/``. The + * ``failing_device`` symlink points to the device that crashed and created the + * coredump. + * + * The following characteristics are observed by xe when creating a device + * coredump: + * + * **Snapshot at hang**: + * The 'data' file contains a snapshot of the HW and driver states at the time + * the hang happened. Due to the driver recovering from resets/crashes, it may + * not correspond to the state of the system when the file is read by + * userspace. + * + * **Coredump release**: + * After a coredump is generated, it stays in kernel memory until released by + * userspace by writing anything to it, or after an internal timer expires. The + * exact timeout may vary and should not be relied upon. Example to release + * a coredump: + * + * .. code-block:: shell + * + * $ > /sys/class/drm/card0/device/devcoredump/data + * + * **First failure only**: + * In general, the first hang is the most critical one since the following + * hangs can be a consequence of the initial hang. For this reason a snapshot + * is taken only for the first failure. Until the devcoredump is released by + * userspace or kernel, all subsequent hangs do not override the snapshot nor + * create new ones. Devcoredump has a delayed work queue that will eventually + * delete the file node and free all the dump information. + */ + +#ifdef CONFIG_DEV_COREDUMP + +/* 1 hour timeout */ +#define XE_COREDUMP_TIMEOUT_JIFFIES (60 * 60 * HZ) + +static struct xe_device *coredump_to_xe(const struct xe_devcoredump *coredump) +{ + return container_of(coredump, struct xe_device, devcoredump); +} + +static struct xe_guc *exec_queue_to_guc(struct xe_exec_queue *q) +{ + return &q->gt->uc.guc; +} + +static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count, + ssize_t start, + struct xe_devcoredump *coredump) +{ + struct xe_device *xe; + struct xe_devcoredump_snapshot *ss; + struct drm_printer p; + struct drm_print_iterator iter; + struct timespec64 ts; + int i; + + xe = coredump_to_xe(coredump); + ss = &coredump->snapshot; + + iter.data = buffer; + iter.start = start; + iter.remain = count; + + p = drm_coredump_printer(&iter); + + drm_puts(&p, "**** Xe Device Coredump ****\n"); + drm_printf(&p, "Reason: %s\n", ss->reason); + drm_printf(&p, "kernel: %s\n", init_utsname()->release); + drm_puts(&p, "module: " KBUILD_MODNAME "\n"); + + ts = ktime_to_timespec64(ss->snapshot_time); + drm_printf(&p, "Snapshot time: %ptSp\n", &ts); + ts = ktime_to_timespec64(ss->boot_time); + drm_printf(&p, "Uptime: %ptSp\n", &ts); + drm_printf(&p, "Process: %s [%d]\n", ss->process_name, ss->pid); + xe_device_snapshot_print(xe, &p); + + drm_printf(&p, "\n**** GT #%d ****\n", ss->gt->info.id); + drm_printf(&p, "\tTile: %d\n", ss->gt->tile->id); + + drm_puts(&p, "\n**** GuC Log ****\n"); + xe_guc_log_snapshot_print(ss->guc.log, &p); + drm_puts(&p, "\n**** GuC CT ****\n"); + xe_guc_ct_snapshot_print(ss->guc.ct, &p); + + drm_puts(&p, "\n**** Contexts ****\n"); + xe_guc_exec_queue_snapshot_print(ss->ge, &p); + + drm_puts(&p, "\n**** Job ****\n"); + xe_sched_job_snapshot_print(ss->job, &p); + + drm_puts(&p, "\n**** HW Engines ****\n"); + for (i = 0; i < XE_NUM_HW_ENGINES; i++) + if (ss->hwe[i]) + xe_engine_snapshot_print(ss->hwe[i], &p); + + drm_puts(&p, "\n**** VM state ****\n"); + xe_vm_snapshot_print(ss->vm, &p); + + return count - iter.remain; +} + +static void xe_devcoredump_snapshot_free(struct xe_devcoredump_snapshot *ss) +{ + int i; + + kfree(ss->reason); + ss->reason = NULL; + + xe_guc_log_snapshot_free(ss->guc.log); + ss->guc.log = NULL; + + xe_guc_ct_snapshot_free(ss->guc.ct); + ss->guc.ct = NULL; + + if (!IS_ERR_OR_NULL(ss->gt)) + xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); + ss->matched_node = NULL; + + xe_guc_exec_queue_snapshot_free(ss->ge); + ss->ge = NULL; + + xe_sched_job_snapshot_free(ss->job); + ss->job = NULL; + + for (i = 0; i < XE_NUM_HW_ENGINES; i++) + if (ss->hwe[i]) { + xe_hw_engine_snapshot_free(ss->hwe[i]); + ss->hwe[i] = NULL; + } + + xe_vm_snapshot_free(ss->vm); + ss->vm = NULL; +} + +#define XE_DEVCOREDUMP_CHUNK_MAX (SZ_512M + SZ_1G) + +/** + * xe_devcoredump_read() - Read data from the Xe device coredump snapshot + * @buffer: Destination buffer to copy the coredump data into + * @offset: Offset in the coredump data to start reading from + * @count: Number of bytes to read + * @data: Pointer to the xe_devcoredump structure + * @datalen: Length of the data (unused) + * + * Reads a chunk of the coredump snapshot data into the provided buffer. + * If the devcoredump is smaller than 1.5 GB (XE_DEVCOREDUMP_CHUNK_MAX), + * it is read directly from a pre-written buffer. For larger devcoredumps, + * the pre-written buffer must be periodically repopulated from the snapshot + * state due to kmalloc size limitations. + * + * Return: Number of bytes copied on success, or a negative error code on failure. + */ +static ssize_t xe_devcoredump_read(char *buffer, loff_t offset, + size_t count, void *data, size_t datalen) +{ + struct xe_devcoredump *coredump = data; + struct xe_devcoredump_snapshot *ss; + ssize_t byte_copied = 0; + u32 chunk_offset; + ssize_t new_chunk_position; + bool pm_needed = false; + int ret = 0; + + if (!coredump) + return -ENODEV; + + ss = &coredump->snapshot; + + /* Ensure delayed work is captured before continuing */ + flush_work(&ss->work); + + pm_needed = ss->read.size > XE_DEVCOREDUMP_CHUNK_MAX; + if (pm_needed) + xe_pm_runtime_get(gt_to_xe(ss->gt)); + + mutex_lock(&coredump->lock); + + if (!ss->read.buffer) { + ret = -ENODEV; + goto unlock; + } + + if (offset >= ss->read.size) + goto unlock; + + new_chunk_position = div_u64_rem(offset, + XE_DEVCOREDUMP_CHUNK_MAX, + &chunk_offset); + + if (offset >= ss->read.chunk_position + XE_DEVCOREDUMP_CHUNK_MAX || + offset < ss->read.chunk_position) { + ss->read.chunk_position = new_chunk_position * + XE_DEVCOREDUMP_CHUNK_MAX; + + __xe_devcoredump_read(ss->read.buffer, + XE_DEVCOREDUMP_CHUNK_MAX, + ss->read.chunk_position, coredump); + } + + byte_copied = count < ss->read.size - offset ? count : + ss->read.size - offset; + memcpy(buffer, ss->read.buffer + chunk_offset, byte_copied); + +unlock: + mutex_unlock(&coredump->lock); + + if (pm_needed) + xe_pm_runtime_put(gt_to_xe(ss->gt)); + + return byte_copied ? byte_copied : ret; +} + +static void xe_devcoredump_free(void *data) +{ + struct xe_devcoredump *coredump = data; + + /* Our device is gone. Nothing to do... */ + if (!data || !coredump_to_xe(coredump)) + return; + + if (coredump->captured) + cancel_work_sync(&coredump->snapshot.work); + + mutex_lock(&coredump->lock); + + xe_devcoredump_snapshot_free(&coredump->snapshot); + kvfree(coredump->snapshot.read.buffer); + + /* To prevent stale data on next snapshot, clear everything */ + memset(&coredump->snapshot, 0, sizeof(coredump->snapshot)); + coredump->captured = false; + drm_info(&coredump_to_xe(coredump)->drm, + "Xe device coredump has been deleted.\n"); + + mutex_unlock(&coredump->lock); +} + +static void xe_devcoredump_deferred_snap_work(struct work_struct *work) +{ + struct xe_devcoredump_snapshot *ss = container_of(work, typeof(*ss), work); + struct xe_devcoredump *coredump = container_of(ss, typeof(*coredump), snapshot); + struct xe_device *xe = coredump_to_xe(coredump); + + /* + * NB: Despite passing a GFP_ flags parameter here, more allocations are done + * internally using GFP_KERNEL explicitly. Hence this call must be in the worker + * thread and not in the initial capture call. + */ + dev_coredumpm_timeout(gt_to_xe(ss->gt)->drm.dev, THIS_MODULE, coredump, 0, GFP_KERNEL, + xe_devcoredump_read, xe_devcoredump_free, + XE_COREDUMP_TIMEOUT_JIFFIES); + + guard(xe_pm_runtime)(xe); + + /* keep going if fw fails as we still want to save the memory and SW data */ + xe_with_force_wake(fw_ref, gt_to_fw(ss->gt), XE_FORCEWAKE_ALL) { + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n"); + xe_vm_snapshot_capture_delayed(ss->vm); + xe_guc_exec_queue_snapshot_capture_delayed(ss->ge); + } + + ss->read.chunk_position = 0; + + /* Calculate devcoredump size */ + ss->read.size = __xe_devcoredump_read(NULL, LONG_MAX, 0, coredump); + + if (ss->read.size > XE_DEVCOREDUMP_CHUNK_MAX) { + ss->read.buffer = kvmalloc(XE_DEVCOREDUMP_CHUNK_MAX, + GFP_USER); + if (!ss->read.buffer) + return; + + __xe_devcoredump_read(ss->read.buffer, + XE_DEVCOREDUMP_CHUNK_MAX, + 0, coredump); + } else { + ss->read.buffer = kvmalloc(ss->read.size, GFP_USER); + if (!ss->read.buffer) + return; + + __xe_devcoredump_read(ss->read.buffer, ss->read.size, 0, + coredump); + xe_devcoredump_snapshot_free(ss); + } +} + +static void devcoredump_snapshot(struct xe_devcoredump *coredump, + struct xe_exec_queue *q, + struct xe_sched_job *job) +{ + struct xe_devcoredump_snapshot *ss = &coredump->snapshot; + struct xe_guc *guc = exec_queue_to_guc(q); + const char *process_name = "no process"; + bool cookie; + + ss->snapshot_time = ktime_get_real(); + ss->boot_time = ktime_get_boottime(); + + if (q->vm && q->vm->xef) { + process_name = q->vm->xef->process_name; + ss->pid = q->vm->xef->pid; + } + + strscpy(ss->process_name, process_name); + + ss->gt = q->gt; + INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work); + + /* keep going if fw fails as we still want to save the memory and SW data */ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); + + cookie = dma_fence_begin_signalling(); + + ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true); + ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct); + ss->ge = xe_guc_exec_queue_snapshot_capture(q); + if (job) + ss->job = xe_sched_job_snapshot_capture(job); + ss->vm = xe_vm_snapshot_capture(q->vm); + + xe_engine_snapshot_capture_for_queue(q); + + queue_work(system_dfl_wq, &ss->work); + + dma_fence_end_signalling(cookie); +} + +/** + * xe_devcoredump - Take the required snapshots and initialize coredump device. + * @q: The faulty xe_exec_queue, where the issue was detected. + * @job: The faulty xe_sched_job, where the issue was detected. + * @fmt: Printf format + args to describe the reason for the core dump + * + * This function should be called at the crash time within the serialized + * gt_reset. It is skipped if we still have the core dump device available + * with the information of the 'first' snapshot. + */ +__printf(3, 4) +void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...) +{ + struct xe_device *xe = gt_to_xe(q->gt); + struct xe_devcoredump *coredump = &xe->devcoredump; + va_list varg; + + mutex_lock(&coredump->lock); + + if (coredump->captured) { + drm_dbg(&xe->drm, "Multiple hangs are occurring, but only the first snapshot was taken\n"); + mutex_unlock(&coredump->lock); + return; + } + + coredump->captured = true; + + va_start(varg, fmt); + coredump->snapshot.reason = kvasprintf(GFP_ATOMIC, fmt, varg); + va_end(varg); + + devcoredump_snapshot(coredump, q, job); + + drm_info(&xe->drm, "Xe device coredump has been created\n"); + drm_info(&xe->drm, "Check your /sys/class/drm/card%d/device/devcoredump/data\n", + xe->drm.primary->index); + + mutex_unlock(&coredump->lock); +} + +static void xe_driver_devcoredump_fini(void *arg) +{ + struct drm_device *drm = arg; + + dev_coredump_put(drm->dev); +} + +int xe_devcoredump_init(struct xe_device *xe) +{ + int err; + + err = drmm_mutex_init(&xe->drm, &xe->devcoredump.lock); + if (err) + return err; + + if (IS_ENABLED(CONFIG_LOCKDEP)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&xe->devcoredump.lock); + fs_reclaim_release(GFP_KERNEL); + } + + return devm_add_action_or_reset(xe->drm.dev, xe_driver_devcoredump_fini, &xe->drm); +} + +#endif + +/** + * xe_print_blob_ascii85 - print a BLOB to some useful location in ASCII85 + * + * The output is split into multiple calls to drm_puts() because some print + * targets, e.g. dmesg, cannot handle arbitrarily long lines. These targets may + * add newlines, as is the case with dmesg: each drm_puts() call creates a + * separate line. + * + * There is also a scheduler yield call to prevent the 'task has been stuck for + * 120s' kernel hang check feature from firing when printing to a slow target + * such as dmesg over a serial port. + * + * @p: the printer object to output to + * @prefix: optional prefix to add to output string + * @suffix: optional suffix to add at the end. 0 disables it and is + * not added to the output, which is useful when using multiple calls + * to dump data to @p + * @blob: the Binary Large OBject to dump out + * @offset: offset in bytes to skip from the front of the BLOB, must be a multiple of sizeof(u32) + * @size: the size in bytes of the BLOB, must be a multiple of sizeof(u32) + */ +void xe_print_blob_ascii85(struct drm_printer *p, const char *prefix, char suffix, + const void *blob, size_t offset, size_t size) +{ + const u32 *blob32 = (const u32 *)blob; + char buff[ASCII85_BUFSZ], *line_buff; + size_t line_pos = 0; + +#define DMESG_MAX_LINE_LEN 800 + /* Always leave space for the suffix char and the \0 */ +#define MIN_SPACE (ASCII85_BUFSZ + 2) /* 85 + "\0" */ + + if (size & 3) + drm_printf(p, "Size not word aligned: %zu", size); + if (offset & 3) + drm_printf(p, "Offset not word aligned: %zu", offset); + + line_buff = kzalloc(DMESG_MAX_LINE_LEN, GFP_ATOMIC); + if (!line_buff) { + drm_printf(p, "Failed to allocate line buffer\n"); + return; + } + + blob32 += offset / sizeof(*blob32); + size /= sizeof(*blob32); + + if (prefix) { + strscpy(line_buff, prefix, DMESG_MAX_LINE_LEN - MIN_SPACE - 2); + line_pos = strlen(line_buff); + + line_buff[line_pos++] = ':'; + line_buff[line_pos++] = ' '; + } + + while (size--) { + u32 val = *(blob32++); + + strscpy(line_buff + line_pos, ascii85_encode(val, buff), + DMESG_MAX_LINE_LEN - line_pos); + line_pos += strlen(line_buff + line_pos); + + if ((line_pos + MIN_SPACE) >= DMESG_MAX_LINE_LEN) { + line_buff[line_pos++] = 0; + + drm_puts(p, line_buff); + + line_pos = 0; + + /* Prevent 'stuck thread' time out errors */ + cond_resched(); + } + } + + if (suffix) + line_buff[line_pos++] = suffix; + + if (line_pos) { + line_buff[line_pos++] = 0; + drm_puts(p, line_buff); + } + + kfree(line_buff); + +#undef MIN_SPACE +#undef DMESG_MAX_LINE_LEN +} diff --git a/drivers/gpu/drm/xe/xe_devcoredump.h b/drivers/gpu/drm/xe/xe_devcoredump.h new file mode 100644 index 000000000..5391a80a4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_devcoredump.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_DEVCOREDUMP_H_ +#define _XE_DEVCOREDUMP_H_ + +#include + +struct drm_printer; +struct xe_device; +struct xe_exec_queue; +struct xe_sched_job; + +#ifdef CONFIG_DEV_COREDUMP +void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...); +int xe_devcoredump_init(struct xe_device *xe); +#else +static inline void xe_devcoredump(struct xe_exec_queue *q, + struct xe_sched_job *job, + const char *fmt, ...) +{ +} + +static inline int xe_devcoredump_init(struct xe_device *xe) +{ + return 0; +} +#endif + +void xe_print_blob_ascii85(struct drm_printer *p, const char *prefix, char suffix, + const void *blob, size_t offset, size_t size); + +#endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump_types.h b/drivers/gpu/drm/xe/xe_devcoredump_types.h new file mode 100644 index 000000000..a174385a6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_devcoredump_types.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_DEVCOREDUMP_TYPES_H_ +#define _XE_DEVCOREDUMP_TYPES_H_ + +#include +#include + +#include "xe_hw_engine_types.h" + +struct xe_device; +struct xe_gt; + +/** + * struct xe_devcoredump_snapshot - Crash snapshot + * + * This struct contains all the useful information quickly captured at the time + * of the crash. So, any subsequent reads of the coredump points to a data that + * shows the state of the GPU of when the issue has happened. + */ +struct xe_devcoredump_snapshot { + /** @snapshot_time: Time of this capture. */ + ktime_t snapshot_time; + /** @boot_time: Relative boot time so the uptime can be calculated. */ + ktime_t boot_time; + /** @process_name: Name of process that triggered this gpu hang */ + char process_name[TASK_COMM_LEN]; + /** @pid: Process id of process that triggered this gpu hang */ + pid_t pid; + /** @reason: The reason the coredump was triggered */ + char *reason; + + /** @gt: Affected GT, used by forcewake for delayed capture */ + struct xe_gt *gt; + /** @work: Workqueue for deferred capture outside of signaling context */ + struct work_struct work; + + /** @guc: GuC snapshots */ + struct { + /** @guc.ct: GuC CT snapshot */ + struct xe_guc_ct_snapshot *ct; + /** @guc.log: GuC log snapshot */ + struct xe_guc_log_snapshot *log; + } guc; + + /** @ge: GuC Submission Engine snapshot */ + struct xe_guc_submit_exec_queue_snapshot *ge; + + /** @hwe: HW Engine snapshot array */ + struct xe_hw_engine_snapshot *hwe[XE_NUM_HW_ENGINES]; + /** @job: Snapshot of job state */ + struct xe_sched_job_snapshot *job; + /** + * @matched_node: The matched capture node for timedout job + * this single-node tracker works because devcoredump will always only + * produce one hw-engine capture per devcoredump event + */ + struct __guc_capture_parsed_output *matched_node; + /** @vm: Snapshot of VM state */ + struct xe_vm_snapshot *vm; + + /** @read: devcoredump in human readable format */ + struct { + /** @read.size: size of devcoredump in human readable format */ + ssize_t size; + /** @read.chunk_position: position of devcoredump chunk */ + ssize_t chunk_position; + /** @read.buffer: buffer of devcoredump in human readable format */ + char *buffer; + } read; +}; + +/** + * struct xe_devcoredump - Xe devcoredump main structure + * + * This struct represents the live and active dev_coredump node. + * It is created/populated at the time of a crash/error. Then it + * is read later when user access the device coredump data file + * for reading the information. + */ +struct xe_devcoredump { + /** @lock: protects access to entire structure */ + struct mutex lock; + /** @captured: The snapshot of the first hang has already been taken */ + bool captured; + /** @snapshot: Snapshot is captured at time of the first crash */ + struct xe_devcoredump_snapshot snapshot; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c new file mode 100644 index 000000000..ee732e549 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device.c @@ -0,0 +1,1563 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_device.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "display/xe_display.h" +#include "instructions/xe_gpu_commands.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" +#include "xe_bo.h" +#include "xe_bo_evict.h" +#include "xe_configfs.h" +#include "xe_debugfs.h" +#include "xe_defaults.h" +#include "xe_devcoredump.h" +#include "xe_device_sysfs.h" +#include "xe_dma_buf.h" +#include "xe_drm_client.h" +#include "xe_drv.h" +#include "xe_exec.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_vf.h" +#include "xe_guc.h" +#include "xe_guc_pc.h" +#include "xe_hw_engine_group.h" +#include "xe_hwmon.h" +#include "xe_i2c.h" +#include "xe_irq.h" +#include "xe_late_bind_fw.h" +#include "xe_mmio.h" +#include "xe_module.h" +#include "xe_nvm.h" +#include "xe_oa.h" +#include "xe_observation.h" +#include "xe_pagefault.h" +#include "xe_pat.h" +#include "xe_pcode.h" +#include "xe_pm.h" +#include "xe_pmu.h" +#include "xe_psmi.h" +#include "xe_pxp.h" +#include "xe_query.h" +#include "xe_ras.h" +#include "xe_shrinker.h" +#include "xe_soc_remapper.h" +#include "xe_survivability_mode.h" +#include "xe_sriov.h" +#include "xe_svm.h" +#include "xe_sysctrl.h" +#include "xe_tile.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_ttm_sys_mgr.h" +#include "xe_vm.h" +#include "xe_vm_madvise.h" +#include "xe_vram.h" +#include "xe_vram_types.h" +#include "xe_vsec.h" +#include "xe_wait_user_fence.h" +#include "xe_wa.h" + +#include +#include + +static int xe_file_open(struct drm_device *dev, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_drm_client *client; + struct xe_file *xef; + int ret = -ENOMEM; + struct task_struct *task = NULL; + + xef = kzalloc_obj(*xef); + if (!xef) + return ret; + + client = xe_drm_client_alloc(); + if (!client) { + kfree(xef); + return ret; + } + + xef->drm = file; + xef->client = client; + xef->xe = xe; + + mutex_init(&xef->vm.lock); + xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); + + mutex_init(&xef->exec_queue.lock); + xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); + + file->driver_priv = xef; + kref_init(&xef->refcount); + + task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); + if (task) { + xef->process_name = kstrdup(task->comm, GFP_KERNEL); + xef->pid = task->pid; + put_task_struct(task); + } + + return 0; +} + +static void xe_file_destroy(struct kref *ref) +{ + struct xe_file *xef = container_of(ref, struct xe_file, refcount); + + xa_destroy(&xef->exec_queue.xa); + mutex_destroy(&xef->exec_queue.lock); + xa_destroy(&xef->vm.xa); + mutex_destroy(&xef->vm.lock); + + xe_drm_client_put(xef->client); + kfree(xef->process_name); + kfree(xef); +} + +/** + * xe_file_get() - Take a reference to the xe file object + * @xef: Pointer to the xe file + * + * Anyone with a pointer to xef must take a reference to the xe file + * object using this call. + * + * Return: xe file pointer + */ +struct xe_file *xe_file_get(struct xe_file *xef) +{ + kref_get(&xef->refcount); + return xef; +} + +/** + * xe_file_put() - Drop a reference to the xe file object + * @xef: Pointer to the xe file + * + * Used to drop reference to the xef object + */ +void xe_file_put(struct xe_file *xef) +{ + kref_put(&xef->refcount, xe_file_destroy); +} + +static void xe_file_close(struct drm_device *dev, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = file->driver_priv; + struct xe_vm *vm; + struct xe_exec_queue *q; + unsigned long idx; + + guard(xe_pm_runtime)(xe); + + /* + * No need for exec_queue.lock here as there is no contention for it + * when FD is closing as IOCTLs presumably can't be modifying the + * xarray. Taking exec_queue.lock here causes undue dependency on + * vm->lock taken during xe_exec_queue_kill(). + */ + xa_for_each(&xef->exec_queue.xa, idx, q) { + if (q->vm && q->hwe->hw_engine_group) + xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); + xe_exec_queue_kill(q); + xe_exec_queue_put(q); + } + xa_for_each(&xef->vm.xa, idx, vm) + xe_vm_close_and_put(vm); + + xe_file_put(xef); +} + +static const struct drm_ioctl_desc xe_ioctls[] = { + DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl, + DRM_RENDER_ALLOW), +}; + +static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct drm_file *file_priv = file->private_data; + struct xe_device *xe = to_xe_device(file_priv->minor->dev); + long ret; + + if (xe_device_wedged(xe)) + return -ECANCELED; + + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); + if (ret >= 0) + ret = drm_ioctl(file, cmd, arg); + + return ret; +} + +#ifdef CONFIG_COMPAT +static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct drm_file *file_priv = file->private_data; + struct xe_device *xe = to_xe_device(file_priv->minor->dev); + long ret; + + if (xe_device_wedged(xe)) + return -ECANCELED; + + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); + if (ret >= 0) + ret = drm_compat_ioctl(file, cmd, arg); + + return ret; +} +#else +/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ +#define xe_drm_compat_ioctl NULL +#endif + +static void barrier_open(struct vm_area_struct *vma) +{ + drm_dev_get(vma->vm_private_data); +} + +static void barrier_close(struct vm_area_struct *vma) +{ + drm_dev_put(vma->vm_private_data); +} + +static void barrier_release_dummy_page(struct drm_device *dev, void *res) +{ + struct page *dummy_page = (struct page *)res; + + __free_page(dummy_page); +} + +static vm_fault_t barrier_fault(struct vm_fault *vmf) +{ + struct drm_device *dev = vmf->vma->vm_private_data; + struct vm_area_struct *vma = vmf->vma; + vm_fault_t ret = VM_FAULT_NOPAGE; + pgprot_t prot; + int idx; + + prot = vma_get_page_prot(vma); + + if (drm_dev_enter(dev, &idx)) { + unsigned long pfn; + +#define LAST_DB_PAGE_OFFSET 0x7ff001 + pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + + LAST_DB_PAGE_OFFSET); + ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, + pgprot_noncached(prot)); + drm_dev_exit(idx); + } else { + struct page *page; + + /* Allocate new dummy page to map all the VA range in this VMA to it*/ + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + /* Set the page to be freed using drmm release action */ + if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) + return VM_FAULT_OOM; + + ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), + prot); + } + + return ret; +} + +static const struct vm_operations_struct vm_ops_barrier = { + .open = barrier_open, + .close = barrier_close, + .fault = barrier_fault, +}; + +static int xe_pci_barrier_mmap(struct file *filp, + struct vm_area_struct *vma) +{ + struct drm_file *priv = filp->private_data; + struct drm_device *dev = priv->minor->dev; + struct xe_device *xe = to_xe_device(dev); + + if (!IS_DGFX(xe)) + return -EINVAL; + + if (vma->vm_end - vma->vm_start > SZ_4K) + return -EINVAL; + + if (vma_is_cow_mapping(vma)) + return -EINVAL; + + if (vma->vm_flags & (VM_READ | VM_EXEC)) + return -EINVAL; + + vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); + vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); + vma->vm_ops = &vm_ops_barrier; + vma->vm_private_data = dev; + drm_dev_get(vma->vm_private_data); + + return 0; +} + +static int xe_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *priv = filp->private_data; + struct drm_device *dev = priv->minor->dev; + + if (drm_dev_is_unplugged(dev)) + return -ENODEV; + + switch (vma->vm_pgoff) { + case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: + return xe_pci_barrier_mmap(filp, vma); + } + + return drm_gem_mmap(filp, vma); +} + +static const struct file_operations xe_driver_fops = { + .owner = THIS_MODULE, + .open = drm_open, + .release = drm_release_noglobal, + .unlocked_ioctl = xe_drm_ioctl, + .mmap = xe_mmap, + .poll = drm_poll, + .read = drm_read, + .compat_ioctl = xe_drm_compat_ioctl, + .llseek = noop_llseek, +#ifdef CONFIG_PROC_FS + .show_fdinfo = drm_show_fdinfo, +#endif + .fop_flags = FOP_UNSIGNED_OFFSET, +}; + +/** + * xe_is_xe_file() - Is the file an xe device file? + * @file: The file. + * + * Checks whether the file is opened against + * an xe device. + * + * Return: %true if an xe file, %false if not. + */ +bool xe_is_xe_file(const struct file *file) +{ + return file->f_op == &xe_driver_fops; +} + +static const struct drm_driver regular_driver = { + .driver_features = + XE_DISPLAY_DRIVER_FEATURES | + DRIVER_GEM | + DRIVER_RENDER | DRIVER_SYNCOBJ | + DRIVER_SYNCOBJ_TIMELINE, + .open = xe_file_open, + .postclose = xe_file_close, + + .gem_prime_import = xe_gem_prime_import, + + .dumb_create = xe_bo_dumb_create, + .dumb_map_offset = drm_gem_ttm_dumb_map_offset, +#ifdef CONFIG_PROC_FS + .show_fdinfo = xe_drm_client_fdinfo, +#endif + .ioctls = xe_ioctls, + .num_ioctls = ARRAY_SIZE(xe_ioctls), + .fops = &xe_driver_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, + XE_DISPLAY_DRIVER_OPS, +}; + +#ifdef CONFIG_PCI_IOV +static const struct drm_ioctl_desc xe_ioctls_admin_only[] = { + DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), +}; + +static const struct drm_driver admin_only_driver = { + .driver_features = + DRIVER_GEM | DRIVER_RENDER, + .open = xe_file_open, + .postclose = xe_file_close, + .ioctls = xe_ioctls_admin_only, + .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only), + .fops = &xe_driver_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +/** + * xe_device_is_admin_only() - Check whether device is admin only or not. + * @xe: the &xe_device to check + * + * Return: true if the device is admin only, false otherwise. + */ +bool xe_device_is_admin_only(const struct xe_device *xe) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe); + return xe->drm.driver == &admin_only_driver; +} +#endif + +static void xe_device_destroy(struct drm_device *dev, void *dummy) +{ + struct xe_device *xe = to_xe_device(dev); + + xe_bo_dev_fini(&xe->bo_device); + + if (xe->preempt_fence_wq) + destroy_workqueue(xe->preempt_fence_wq); + + if (xe->ordered_wq) + destroy_workqueue(xe->ordered_wq); + + if (xe->unordered_wq) + destroy_workqueue(xe->unordered_wq); + + if (xe->destroy_wq) + destroy_workqueue(xe->destroy_wq); + + ttm_device_fini(&xe->ttm); +} + +/** + * xe_device_create() - Create a new &xe_device instance + * @pdev: the parent &pci_dev + * + * Allocate and initialize a device managed Xe device structure. + * + * Return: pointer to new &xe_device on success, or ERR_PTR on failure. + */ +struct xe_device *xe_device_create(struct pci_dev *pdev) +{ + const struct drm_driver *driver = ®ular_driver; + struct xe_device *xe; + int err; + +#ifdef CONFIG_PCI_IOV + /* + * Since XE device is not initialized yet, read from configfs + * directly to decide whether we are in admin-only PF mode or not. + */ + if (xe_configfs_admin_only_pf(pdev)) + driver = &admin_only_driver; +#endif + + err = aperture_remove_conflicting_pci_devices(pdev, driver->name); + if (err) + return ERR_PTR(err); + + xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm); + if (IS_ERR(xe)) + return xe; + + err = xe_device_init_early(xe); + if (err) + return ERR_PTR(err); + + return xe; +} +ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_device_init_early() - Initialize a new &xe_device instance + * @xe: the &xe_device to initialize + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_device_init_early(struct xe_device *xe) +{ + int err; + + err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, + xe->drm.anon_inode->i_mapping, + xe->drm.vma_offset_manager, + TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M))); + if (err) + return err; + + xe_bo_dev_init(&xe->bo_device); + err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); + if (err) + return err; + + err = xe_shrinker_create(xe); + if (err) + return err; + + xe->atomic_svm_timeslice_ms = 5; + xe->min_run_period_lr_ms = 5; + + err = xe_irq_init(xe); + if (err) + return err; + + xe_validation_device_init(&xe->val); + + init_waitqueue_head(&xe->ufence_wq); + + init_rwsem(&xe->usm.lock); + + err = xe_pagemap_shrinker_create(xe); + if (err) + return err; + + xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + /* Trigger a large asid and an early asid wrap. */ + u32 asid; + + BUILD_BUG_ON(XE_MAX_ASID < 2); + err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, + XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), + &xe->usm.next_asid, GFP_KERNEL); + drm_WARN_ON(&xe->drm, err); + if (err >= 0) + xa_erase(&xe->usm.asid_to_vm, asid); + } + + err = xe_bo_pinned_init(xe); + if (err) + return err; + + xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", + WQ_MEM_RECLAIM); + xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); + xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0); + xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0); + if (!xe->ordered_wq || !xe->unordered_wq || + !xe->preempt_fence_wq || !xe->destroy_wq) { + /* + * Cleanup done in xe_device_destroy via + * drmm_add_action_or_reset register above + */ + drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); + return -ENOMEM; + } + + err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); + if (err) + return err; + + err = xe_pm_init_early(xe); + if (err) + return err; + + return 0; +} + +static bool xe_driver_flr_disabled(struct xe_device *xe) +{ + if (IS_SRIOV_VF(xe)) + return true; + + if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { + drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); + return true; + } + + return false; +} + +/* + * The driver-initiated FLR is the highest level of reset that we can trigger + * from within the driver. It is different from the PCI FLR in that it doesn't + * fully reset the SGUnit and doesn't modify the PCI config space and therefore + * it doesn't require a re-enumeration of the PCI BARs. However, the + * driver-initiated FLR does still cause a reset of both GT and display and a + * memory wipe of local and stolen memory, so recovery would require a full HW + * re-init and saving/restoring (or re-populating) the wiped memory. Since we + * perform the FLR as the very last action before releasing access to the HW + * during the driver release flow, we don't attempt recovery at all, because + * if/when a new instance of Xe is bound to the device it will do a full + * re-init anyway. + */ +static void __xe_driver_flr(struct xe_device *xe) +{ + const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + int ret; + + drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); + + /* + * Make sure any pending FLR requests have cleared by waiting for the + * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS + * to make sure it's not still set from a prior attempt (it's a write to + * clear bit). + * Note that we should never be in a situation where a previous attempt + * is still pending (unless the HW is totally dead), but better to be + * safe in case something unexpected happens + */ + ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); + if (ret) { + drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); + return; + } + xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); + + /* Trigger the actual Driver-FLR */ + xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); + + /* Wait for hardware teardown to complete */ + ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); + if (ret) { + drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); + return; + } + + /* Wait for hardware/firmware re-init to complete */ + ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, + flr_timeout, NULL, false); + if (ret) { + drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); + return; + } + + /* Clear sticky completion status */ + xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); +} + +static void xe_driver_flr(struct xe_device *xe) +{ + if (xe_driver_flr_disabled(xe)) + return; + + __xe_driver_flr(xe); +} + +static void xe_driver_flr_fini(void *arg) +{ + struct xe_device *xe = arg; + + if (xe->needs_flr_on_fini) + xe_driver_flr(xe); +} + +static void xe_device_sanitize(void *arg) +{ + struct xe_device *xe = arg; + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) + xe_gt_sanitize(gt); +} + +static int xe_set_dma_info(struct xe_device *xe) +{ + unsigned int mask_size = xe->info.dma_mask_size; + int err; + + dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); + + err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); + if (err) + goto mask_err; + + err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); + if (err) + goto mask_err; + + return 0; + +mask_err: + drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); + return err; +} + +static void assert_lmem_ready(struct xe_device *xe) +{ + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) + return; + + xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & + LMEM_INIT); +} + +static void vf_update_device_info(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_VF(xe)); + /* disable features that are not available/applicable to VFs */ + xe->info.probe_display = 0; + xe->info.has_heci_cscfi = 0; + xe->info.has_heci_gscfi = 0; + xe->info.has_i2c = 0; + xe->info.has_late_bind = 0; + xe->info.skip_guc_pc = 1; + xe->info.skip_pcode = 1; + xe->info.has_drm_ras = false; +} + +static int xe_device_vram_alloc(struct xe_device *xe) +{ + struct xe_vram_region *vram; + + if (!IS_DGFX(xe)) + return 0; + + vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); + if (!vram) + return -ENOMEM; + + xe->mem.vram = vram; + return 0; +} + +/** + * xe_device_probe_early: Device early probe + * @xe: xe device instance + * + * Initialize MMIO resources that don't require any + * knowledge about tile count. Also initialize pcode and + * check vram initialization on root tile. + * + * Return: 0 on success, error code on failure + */ +int xe_device_probe_early(struct xe_device *xe) +{ + int err; + + xe_wa_device_init(xe); + xe_wa_process_device_oob(xe); + + err = xe_mmio_probe_early(xe); + if (err) + return err; + + xe_sriov_probe_early(xe); + + if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) { + xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n"); + return -ENODEV; + } + + if (IS_SRIOV_VF(xe)) + vf_update_device_info(xe); + + /* + * Check for pcode uncore_init status to confirm if the SoC + * initialization is complete. Until done, any MMIO or lmem access from + * the driver will be blocked + */ + err = xe_pcode_probe_early(xe); + if (err || xe_survivability_mode_is_requested(xe)) { + int save_err = err; + + /* + * Try to leave device in survivability mode if device is + * possible, but still return the previous error for error + * propagation + */ + err = xe_survivability_mode_boot_enable(xe); + if (err) + return err; + + return save_err; + } + + /* + * Make sure the lmem is initialized and ready to use. xe_pcode_ready() + * is flagged after full initialization is complete. Assert if lmem is + * not initialized. + */ + assert_lmem_ready(xe); + + xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ? + XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode; + drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n", + xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode)); + + err = xe_device_vram_alloc(xe); + if (err) + return err; + + return 0; +} +ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ + +static int probe_has_flat_ccs(struct xe_device *xe) +{ + struct xe_gt *gt; + u32 reg; + + /* Always enabled/disabled, no runtime check to do */ + if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) + return 0; + + gt = xe_root_mmio_gt(xe); + if (!gt) + return 0; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); + xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); + + if (!xe->info.has_flat_ccs) + drm_dbg(&xe->drm, + "Flat CCS has been disabled in bios, May lead to performance impact"); + + return 0; +} + +/* + * Detect if the driver is being run on pre-production hardware. We don't + * keep workarounds for pre-production hardware long term, so print an + * error and add taint if we're being loaded on a pre-production platform + * for which the pre-prod workarounds have already been removed. + * + * The general policy is that we'll remove any workarounds that only apply to + * pre-production hardware around the time force_probe restrictions are lifted + * for a platform of the next major IP generation (for example, Xe2 pre-prod + * workarounds should be removed around the time the first Xe3 platforms have + * force_probe lifted). + */ +static void detect_preproduction_hw(struct xe_device *xe) +{ + struct xe_gt *gt; + int id; + + /* + * SR-IOV VFs don't have access to the FUSE2 register, so we can't + * check pre-production status there. But the host OS will notice + * and report the pre-production status, which should be enough to + * help us catch mistaken use of pre-production hardware. + */ + if (IS_SRIOV_VF(xe)) + return; + + /* + * The "SW_CAP" fuse contains a bit indicating whether the device is a + * production or pre-production device. This fuse is reflected through + * the GT "FUSE2" register, even though the contents of the fuse are + * not GT-specific. Every GT's reflection of this fuse should show the + * same value, so we'll just use the first available GT for lookup. + */ + for_each_gt(gt, xe, id) + break; + + if (!gt) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) { + xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n"); + return; + } + + if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW) + return; + + xe_info(xe, "Pre-production hardware detected.\n"); + if (!xe->info.has_pre_prod_wa) { + xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n"); + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); + } +} + +static void xe_device_wedged_fini(struct drm_device *drm, void *arg) +{ + struct xe_device *xe = arg; + + if (atomic_read(&xe->wedged.flag)) + xe_pm_runtime_put(xe); +} + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe) +{ + int err; + + err = drmm_mutex_init(&xe->drm, &xe->page_size_alloc_ctrl.lock); + if (err) + return err; + + xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE; + xe->page_size_alloc_ctrl.cur_index = 0; + + return 0; +} +#else +static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe) +{ + return 0; +} +#endif + +int xe_device_probe(struct xe_device *xe) +{ + struct xe_tile *tile; + struct xe_gt *gt; + int err; + u8 id; + + xe_pat_init_early(xe); + + err = xe_sriov_init(xe); + if (err) + return err; + + xe->info.mem_region_mask = 1; + + err = xe_set_dma_info(xe); + if (err) + return err; + + err = xe_mmio_probe_tiles(xe); + if (err) + return err; + + for_each_gt(gt, xe, id) { + err = xe_gt_init_early(gt); + if (err) + return err; + } + + /* + * Wa_16029380221: The affected GT will always use non-coherent + * access to page tables, so we must do uncached writes from the + * CPU. + */ + for_each_gt(gt, xe, id) + if (XE_GT_WA(gt, 16029380221)) + xe->info.has_cached_pt = false; + + for_each_tile(tile, xe, id) { + err = xe_ggtt_init_early(tile->mem.ggtt); + if (err) + return err; + } + + /* + * From here on, if a step fails, make sure a Driver-FLR is triggereed + */ + err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); + if (err) + return err; + + err = probe_has_flat_ccs(xe); + if (err) + return err; + + err = xe_vram_probe(xe); + if (err) + return err; + + for_each_tile(tile, xe, id) { + err = xe_tile_init_noalloc(tile); + if (err) + return err; + } + + /* + * Allow allocations only now to ensure xe_display_init_early() + * is the first to allocate, always. + */ + err = xe_ttm_sys_mgr_init(xe); + if (err) + return err; + + /* Allocate and map stolen after potential VRAM resize */ + err = xe_ttm_stolen_mgr_init(xe); + if (err) + return err; + + err = xe_soc_remapper_init(xe); + if (err) + return err; + + err = xe_sysctrl_init(xe); + if (err) + return err; + + xe_ras_init(xe); + + /* + * Now that GT is initialized (TTM in particular), + * we can try to init display, and inherit the initial fb. + * This is the reason the first allocation needs to be done + * inside display. + */ + err = xe_display_init_early(xe); + if (err) + return err; + + for_each_tile(tile, xe, id) { + err = xe_tile_init(tile); + if (err) + return err; + } + + err = xe_irq_install(xe); + if (err) + return err; + + for_each_gt(gt, xe, id) { + err = xe_gt_init(gt); + if (err) + return err; + } + + err = xe_pagefault_init(xe); + if (err) + return err; + + if (xe->tiles->media_gt && + XE_GT_WA(xe->tiles->media_gt, 15015404425_disable)) + XE_DEVICE_WA_DISABLE(xe, 15015404425); + + err = xe_devcoredump_init(xe); + if (err) + return err; + + xe_nvm_init(xe); + + err = xe_heci_gsc_init(xe); + if (err) + return err; + + err = xe_late_bind_init(&xe->late_bind); + if (err) + return err; + + err = xe_oa_init(xe); + if (err) + return err; + + err = xe_display_init(xe); + if (err) + return err; + + err = xe_pxp_init(xe); + if (err) + return err; + + err = xe_psmi_init(xe); + if (err) + return err; + + err = xe_debug_page_size_alloc_ctrl_init(xe); + if (err) + return err; + + err = drm_dev_register(&xe->drm, 0); + if (err) + return err; + + xe_display_register(xe); + + err = xe_oa_register(xe); + if (err) + goto err_unregister_display; + + err = xe_pmu_register(&xe->pmu); + if (err) + goto err_unregister_display; + + err = xe_device_sysfs_init(xe); + if (err) + goto err_unregister_display; + + xe_debugfs_register(xe); + + err = xe_hwmon_register(xe); + if (err) + goto err_unregister_display; + + err = xe_i2c_probe(xe); + if (err) + goto err_unregister_display; + + for_each_gt(gt, xe, id) + xe_gt_sanitize_freq(gt); + + xe_vsec_init(xe); + + err = xe_sriov_init_late(xe); + if (err) + goto err_unregister_display; + + detect_preproduction_hw(xe); + + err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe); + if (err) + goto err_unregister_display; + + /* + * Process and log any errors detected by hardware. Possible results can + * include declaring the device as wedged, which must be done only after + * xe_device_wedged_fini() is registered. + */ + xe_ras_process_errors(xe); + + err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); + if (err) + goto err_unregister_display; + + return 0; + +err_unregister_display: + xe_display_unregister(xe); + drm_dev_unregister(&xe->drm); + + return err; +} + +void xe_device_remove(struct xe_device *xe) +{ + xe_display_unregister(xe); + + drm_dev_unplug(&xe->drm); + + xe_bo_pci_dev_remove_all(xe); +} + +void xe_device_shutdown(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + drm_dbg(&xe->drm, "Shutting down device\n"); + + xe_display_shutdown(xe); + + xe_irq_suspend(xe); + + for_each_gt(gt, xe, id) + xe_gt_shutdown(gt); + + xe_display_shutdown_late(xe); + + if (!xe_driver_flr_disabled(xe)) { + /* BOOM! */ + __xe_driver_flr(xe); + } +} + +/** + * xe_device_wmb() - Device specific write memory barrier + * @xe: the &xe_device + * + * While wmb() is sufficient for a barrier if we use system memory, on discrete + * platforms with device memory we additionally need to issue a register write. + * Since it doesn't matter which register we write to, use the read-only VF_CAP + * register that is also marked as accessible by the VFs. + */ +void xe_device_wmb(struct xe_device *xe) +{ + wmb(); + if (IS_DGFX(xe)) + xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); +} + +/* + * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt. + */ +static void tdf_request_sync(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return; + + xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); + + /* + * FIXME: We can likely do better here with our choice of + * timeout. Currently we just assume the worst case, i.e. 150us, + * which is believed to be sufficient to cover the worst case + * scenario on current platforms if all cache entries are + * transient and need to be flushed.. + */ + if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, + 300, NULL, false)) + xe_gt_err_once(gt, "TD flush timeout\n"); + } +} + +/** + * xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW + * @xe: The device to check. + * + * Return: true if the HW device optimizing L2 flush, false otherwise. + */ +bool xe_device_is_l2_flush_optimized(struct xe_device *xe) +{ + /* XA is *always* flushed, like at the end-of-submssion (and maybe other + * places), just that internally as an optimisation hw doesn't need to make + * that a full flush (which will also include XA) when Media is + * off/powergated, since it doesn't need to worry about GT caches vs Media + * coherency, and only CPU vs GPU coherency, so can make that flush a + * targeted XA flush, since stuff tagged with XA now means it's shared with + * the CPU. The main implication is that we now need to somehow flush non-XA before + * freeing system memory pages, otherwise dirty cachelines could be flushed after the free + * (like if Media suddenly turns on and does a full flush) + */ + if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe)) + return true; + return false; +} + +void xe_device_l2_flush(struct xe_device *xe) +{ + struct xe_gt *gt; + + gt = xe_root_mmio_gt(xe); + if (!gt) + return; + + if (!XE_GT_WA(gt, 16023588340)) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return; + + spin_lock(>->global_invl_lock); + + xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); + if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true)) + xe_gt_err_once(gt, "Global invalidation timeout\n"); + + spin_unlock(>->global_invl_lock); +} + +/** + * xe_device_td_flush() - Flush transient L3 cache entries + * @xe: The device + * + * Display engine has direct access to memory and is never coherent with L3/L4 + * caches (or CPU caches), however KMD is responsible for specifically flushing + * transient L3 GPU cache entries prior to the flip sequence to ensure scanout + * can happen from such a surface without seeing corruption. + * + * Display surfaces can be tagged as transient by mapping it using one of the + * various L3:XD PAT index modes on Xe2. + * + * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed + * at the end of each submission via PIPE_CONTROL for compute/render, since SA + * Media is not coherent with L3 and we want to support render-vs-media + * usescases. For other engines like copy/blt the HW internally forces uncached + * behaviour, hence why we can skip the TDF on such platforms. + */ +void xe_device_td_flush(struct xe_device *xe) +{ + struct xe_gt *root_gt; + + /* + * From Xe3p onward the HW takes care of flush of TD entries also along + * with flushing XA entries, which will be at the usual sync points, + * like at the end of submission, so no manual flush is needed here. + */ + if (GRAPHICS_VER(xe) >= 35) + return; + + if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) + return; + + root_gt = xe_root_mmio_gt(xe); + if (!root_gt) + return; + + if (XE_GT_WA(root_gt, 16023588340)) { + /* A transient flush is not sufficient: flush the L2 */ + xe_device_l2_flush(xe); + } else { + xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); + tdf_request_sync(xe); + xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc); + } +} + +u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) +{ + return xe_device_has_flat_ccs(xe) ? + DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; +} + +/** + * xe_device_assert_mem_access - Inspect the current runtime_pm state. + * @xe: xe device instance + * + * To be used before any kind of memory access. It will splat a debug warning + * if the device is currently sleeping. But it doesn't guarantee in any way + * that the device is going to remain awake. Xe PM runtime get and put + * functions might be added to the outer bound of the memory access, while + * this check is intended for inner usage to splat some warning if the worst + * case has just happened. + */ +void xe_device_assert_mem_access(struct xe_device *xe) +{ + xe_assert(xe, !xe_pm_runtime_suspended(xe)); +} + +void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) +{ + struct xe_gt *gt; + u8 id; + + drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); + drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); + + for_each_gt(gt, xe, id) { + drm_printf(p, "GT id: %u\n", id); + drm_printf(p, "\tTile: %u\n", gt->tile->id); + drm_printf(p, "\tType: %s\n", + gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); + drm_printf(p, "\tIP ver: %u.%u.%u\n", + REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), + REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), + REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); + drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); + } +} + +u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) +{ + return sign_extend64(address, xe->info.va_bits - 1); +} + +u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) +{ + return address & GENMASK_ULL(xe->info.va_bits - 1, 0); +} + +/** + * DOC: Xe Device Wedging + * + * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. + * When device is in wedged state, every IOCTL will be blocked and GT cannot + * be used. The conditions under which the driver declares the device wedged + * depend on the wedged mode configuration (see &enum xe_wedged_mode). The + * default recovery method for a wedged state is rebind/bus-reset. + * + * Another recovery method is vendor-specific. Below are the cases that send + * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. + * + * Case: Firmware Flash + * -------------------- + * + * Identification Hint + * +++++++++++++++++++ + * + * ``WEDGED=vendor-specific`` drm device wedged uevent with + * :ref:`Runtime Survivability mode ` is used to notify + * admin/userspace consumer about the need for a firmware flash. + * + * Recovery Procedure + * ++++++++++++++++++ + * + * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow + * the below steps + * + * - Check Runtime Survivability mode sysfs. + * If enabled, firmware flash is required to recover the device. + * + * /sys/bus/pci/devices//survivability_mode + * + * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash + * firmware and restore device to normal operation. + */ + +/** + * xe_device_set_wedged_method - Set wedged recovery method + * @xe: xe device instance + * @method: recovery method to set + * + * Set wedged recovery method to be sent in drm wedged uevent. + */ +void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method) +{ + xe->wedged.method = method; +} + +/** + * xe_device_declare_wedged - Declare device wedged + * @xe: xe device instance + * + * This is a final state that can only be cleared with the recovery method + * specified in the drm wedged uevent. The method can be set using + * xe_device_set_wedged_method before declaring the device as wedged. If no method + * is set, reprobe (unbind/re-bind) will be sent by default. + * + * In this state every IOCTL will be blocked so the GT cannot be used. + * In general it will be called upon any critical error such as gt reset + * failure or guc loading failure. Userspace will be notified of this state + * through device wedged uevent. + * If xe.wedged module parameter is set to 2, this function will be called + * on every single execution timeout (a.k.a. GPU hang) right after devcoredump + * snapshot capture. In this mode, GT reset won't be attempted so the state of + * the issue is preserved for further debugging. + */ +void xe_device_declare_wedged(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) { + drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); + return; + } + + if (!atomic_xchg(&xe->wedged.flag, 1)) { + xe->needs_flr_on_fini = true; + xe_pm_runtime_get_noresume(xe); + drm_err(&xe->drm, + "CRITICAL: Xe has declared device %s as wedged.\n" + "IOCTLs and executions are blocked.\n" + "For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n" + "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", + dev_name(xe->drm.dev)); + } + + for_each_gt(gt, xe, id) + xe_gt_declare_wedged(gt); + + if (xe_device_wedged(xe)) { + /* + * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging + * hangs, so wedge the device with 'none' recovery method and have + * it available to the user for debugging. + */ + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) + xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE); + /* If no wedge recovery method is set, use default */ + else if (!xe->wedged.method) + xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND | + DRM_WEDGE_RECOVERY_BUS_RESET); + + /* Notify userspace of wedged device */ + drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); + } +} + +/** + * xe_device_validate_wedged_mode - Check if given mode is supported + * @xe: the &xe_device + * @mode: requested mode to validate + * + * Check whether the provided wedged mode is supported. + * + * Return: 0 if mode is supported, error code otherwise. + */ +int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode) +{ + if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { + drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode); + return -EINVAL; + } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) || + (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) { + drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n", + mode, xe_wedged_mode_to_string(mode), + xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); + return -EPERM; + } + + return 0; +} + +/** + * xe_wedged_mode_to_string - Convert enum value to string. + * @mode: the &xe_wedged_mode to convert + * + * Returns: wedged mode as a user friendly string. + */ +const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode) +{ + switch (mode) { + case XE_WEDGED_MODE_NEVER: + return "never"; + case XE_WEDGED_MODE_UPON_CRITICAL_ERROR: + return "upon-critical-error"; + case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: + return "upon-any-hang-no-reset"; + default: + return ""; + } +} + +/** + * xe_device_asid_to_vm() - Find VM from ASID + * @xe: the &xe_device + * @asid: Address space ID + * + * Find a VM from ASID and take a reference to VM which caller must drop. + * Reclaim safe. + * + * Return: VM on success, ERR_PTR on failure + */ +struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid) +{ + struct xe_vm *vm; + + down_read(&xe->usm.lock); + vm = xa_load(&xe->usm.asid_to_vm, asid); + if (vm) + xe_vm_get(vm); + else + vm = ERR_PTR(-EINVAL); + up_read(&xe->usm.lock); + + return vm; +} diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h new file mode 100644 index 000000000..6c4cfaebc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device.h @@ -0,0 +1,300 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_DEVICE_H_ +#define _XE_DEVICE_H_ + +#include + +#include "xe_device_types.h" +#include "xe_gt_types.h" +#include "xe_sriov.h" + +struct xe_vm; + +static inline struct xe_device *to_xe_device(const struct drm_device *dev) +{ + return container_of(dev, struct xe_device, drm); +} + +static inline struct xe_device *kdev_to_xe_device(struct device *kdev) +{ + struct drm_device *drm = dev_get_drvdata(kdev); + + return drm ? to_xe_device(drm) : NULL; +} + +static inline struct xe_device *pdev_to_xe_device(struct pci_dev *pdev) +{ + struct drm_device *drm = pci_get_drvdata(pdev); + + return drm ? to_xe_device(drm) : NULL; +} + +static inline struct xe_device *xe_device_const_cast(const struct xe_device *xe) +{ + return (struct xe_device *)xe; +} + +static inline struct xe_device *ttm_to_xe_device(struct ttm_device *ttm) +{ + return container_of(ttm, struct xe_device, ttm); +} + +struct xe_device *xe_device_create(struct pci_dev *pdev); +int xe_device_init_early(struct xe_device *xe); +int xe_device_probe_early(struct xe_device *xe); +int xe_device_probe(struct xe_device *xe); +void xe_device_remove(struct xe_device *xe); +void xe_device_shutdown(struct xe_device *xe); + +void xe_device_wmb(struct xe_device *xe); + +static inline struct xe_file *to_xe_file(const struct drm_file *file) +{ + return file->driver_priv; +} + +static inline struct xe_tile *xe_device_get_root_tile(struct xe_device *xe) +{ + return &xe->tiles[0]; +} + +static inline struct xe_gt *xe_device_get_gt(struct xe_device *xe, u8 gt_id) +{ + struct xe_tile *tile; + struct xe_gt *gt; + + if (gt_id >= xe->info.tile_count * xe->info.max_gt_per_tile) + return NULL; + + tile = &xe->tiles[gt_id / xe->info.max_gt_per_tile]; + switch (gt_id % xe->info.max_gt_per_tile) { + default: + xe_assert(xe, false); + fallthrough; + case 0: + gt = tile->primary_gt; + break; + case 1: + gt = tile->media_gt; + break; + } + + if (!gt) + return NULL; + + drm_WARN_ON(&xe->drm, gt->info.id != gt_id); + drm_WARN_ON(&xe->drm, gt->info.type == XE_GT_TYPE_UNINITIALIZED); + + return gt; +} + +/* + * Provide a GT structure suitable for performing non-GT MMIO operations against + * the primary tile. Primarily intended for early tile initialization, display + * handling, top-most interrupt enable/disable, etc. Since anything using the + * MMIO handle returned by this function doesn't need GSI offset translation, + * we'll return the primary GT from the root tile. + * + * FIXME: Fix the driver design so that 'gt' isn't the target of all MMIO + * operations. + * + * Returns the primary gt of the root tile. + */ +static inline struct xe_gt *xe_root_mmio_gt(struct xe_device *xe) +{ + return xe_device_get_root_tile(xe)->primary_gt; +} + +static inline struct xe_mmio *xe_root_tile_mmio(struct xe_device *xe) +{ + return &xe->tiles[0].mmio; +} + +static inline bool xe_device_uc_enabled(struct xe_device *xe) +{ + return true; +} + +#define for_each_tile(tile__, xe__, id__) \ + for ((id__) = 0; (id__) < (xe__)->info.tile_count; (id__)++) \ + for_each_if((tile__) = &(xe__)->tiles[(id__)]) + +#define for_each_remote_tile(tile__, xe__, id__) \ + for ((id__) = 1; (id__) < (xe__)->info.tile_count; (id__)++) \ + for_each_if((tile__) = &(xe__)->tiles[(id__)]) + +#define for_each_gt(gt__, xe__, id__) \ + for ((id__) = 0; (id__) < (xe__)->info.tile_count * (xe__)->info.max_gt_per_tile; (id__)++) \ + for_each_if((gt__) = xe_device_get_gt((xe__), (id__))) + +#define for_each_gt_with_type(gt__, xe__, id__, typemask__) \ + for_each_gt((gt__), (xe__), (id__)) \ + for_each_if((typemask__) & BIT((gt__)->info.type)) + +#define for_each_gt_on_tile(gt__, tile__, id__) \ + for_each_gt((gt__), (tile__)->xe, (id__)) \ + for_each_if((gt__)->tile == (tile__)) + +static inline struct xe_force_wake *gt_to_fw(struct xe_gt *gt) +{ + return >->pm.fw; +} + +void xe_device_assert_mem_access(struct xe_device *xe); + +static inline bool xe_device_has_flat_ccs(const struct xe_device *xe) +{ + return xe->info.has_flat_ccs; +} + +static inline bool xe_device_has_sriov(const struct xe_device *xe) +{ + return xe->info.has_sriov; +} + +static inline bool xe_device_has_msix(const struct xe_device *xe) +{ + return xe->irq.msix.nvec > 0; +} + +static inline bool xe_device_has_memirq(const struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) >= 1250; +} + +static inline bool xe_device_uses_memirq(const struct xe_device *xe) +{ + return xe_device_has_memirq(xe) && (IS_SRIOV_VF(xe) || xe_device_has_msix(xe)); +} + +static inline bool xe_device_has_lmtt(const struct xe_device *xe) +{ + return IS_DGFX(xe); +} + +static inline bool xe_device_has_mert(const struct xe_device *xe) +{ + return xe->info.has_mert; +} + +static inline bool xe_device_is_in_reset(struct xe_device *xe) +{ + return atomic_read(&xe->in_reset); +} + +static inline void xe_device_set_in_reset(struct xe_device *xe) +{ + atomic_set(&xe->in_reset, 1); +} + +static inline void xe_device_clear_in_reset(struct xe_device *xe) +{ + atomic_set(&xe->in_reset, 0); +} + +u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size); + +void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p); + +u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address); +u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address); + +bool xe_device_is_l2_flush_optimized(struct xe_device *xe); +void xe_device_td_flush(struct xe_device *xe); +void xe_device_l2_flush(struct xe_device *xe); + +static inline bool xe_device_wedged(struct xe_device *xe) +{ + return atomic_read(&xe->wedged.flag); +} + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +static inline bool xe_debug_page_size_supported(struct xe_device *xe) +{ + return IS_DGFX(xe); +} + +static inline bool xe_debug_page_size_mode_not_none(struct xe_device *xe) +{ + enum xe_page_size_alloc_ctrl_mode mode; + + if (!xe_debug_page_size_supported(xe)) + return false; + + mode = READ_ONCE(xe->page_size_alloc_ctrl.mode); + + return mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M || + mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G || + mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED; +} + +static inline bool xe_debug_page_size_mode_is_mixed(struct xe_device *xe) +{ + enum xe_page_size_alloc_ctrl_mode mode; + + if (!xe_debug_page_size_supported(xe)) + return false; + + mode = READ_ONCE(xe->page_size_alloc_ctrl.mode); + + return mode == XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED; +} +#else +static inline bool xe_debug_page_size_supported(struct xe_device *xe) +{ + return false; +} + +static inline bool xe_debug_page_size_mode_not_none(struct xe_device *xe) +{ + return false; +} + +static inline bool xe_debug_page_size_mode_is_mixed(struct xe_device *xe) +{ + return false; +} +#endif + +void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method); +void xe_device_declare_wedged(struct xe_device *xe); +int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode); +const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode); + +struct xe_file *xe_file_get(struct xe_file *xef); +void xe_file_put(struct xe_file *xef); + +int xe_is_injection_active(void); + +bool xe_is_xe_file(const struct file *file); + +struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid); + +#ifdef CONFIG_PCI_IOV +bool xe_device_is_admin_only(const struct xe_device *xe); +#else +static inline bool xe_device_is_admin_only(const struct xe_device *xe) +{ + return false; +} +#endif + +/* + * Occasionally it is seen that the G2H worker starts running after a delay of more than + * a second even after being queued and activated by the Linux workqueue subsystem. This + * leads to G2H timeout error. The root cause of issue lies with scheduling latency of + * Lunarlake Hybrid CPU. Issue disappears if we disable Lunarlake atom cores from BIOS + * and this is beyond xe kmd. + * + * TODO: Drop this change once workqueue scheduling delay issue is fixed on LNL Hybrid CPU. + */ +#define LNL_FLUSH_WORKQUEUE(wq__) \ + flush_workqueue(wq__) +#define LNL_FLUSH_WORK(wrk__) \ + flush_work(wrk__) + +#endif diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c new file mode 100644 index 000000000..a73e0e957 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device_sysfs.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_device_sysfs.h" +#include "xe_mmio.h" +#include "xe_pcode_api.h" +#include "xe_pcode.h" +#include "xe_pm.h" + +/** + * DOC: Xe device sysfs + * Xe driver requires exposing certain tunable knobs controlled by user space for + * each graphics device. Considering this, we need to add sysfs attributes at device + * level granularity. + * These sysfs attributes will be available under pci device kobj directory. + * + * vram_d3cold_threshold - Report/change vram used threshold(in MB) below + * which vram save/restore is permissible during runtime D3cold entry/exit. + * + * lb_fan_control_version - Fan control version provisioned by late binding. + * Exposed only if supported by the device. + * + * lb_voltage_regulator_version - Voltage regulator version provisioned by late + * binding. Exposed only if supported by the device. + */ + +static ssize_t +vram_d3cold_threshold_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + + return sysfs_emit(buf, "%d\n", xe->d3cold.vram_threshold); +} + +static ssize_t +vram_d3cold_threshold_store(struct device *dev, struct device_attribute *attr, + const char *buff, size_t count) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + u32 vram_d3cold_threshold; + int ret; + + ret = kstrtou32(buff, 0, &vram_d3cold_threshold); + if (ret) + return ret; + + drm_dbg(&xe->drm, "vram_d3cold_threshold: %u\n", vram_d3cold_threshold); + + guard(xe_pm_runtime)(xe); + ret = xe_pm_set_vram_threshold(xe, vram_d3cold_threshold); + + return ret ?: count; +} + +static DEVICE_ATTR_RW(vram_d3cold_threshold); + +static struct attribute *vram_attrs[] = { + &dev_attr_vram_d3cold_threshold.attr, + NULL +}; + +static const struct attribute_group vram_attr_group = { + .attrs = vram_attrs, +}; + +static ssize_t +lb_fan_control_version_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); + struct xe_tile *root = xe_device_get_root_tile(xe); + u32 cap = 0, ver_low = FAN_TABLE, ver_high = FAN_TABLE; + u16 major = 0, minor = 0, hotfix = 0, build = 0; + int ret; + + guard(xe_pm_runtime)(xe); + + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0), + &cap, NULL); + if (ret) + return ret; + + if (REG_FIELD_GET(V1_FAN_PROVISIONED, cap)) { + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0), + &ver_low, NULL); + if (ret) + return ret; + + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0), + &ver_high, NULL); + if (ret) + return ret; + + major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low); + minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low); + hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high); + build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high); + } + + return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build); +} +static DEVICE_ATTR_ADMIN_RO(lb_fan_control_version); + +static ssize_t +lb_voltage_regulator_version_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); + struct xe_tile *root = xe_device_get_root_tile(xe); + u32 cap = 0, ver_low = VR_CONFIG, ver_high = VR_CONFIG; + u16 major = 0, minor = 0, hotfix = 0, build = 0; + int ret; + + guard(xe_pm_runtime)(xe); + + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0), + &cap, NULL); + if (ret) + return ret; + + if (REG_FIELD_GET(VR_PARAMS_PROVISIONED, cap)) { + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0), + &ver_low, NULL); + if (ret) + return ret; + + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0), + &ver_high, NULL); + if (ret) + return ret; + + major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low); + minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low); + hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high); + build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high); + } + + return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build); +} +static DEVICE_ATTR_ADMIN_RO(lb_voltage_regulator_version); + +static struct attribute *late_bind_attrs[] = { + &dev_attr_lb_fan_control_version.attr, + &dev_attr_lb_voltage_regulator_version.attr, + NULL +}; + +static umode_t late_bind_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); + struct xe_tile *root = xe_device_get_root_tile(xe); + u32 cap = 0; + int ret; + + ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0), + &cap, NULL); + if (ret) + return 0; + + if (attr == &dev_attr_lb_fan_control_version.attr && + REG_FIELD_GET(V1_FAN_SUPPORTED, cap)) + return attr->mode; + if (attr == &dev_attr_lb_voltage_regulator_version.attr && + REG_FIELD_GET(VR_PARAMS_SUPPORTED, cap)) + return attr->mode; + + return 0; +} + +static const struct attribute_group late_bind_attr_group = { + .attrs = late_bind_attrs, + .is_visible = late_bind_attr_is_visible, +}; + +/** + * DOC: PCIe Gen5 Limitations + * + * Default link speed of discrete GPUs is determined by configuration parameters + * stored in their flash memory, which are subject to override through user + * initiated firmware updates. It has been observed that devices configured with + * PCIe Gen5 as their default link speed can come across link quality issues due + * to host or motherboard limitations and may have to auto-downgrade their link + * to PCIe Gen4 speed when faced with unstable link at Gen5, which makes + * firmware updates rather risky on such setups. It is required to ensure that + * the device is capable of auto-downgrading its link to PCIe Gen4 speed before + * pushing the firmware image with PCIe Gen5 as default configuration. This can + * be done by reading ``auto_link_downgrade_capable`` sysfs entry, which will + * denote if the device is capable of auto-downgrading its link to PCIe Gen4 + * speed with boolean output value of ``0`` or ``1``, meaning `incapable` or + * `capable` respectively. + * + * .. code-block:: shell + * + * $ cat /sys/bus/pci/devices//auto_link_downgrade_capable + * + * Pushing the firmware image with PCIe Gen5 as default configuration on a auto + * link downgrade incapable device and facing link instability due to host or + * motherboard limitations can result in driver failing to bind to the device, + * making further firmware updates impossible with RMA being the only last + * resort. + * + * Link downgrade status of auto link downgrade capable devices is available + * through ``auto_link_downgrade_status`` sysfs entry with boolean output value + * of ``0`` or ``1``, where ``0`` means no auto-downgrading was required during + * link training (which is the optimal scenario) and ``1`` means the device has + * auto-downgraded its link to PCIe Gen4 speed due to unstable Gen5 link. + * + * .. code-block:: shell + * + * $ cat /sys/bus/pci/devices//auto_link_downgrade_status + */ + +static ssize_t +auto_link_downgrade_capable_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + u32 cap, val; + + guard(xe_pm_runtime)(xe); + val = xe_mmio_read32(xe_root_tile_mmio(xe), BMG_PCIE_CAP); + + cap = REG_FIELD_GET(LINK_DOWNGRADE, val); + return sysfs_emit(buf, "%u\n", cap == DOWNGRADE_CAPABLE); +} +static DEVICE_ATTR_ADMIN_RO(auto_link_downgrade_capable); + +static ssize_t +auto_link_downgrade_status_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + /* default the auto_link_downgrade status to 0 */ + u32 val = 0; + int ret; + + guard(xe_pm_runtime)(xe); + ret = xe_pcode_read(xe_device_get_root_tile(xe), + PCODE_MBOX(DGFX_PCODE_STATUS, DGFX_GET_INIT_STATUS, 0), + &val, NULL); + + return ret ?: sysfs_emit(buf, "%u\n", REG_FIELD_GET(DGFX_LINK_DOWNGRADE_STATUS, val)); +} +static DEVICE_ATTR_ADMIN_RO(auto_link_downgrade_status); + +static struct attribute *auto_link_downgrade_attrs[] = { + &dev_attr_auto_link_downgrade_capable.attr, + &dev_attr_auto_link_downgrade_status.attr, + NULL +}; + +static const struct attribute_group auto_link_downgrade_attr_group = { + .attrs = auto_link_downgrade_attrs, +}; + +int xe_device_sysfs_init(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + int ret; + + if (xe->d3cold.capable) { + ret = devm_device_add_group(dev, &vram_attr_group); + if (ret) + return ret; + } + + if (xe->info.platform == XE_BATTLEMAGE && !IS_SRIOV_VF(xe)) { + ret = devm_device_add_group(dev, &auto_link_downgrade_attr_group); + if (ret) + return ret; + + ret = devm_device_add_group(dev, &late_bind_attr_group); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.h b/drivers/gpu/drm/xe/xe_device_sysfs.h new file mode 100644 index 000000000..f9e83d8bd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device_sysfs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_DEVICE_SYSFS_H_ +#define _XE_DEVICE_SYSFS_H_ + +struct xe_device; + +int xe_device_sysfs_init(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h new file mode 100644 index 000000000..03a7bb08a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -0,0 +1,678 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022-2023 Intel Corporation + */ + +#ifndef _XE_DEVICE_TYPES_H_ +#define _XE_DEVICE_TYPES_H_ + +#include + +#include +#include +#include + +#include "xe_devcoredump_types.h" +#include "xe_drm_ras_types.h" +#include "xe_heci_gsc.h" +#include "xe_late_bind_fw_types.h" +#include "xe_oa_types.h" +#include "xe_pagefault_types.h" +#include "xe_platform_types.h" +#include "xe_pmu_types.h" +#include "xe_pt_types.h" +#include "xe_sriov_pf_types.h" +#include "xe_sriov_types.h" +#include "xe_sriov_vf_types.h" +#include "xe_sriov_vf_ccs_types.h" +#include "xe_step_types.h" +#include "xe_survivability_mode_types.h" +#include "xe_sysctrl_types.h" +#include "xe_tile_types.h" +#include "xe_validation.h" + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define TEST_VM_OPS_ERROR +#endif + +struct drm_pagemap_shrinker; +struct intel_display; +struct intel_dg_nvm_dev; +struct xe_ggtt; +struct xe_i2c; +struct xe_pat_ops; +struct xe_pxp; +struct xe_ttm_stolen_mgr; +struct xe_vram_region; + +/** + * enum xe_wedged_mode - possible wedged modes + * @XE_WEDGED_MODE_NEVER: Device will never be declared wedged. + * @XE_WEDGED_MODE_UPON_CRITICAL_ERROR: Device will be declared wedged only + * when critical error occurs like GT reset failure or firmware failure. + * This is the default mode. + * @XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: Device will be declared wedged on + * any hang. In this mode, engine resets are disabled to avoid automatic + * recovery attempts. This mode is primarily intended for debugging hangs. + */ +enum xe_wedged_mode { + XE_WEDGED_MODE_NEVER = 0, + XE_WEDGED_MODE_UPON_CRITICAL_ERROR = 1, + XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET = 2, +}; + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE +/** + * enum xe_page_size_alloc_ctrl_mode - User BO page-size allocation control modes + * @XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE: Use the normal allocation policy + * @XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M: Force user BO allocations to 2M pages + * @XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G: Force user BO allocations to 1G pages + * @XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED: Select page sizes in round-robin order + * (4K, 64K, 2M, 1G) + */ +enum xe_page_size_alloc_ctrl_mode { + XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE = 0, + XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M, + XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G, + XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED +}; +#endif + +#define XE_BO_INVALID_OFFSET LONG_MAX + +#define GRAPHICS_VER(xe) ((xe)->info.graphics_verx100 / 100) +#define MEDIA_VER(xe) ((xe)->info.media_verx100 / 100) +#define GRAPHICS_VERx100(xe) ((xe)->info.graphics_verx100) +#define MEDIA_VERx100(xe) ((xe)->info.media_verx100) +#define IS_DGFX(xe) ((xe)->info.is_dgfx) + +#define XE_VRAM_FLAGS_NEED64K BIT(0) + +#define XE_GT0 0 +#define XE_GT1 1 +#define XE_MAX_TILES_PER_DEVICE (XE_GT1 + 1) + +/* + * Highest GT/tile count for any platform. Used only for memory allocation + * sizing. Any logic looping over GTs or mapping userspace GT IDs into GT + * structures should use the per-platform xe->info.max_gt_per_tile instead. + */ +#define XE_MAX_GT_PER_TILE 2 + +#define XE_MAX_ASID (BIT(20)) + +/** + * struct xe_device - Top level struct of Xe device + */ +struct xe_device { + /** @drm: drm device */ + struct drm_device drm; + +#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) + /** @display: display device data, must be placed after drm device member */ + struct intel_display *display; +#endif + + /** @devcoredump: device coredump */ + struct xe_devcoredump devcoredump; + + /** @info: device info */ + struct intel_device_info { + /** @info.platform_name: platform name */ + const char *platform_name; + /** @info.graphics_name: graphics IP name */ + const char *graphics_name; + /** @info.media_name: media IP name */ + const char *media_name; + /** @info.graphics_verx100: graphics IP version */ + u32 graphics_verx100; + /** @info.media_verx100: media IP version */ + u32 media_verx100; + /** @info.mem_region_mask: mask of valid memory regions */ + u32 mem_region_mask; + /** @info.platform: Xe platform enum */ + enum xe_platform platform; + /** @info.subplatform: Xe subplatform enum */ + enum xe_subplatform subplatform; + /** @info.devid: device ID */ + u16 devid; + /** @info.revid: device revision */ + u8 revid; + /** @info.step: stepping information for each IP */ + struct xe_step_info step; + /** @info.dma_mask_size: DMA address bits */ + u8 dma_mask_size; + /** @info.vram_flags: Vram flags */ + u8 vram_flags; + /** @info.tile_count: Number of tiles */ + u8 tile_count; + /** @info.max_gt_per_tile: Number of GT IDs allocated to each tile */ + u8 max_gt_per_tile; + /** @info.multi_lrc_mask: bitmask of engine classes which support multi-lrc */ + u8 multi_lrc_mask; + /** @info.gt_count: Total number of GTs for entire device */ + u8 gt_count; + /** @info.vm_max_level: Max VM level */ + u8 vm_max_level; + /** @info.va_bits: Maximum bits of a virtual address */ + u8 va_bits; + + /* + * Keep all flags below alphabetically sorted + */ + + /** @info.has_access_counter: Device supports access counter */ + u8 has_access_counter:1; + /** @info.has_asid: Has address space ID */ + u8 has_asid:1; + /** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */ + u8 has_atomic_enable_pte_bit:1; + /** @info.has_cached_pt: Supports caching pagetable */ + u8 has_cached_pt:1; + /** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */ + u8 has_device_atomics_on_smem:1; + /** @info.has_drm_ras: Device supports drm_ras (Reliability, Availability, Serviceability) */ + u8 has_drm_ras:1; + /** @info.has_fan_control: Device supports fan control */ + u8 has_fan_control:1; + /** @info.has_flat_ccs: Whether flat CCS metadata is used */ + u8 has_flat_ccs:1; + /** @info.has_gsc_nvm: Device has gsc non-volatile memory */ + u8 has_gsc_nvm:1; + /** @info.has_heci_cscfi: device has heci cscfi */ + u8 has_heci_cscfi:1; + /** @info.has_heci_gscfi: device has heci gscfi */ + u8 has_heci_gscfi:1; + /** @info.has_i2c: Device has I2C controller */ + u8 has_i2c:1; + /** @info.has_late_bind: Device has firmware late binding support */ + u8 has_late_bind:1; + /** @info.has_llc: Device has a shared CPU+GPU last level cache */ + u8 has_llc:1; + /** @info.has_mbx_power_limits: Device has support to manage power limits using + * pcode mailbox commands. + */ + u8 has_mbx_power_limits:1; + /** @info.has_mbx_thermal_info: Device supports thermal mailbox commands */ + u8 has_mbx_thermal_info:1; + /** @info.has_mem_copy_instr: Device supports MEM_COPY instruction */ + u8 has_mem_copy_instr:1; + /** @info.has_mert: Device has standalone MERT */ + u8 has_mert:1; + /** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */ + u8 has_page_reclaim_hw_assist:1; + /** @info.has_pre_prod_wa: Pre-production workarounds still present in driver */ + u8 has_pre_prod_wa:1; + /** @info.has_pxp: Device has PXP support */ + u8 has_pxp:1; + /** @info.has_ctx_tlb_inval: Has context based TLB invalidations */ + u8 has_ctx_tlb_inval:1; + /** @info.has_range_tlb_inval: Has range based TLB invalidations */ + u8 has_range_tlb_inval:1; + /** @info.has_soc_remapper_sysctrl: Has SoC remapper system controller */ + u8 has_soc_remapper_sysctrl:1; + /** @info.has_soc_remapper_telem: Has SoC remapper telemetry support */ + u8 has_soc_remapper_telem:1; + /** @info.has_sriov: Supports SR-IOV */ + u8 has_sriov:1; + /** @info.has_sysctrl: Supports System Controller */ + u8 has_sysctrl:1; + /** @info.has_usm: Device has unified shared memory support */ + u8 has_usm:1; + /** @info.has_64bit_timestamp: Device supports 64-bit timestamps */ + u8 has_64bit_timestamp:1; + /** @info.is_dgfx: is discrete device */ + u8 is_dgfx:1; + /** @info.needs_scratch: needs scratch page for oob prefetch to work */ + u8 needs_scratch:1; + /** + * @info.probe_display: Probe display hardware. If set to + * false, the driver will behave as if there is no display + * hardware present and will not try to read/write to it in any + * way. The display hardware, if it exists, will not be + * exposed to userspace and will be left untouched in whatever + * state the firmware or bootloader left it in. + */ + u8 probe_display:1; + /** @info.skip_guc_pc: Skip GuC based PM feature init */ + u8 skip_guc_pc:1; + /** @info.skip_pcode: skip access to PCODE uC */ + u8 skip_pcode:1; + /** @info.needs_shared_vf_gt_wq: needs shared GT WQ on VF */ + u8 needs_shared_vf_gt_wq:1; + } info; + + /** @wa_active: keep track of active workarounds */ + struct { + /** @wa_active.oob: bitmap with active OOB workarounds */ + unsigned long *oob; + + /** + * @wa_active.oob_initialized: Mark oob as initialized to help detecting misuse + * of XE_DEVICE_WA() - it can only be called on initialization after + * Device OOB WAs have been processed. + */ + bool oob_initialized; + } wa_active; + + /** @survivability: survivability information for device */ + struct xe_survivability survivability; + + /** @irq: device interrupt state */ + struct { + /** @irq.lock: lock for processing irq's on this device */ + spinlock_t lock; + + /** @irq.enabled: interrupts enabled on this device */ + atomic_t enabled; + + /** @irq.msix: irq info for platforms that support MSI-X */ + struct { + /** @irq.msix.nvec: number of MSI-X interrupts */ + u16 nvec; + /** @irq.msix.indexes: used to allocate MSI-X indexes */ + struct xarray indexes; + } msix; + } irq; + + /** @ttm: ttm device */ + struct ttm_device ttm; + + /** @mmio: mmio info for device */ + struct { + /** @mmio.size: size of MMIO space for device */ + size_t size; + /** @mmio.regs: pointer to MMIO space for device */ + void __iomem *regs; + } mmio; + + /** @mem: memory info for device */ + struct { + /** @mem.vram: VRAM info for device */ + struct xe_vram_region *vram; + /** @mem.sys_mgr: system TTM manager */ + struct ttm_resource_manager sys_mgr; + /** @mem.shrinker: system memory shrinker. */ + struct xe_shrinker *shrinker; + /** @mem.stolen_mgr: stolen memory manager. */ + struct xe_ttm_stolen_mgr *stolen_mgr; + } mem; + + /** @sriov: device level virtualization data */ + struct { + /** @sriov.__mode: SR-IOV mode (Don't access directly!) */ + enum xe_sriov_mode __mode; + + union { + /** @sriov.pf: PF specific data */ + struct xe_device_pf pf; + /** @sriov.vf: VF specific data */ + struct xe_device_vf vf; + }; + + /** @sriov.wq: workqueue used by the virtualization workers */ + struct workqueue_struct *wq; + } sriov; + + /** @usm: unified memory state */ + struct { + /** @usm.asid_to_vm: convert an ASID to VM */ + struct xarray asid_to_vm; + /** @usm.next_asid: next ASID, used to cyclical alloc asids */ + u32 next_asid; + /** @usm.lock: protects UM state */ + struct rw_semaphore lock; + /** @usm.pf_wq: page fault work queue, unbound, high priority */ + struct workqueue_struct *pf_wq; + /* + * We pick 4 here because, in the current implementation, it + * yields the best bandwidth utilization of the kernel paging + * engine. + */ +#define XE_PAGEFAULT_QUEUE_COUNT 4 + /** @usm.pf_queue: Page fault queues */ + struct xe_pagefault_queue pf_queue[XE_PAGEFAULT_QUEUE_COUNT]; +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + /** @usm.dpagemap_shrinker: Shrinker for unused pagemaps */ + struct drm_pagemap_shrinker *dpagemap_shrinker; +#endif + } usm; + + /** @pinned: pinned BO state */ + struct { + /** @pinned.lock: protected pinned BO list state */ + spinlock_t lock; + /** @pinned.early: early pinned lists */ + struct { + /** @pinned.early.kernel_bo_present: pinned kernel BO that are present */ + struct list_head kernel_bo_present; + /** @pinned.early.evicted: pinned BO that have been evicted */ + struct list_head evicted; + } early; + /** @pinned.late: late pinned lists */ + struct { + /** @pinned.late.kernel_bo_present: pinned kernel BO that are present */ + struct list_head kernel_bo_present; + /** @pinned.late.evicted: pinned BO that have been evicted */ + struct list_head evicted; + /** @pinned.late.external: pinned external and dma-buf. */ + struct list_head external; + } late; + } pinned; + + /** @ufence_wq: user fence wait queue */ + wait_queue_head_t ufence_wq; + + /** @preempt_fence_wq: used to serialize preempt fences */ + struct workqueue_struct *preempt_fence_wq; + + /** @ordered_wq: used to serialize compute mode resume */ + struct workqueue_struct *ordered_wq; + + /** @unordered_wq: used to serialize unordered work */ + struct workqueue_struct *unordered_wq; + + /** @destroy_wq: used to serialize SVM pagemap destroy work */ + struct workqueue_struct *destroy_wq; + + /** @tiles: device tiles */ + struct xe_tile tiles[XE_MAX_TILES_PER_DEVICE]; + + /** + * @mem_access: keep track of memory access in the device, possibly + * triggering additional actions when they occur. + */ + struct { + /** + * @mem_access.vram_userfault: Encapsulate vram_userfault + * related stuff + */ + struct { + /** + * @mem_access.vram_userfault.lock: Protects access to + * @mem_access.vram_userfault.list Using mutex instead of spinlock + * as lock is applied to entire list operation which + * may sleep + */ + struct mutex lock; + + /** + * @mem_access.vram_userfault.list: Keep list of userfaulted + * vram bo, which require to release their mmap mappings + * at runtime suspend path + */ + struct list_head list; + } vram_userfault; + } mem_access; + + /** + * @pat: Encapsulate PAT related stuff + */ + struct { + /** @pat.ops: Internal operations to abstract platforms */ + const struct xe_pat_ops *ops; + /** @pat.table: PAT table to program in the HW */ + const struct xe_pat_table_entry *table; + /** @pat.n_entries: Number of PAT entries */ + int n_entries; + /** @pat.pat_ats: PAT entry for PCIe ATS responses */ + const struct xe_pat_table_entry *pat_ats; + /** @pat.pat_primary_pta: primary GT PAT entry for page table accesses */ + const struct xe_pat_table_entry *pat_primary_pta; + /** @pat.pat_media_pta: media GT PAT entry for page table accesses */ + const struct xe_pat_table_entry *pat_media_pta; + /** @pat.pat_primary_tr_pta: primary GT PAT entry for TRTT page table accesses */ + const struct xe_pat_table_entry *pat_primary_tr_pta; + /** @pat.pat_media_tr_pta: media GT PAT entry for TRTT page table accesses */ + const struct xe_pat_table_entry *pat_media_tr_pta; + u16 idx[__XE_CACHE_LEVEL_COUNT]; + } pat; + + /** @d3cold: Encapsulate d3cold related stuff */ + struct { + /** @d3cold.capable: Indicates if root port is d3cold capable */ + bool capable; + + /** @d3cold.allowed: Indicates if d3cold is a valid device state */ + bool allowed; + + /** + * @d3cold.vram_threshold: + * + * This represents the permissible threshold(in megabytes) + * for vram save/restore. d3cold will be disallowed, + * when vram_usages is above or equals the threshold value + * to avoid the vram save/restore latency. + * Default threshold value is 300mb. + */ + u32 vram_threshold; + /** @d3cold.lock: protect vram_threshold */ + struct mutex lock; + } d3cold; + + /** @pm_notifier: Our PM notifier to perform actions in response to various PM events. */ + struct notifier_block pm_notifier; + /** @pm_block: Completion to block validating tasks on suspend / hibernate prepare */ + struct completion pm_block; + /** @rebind_resume_list: List of wq items to kick on resume. */ + struct list_head rebind_resume_list; + /** @rebind_resume_lock: Lock to protect the rebind_resume_list */ + struct mutex rebind_resume_lock; + + /** @pmt: Support the PMT driver callback interface */ + struct { + /** @pmt.lock: protect access for telemetry data */ + struct mutex lock; + } pmt; + + /** @soc_remapper: SoC remapper object */ + struct { + /** @soc_remapper.lock: Serialize access to SoC Remapper's index registers */ + spinlock_t lock; + + /** @soc_remapper.set_telem_region: Set telemetry index */ + void (*set_telem_region)(struct xe_device *xe, u32 index); + + /** @soc_remapper.set_sysctrl_region: Set system controller index */ + void (*set_sysctrl_region)(struct xe_device *xe, u32 index); + } soc_remapper; + + /** + * @pm_callback_task: Track the active task that is running in either + * the runtime_suspend or runtime_resume callbacks. + */ + struct task_struct *pm_callback_task; + + /** @hwmon: hwmon subsystem integration */ + struct xe_hwmon *hwmon; + + /** @heci_gsc: graphics security controller */ + struct xe_heci_gsc heci_gsc; + + /** @nvm: discrete graphics non-volatile memory */ + struct intel_dg_nvm_dev *nvm; + + /** @late_bind: xe mei late bind interface */ + struct xe_late_bind late_bind; + +#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE + /** + * @page_size_alloc_ctrl: User BO page-size allocation + * debug control state + */ + struct { + /** @page_size_alloc_ctrl.mode: xe page size allocation control mode */ + enum xe_page_size_alloc_ctrl_mode mode; + /** @page_size_alloc_ctrl.cur_index: Round-robin index used by mixed mode */ + u32 cur_index; + /** @page_size_alloc_ctrl.lock: Protects @mode and @cur_index */ + struct mutex lock; + } page_size_alloc_ctrl; +#endif + /** @oa: oa observation subsystem */ + struct xe_oa oa; + + /** @pxp: Encapsulate Protected Xe Path support */ + struct xe_pxp *pxp; + + /** @needs_flr_on_fini: requests function-reset on fini */ + bool needs_flr_on_fini; + + /** @in_reset: Indicates if device is in reset */ + atomic_t in_reset; + + /** @wedged: Struct to control Wedged States and mode */ + struct { + /** @wedged.flag: Xe device faced a critical error and is now blocked. */ + atomic_t flag; + /** @wedged.mode: Mode controlled by kernel parameter and debugfs */ + enum xe_wedged_mode mode; + /** @wedged.method: Recovery method to be sent in the drm device wedged uevent */ + unsigned long method; + /** @wedged.inconsistent_reset: Inconsistent reset policy state between GTs */ + bool inconsistent_reset; + } wedged; + + /** @devres_group: devres group */ + void *devres_group; + + /** @bo_device: Struct to control async free of BOs */ + struct xe_bo_dev { + /** @bo_device.async_free: Free worker */ + struct work_struct async_free; + /** @bo_device.async_list: List of BOs to be freed */ + struct llist_head async_list; + } bo_device; + + /** @pmu: performance monitoring unit */ + struct xe_pmu pmu; + + /** @ras: RAS structure for device */ + struct xe_drm_ras ras; + + /** @i2c: I2C host controller */ + struct xe_i2c *i2c; + + /** @sc: System Controller */ + struct xe_sysctrl sc; + + /** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */ + u32 atomic_svm_timeslice_ms; + + /** @min_run_period_lr_ms: LR VM (preempt fence mode) timeslice */ + u32 min_run_period_lr_ms; + + /** @min_run_period_pf_ms: LR VM (page fault mode) timeslice */ + u32 min_run_period_pf_ms; + +#ifdef TEST_VM_OPS_ERROR + /** + * @vm_inject_error_position: inject errors at different places in VM + * bind IOCTL based on this value + */ + u8 vm_inject_error_position; +#endif + +#if IS_ENABLED(CONFIG_TRACE_GPU_MEM) + /** + * @global_total_pages: global GPU page usage tracked for gpu_mem + * tracepoints + */ + atomic64_t global_total_pages; +#endif + /** @val: The domain for exhaustive eviction, which is currently per device. */ + struct xe_validation_device val; + + /** @psmi: GPU debugging via additional validation HW */ + struct { + /** @psmi.capture_obj: PSMI buffer for VRAM */ + struct xe_bo *capture_obj[XE_MAX_TILES_PER_DEVICE + 1]; + /** @psmi.region_mask: Mask of valid memory regions */ + u8 region_mask; + } psmi; + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + /** @g2g_test_array: for testing G2G communications */ + u32 *g2g_test_array; + /** @g2g_test_count: for testing G2G communications */ + atomic_t g2g_test_count; +#endif + + /* private: */ + +#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) + /* + * Any fields below this point are the ones used by display. + * They are temporarily added here so xe_device can be desguised as + * drm_i915_private during build. After cleanup these should go away, + * migrating to the right sub-structs + */ + + struct intel_uncore { + spinlock_t lock; + } uncore; +#endif +}; + +/** + * struct xe_file - file handle for Xe driver + */ +struct xe_file { + /** @xe: xe DEVICE **/ + struct xe_device *xe; + + /** @drm: base DRM file */ + struct drm_file *drm; + + /** @vm: VM state for file */ + struct { + /** @vm.xa: xarray to store VMs */ + struct xarray xa; + /** + * @vm.lock: Protects VM lookup + reference and removal from + * file xarray. Not an intended to be an outer lock which does + * thing while being held. + */ + struct mutex lock; + } vm; + + /** @exec_queue: Submission exec queue state for file */ + struct { + /** @exec_queue.xa: xarray to store exece queues */ + struct xarray xa; + /** + * @exec_queue.lock: Protects exec queue lookup + reference and + * removal from file xarray. Not intended to be an outer lock + * which does things while being held. + */ + struct mutex lock; + /** + * @exec_queue.pending_removal: items pending to be removed to + * synchronize GPU state update with ongoing query. + */ + atomic_t pending_removal; + } exec_queue; + + /** @run_ticks: hw engine class run time in ticks for this drm client */ + u64 run_ticks[XE_ENGINE_CLASS_MAX]; + + /** @client: drm client */ + struct xe_drm_client *client; + + /** + * @process_name: process name for file handle, used to safely output + * during error situations where xe file can outlive process + */ + char *process_name; + + /** + * @pid: pid for file handle, used to safely output uring error + * situations where xe file can outlive process + */ + pid_t pid; + + /** @refcount: ref count of this xe file */ + struct kref refcount; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules new file mode 100644 index 000000000..d8dc41851 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules @@ -0,0 +1,9 @@ +22010954014 PLATFORM(DG2) +15015404425 PLATFORM(LUNARLAKE) + PLATFORM(PANTHERLAKE) +22019338487_display PLATFORM(LUNARLAKE) +14022085890 SUBPLATFORM(BATTLEMAGE, G21) +14026539277 PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0) +14026633728 PLATFORM(CRESCENTISLAND) +14026746987 PLATFORM(CRESCENTISLAND) +14026779378 PLATFORM(CRESCENTISLAND) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c new file mode 100644 index 000000000..bf0728838 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_dma_buf.h" + +#include +#include +#include + +#include +#include +#include + +#include "tests/xe_test.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_pm.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vm.h" + +MODULE_IMPORT_NS("DMA_BUF"); + +static int xe_dma_buf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + + if (attach->peer2peer && + pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0) + attach->peer2peer = false; + + if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT)) + return -EOPNOTSUPP; + + xe_pm_runtime_get(to_xe_device(obj->dev)); + return 0; +} + +static void xe_dma_buf_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + + xe_pm_runtime_put(to_xe_device(obj->dev)); +} + +static int xe_dma_buf_pin(struct dma_buf_attachment *attach) +{ + struct dma_buf *dmabuf = attach->dmabuf; + struct drm_gem_object *obj = dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + struct xe_device *xe = xe_bo_device(bo); + struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; + bool allow_vram = true; + int ret; + + list_for_each_entry(attach, &dmabuf->attachments, node) { + if (!attach->peer2peer) { + allow_vram = false; + break; + } + } + + if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT) && + !(xe_bo_is_vram(bo) && allow_vram)) { + drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n"); + return -EINVAL; + } + + if (!allow_vram) { + ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); + if (ret) { + if (ret != -EINTR && ret != -ERESTARTSYS) + drm_dbg(&xe->drm, + "Failed migrating dma-buf to TT memory: %pe\n", + ERR_PTR(ret)); + return ret; + } + } + + ret = xe_bo_pin_external(bo, !allow_vram, exec); + xe_assert(xe, !ret); + + return 0; +} + +static void xe_dma_buf_unpin(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + xe_bo_unpin_external(bo); +} + +static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, + enum dma_data_direction dir) +{ + struct dma_buf *dma_buf = attach->dmabuf; + struct drm_gem_object *obj = dma_buf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; + struct sg_table *sgt; + int r = 0; + + if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT)) + return ERR_PTR(-EOPNOTSUPP); + + if (!xe_bo_is_pinned(bo)) { + if (!attach->peer2peer) + r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); + else + r = xe_bo_validate(bo, NULL, false, exec); + if (r) + return ERR_PTR(r); + } + + switch (bo->ttm.resource->mem_type) { + case XE_PL_TT: + sgt = drm_prime_pages_to_sg(obj->dev, + bo->ttm.ttm->pages, + obj->size >> PAGE_SHIFT); + if (IS_ERR(sgt)) + return sgt; + + if (dma_map_sgtable(attach->dev, sgt, dir, + DMA_ATTR_SKIP_CPU_SYNC)) + goto error_free; + break; + + case XE_PL_VRAM0: + case XE_PL_VRAM1: + r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo), + bo->ttm.resource, 0, + bo->ttm.base.size, attach->dev, + dir, &sgt); + if (r) + return ERR_PTR(r); + break; + default: + return ERR_PTR(-EINVAL); + } + + return sgt; + +error_free: + sg_free_table(sgt); + kfree(sgt); + return ERR_PTR(-EBUSY); +} + +static void xe_dma_buf_unmap(struct dma_buf_attachment *attach, + struct sg_table *sgt, + enum dma_data_direction dir) +{ + if (sg_page(sgt->sgl)) { + dma_unmap_sgtable(attach->dev, sgt, dir, 0); + sg_free_table(sgt); + kfree(sgt); + } else { + xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt); + } +} + +static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, + enum dma_data_direction direction) +{ + struct drm_gem_object *obj = dma_buf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + bool reads = (direction == DMA_BIDIRECTIONAL || + direction == DMA_FROM_DEVICE); + struct xe_validation_ctx ctx; + struct drm_exec exec; + int ret = 0; + + if (!reads) + return 0; + + /* Can we do interruptible lock here? */ + xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) { + ret = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + if (ret) + break; + + ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &ret); + } + + /* If we failed, cpu-access takes place in current placement. */ + return 0; +} + +static void xe_dma_buf_release(struct dma_buf *dmabuf) +{ + struct drm_gem_object *obj = dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + xe_bo_lock(bo, false); + xe_bo_willneed_put_locked(bo); + xe_bo_unlock(bo); + + drm_gem_dmabuf_release(dmabuf); +} + +static const struct dma_buf_ops xe_dmabuf_ops = { + .attach = xe_dma_buf_attach, + .detach = xe_dma_buf_detach, + .pin = xe_dma_buf_pin, + .unpin = xe_dma_buf_unpin, + .map_dma_buf = xe_dma_buf_map, + .unmap_dma_buf = xe_dma_buf_unmap, + .release = xe_dma_buf_release, + .begin_cpu_access = xe_dma_buf_begin_cpu_access, + .mmap = drm_gem_dmabuf_mmap, + .vmap = drm_gem_dmabuf_vmap, + .vunmap = drm_gem_dmabuf_vunmap, +}; + +struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + struct dma_buf *buf; + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = true, + /* We opt to avoid OOM on system pages allocations */ + .gfp_retry_mayfail = true, + .allow_res_evict = false, + }; + int ret; + + if (bo->vm) + return ERR_PTR(-EPERM); + + /* + * Reject exporting purgeable BOs. DONTNEED BOs can be purged + * at any time, making the exported dma-buf unusable. Purged BOs + * have no backing store and are permanently invalid. + */ + ret = xe_bo_lock(bo, true); + if (ret) + return ERR_PTR(ret); + + if (xe_bo_madv_is_dontneed(bo)) { + ret = -EBUSY; + goto out_unlock; + } + + if (xe_bo_is_purged(bo)) { + ret = -EINVAL; + goto out_unlock; + } + + xe_bo_willneed_get_locked(bo); + xe_bo_unlock(bo); + + ret = ttm_bo_setup_export(&bo->ttm, &ctx); + if (ret) + goto out_put; + + buf = drm_gem_prime_export(obj, flags); + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto out_put; + } + + buf->ops = &xe_dmabuf_ops; + return buf; + +out_put: + xe_bo_lock(bo, false); + xe_bo_willneed_put_locked(bo); +out_unlock: + xe_bo_unlock(bo); + return ERR_PTR(ret); +} + +static struct drm_gem_object * +xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) +{ + struct dma_resv *resv = dma_buf->resv; + struct xe_device *xe = to_xe_device(dev); + struct xe_validation_ctx ctx; + struct drm_gem_object *dummy_obj; + struct drm_exec exec; + struct xe_bo *bo; + int ret = 0; + + dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm); + if (!dummy_obj) + return ERR_PTR(-ENOMEM); + + dummy_obj->resv = resv; + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) { + ret = drm_exec_lock_obj(&exec, dummy_obj); + drm_exec_retry_on_contention(&exec); + if (ret) + break; + + bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, + 0, /* Will require 1way or 2way for vm_bind */ + ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, dma_buf, &exec); + drm_exec_retry_on_contention(&exec); + if (IS_ERR(bo)) { + ret = PTR_ERR(bo); + xe_validation_retry_on_oom(&ctx, &ret); + break; + } + } + drm_gem_object_put(dummy_obj); + + return ret ? ERR_PTR(ret) : &bo->ttm.base; +} + +static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->importer_priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; + + XE_WARN_ON(xe_bo_evict(bo, exec)); +} + +static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = { + .allow_peer2peer = true, + .invalidate_mappings = xe_dma_buf_move_notify +}; + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + +struct dma_buf_test_params { + struct xe_test_priv base; + const struct dma_buf_attach_ops *attach_ops; + bool force_different_devices; + u32 mem_mask; +}; + +#define to_dma_buf_test_params(_priv) \ + container_of(_priv, struct dma_buf_test_params, base) +#endif + +struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf) +{ + XE_TEST_DECLARE(struct dma_buf_test_params *test = + to_dma_buf_test_params + (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));) + const struct dma_buf_attach_ops *attach_ops; + struct dma_buf_attachment *attach; + struct drm_gem_object *obj; + + if (dma_buf->ops == &xe_dmabuf_ops) { + obj = dma_buf->priv; + if (obj->dev == dev && + !XE_TEST_ONLY(test && test->force_different_devices)) { + /* + * Importing dmabuf exported from out own gem increases + * refcount on gem itself instead of f_count of dmabuf. + */ + drm_gem_object_get(obj); + return obj; + } + } + + /* + * This needs to happen before the attach, since it will create a new + * attachment for this, and add it to the list of attachments, at which + * point it is globally visible, and at any point the export side can + * call into on invalidate_mappings callback, which require a working + * object. + */ + obj = xe_dma_buf_create_obj(dev, dma_buf); + if (IS_ERR(obj)) + return obj; + + attach_ops = &xe_dma_buf_attach_ops; +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + if (test) + attach_ops = test->attach_ops; +#endif + + attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, obj); + if (IS_ERR(attach)) { + xe_bo_put(gem_to_xe_bo(obj)); + return ERR_CAST(attach); + } + + get_dma_buf(dma_buf); + obj->import_attach = attach; + return obj; +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_dma_buf.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_dma_buf.h b/drivers/gpu/drm/xe/xe_dma_buf.h new file mode 100644 index 000000000..861dd28a8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dma_buf.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_DMA_BUF_H_ +#define _XE_DMA_BUF_H_ + +#include + +struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags); +struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf); + +#endif diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c new file mode 100644 index 000000000..e116fb562 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include "xe_drm_client.h" + +#include +#include +#include +#include +#include + +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_bo_types.h" +#include "xe_device_types.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_hw_engine.h" +#include "xe_pm.h" +#include "xe_trace.h" + +/** + * DOC: DRM Client usage stats + * + * The drm/xe driver implements the DRM client usage stats specification as + * documented in :ref:`drm-client-usage-stats`. + * + * Example of the output showing the implemented key value pairs and entirety of + * the currently possible format options: + * + * :: + * + * pos: 0 + * flags: 0100002 + * mnt_id: 26 + * ino: 685 + * drm-driver: xe + * drm-client-id: 3 + * drm-pdev: 0000:03:00.0 + * drm-total-system: 0 + * drm-shared-system: 0 + * drm-active-system: 0 + * drm-resident-system: 0 + * drm-purgeable-system: 0 + * drm-total-gtt: 192 KiB + * drm-shared-gtt: 0 + * drm-active-gtt: 0 + * drm-resident-gtt: 192 KiB + * drm-total-vram0: 23992 KiB + * drm-shared-vram0: 16 MiB + * drm-active-vram0: 0 + * drm-resident-vram0: 23992 KiB + * drm-total-stolen: 0 + * drm-shared-stolen: 0 + * drm-active-stolen: 0 + * drm-resident-stolen: 0 + * drm-cycles-rcs: 28257900 + * drm-total-cycles-rcs: 7655183225 + * drm-cycles-bcs: 0 + * drm-total-cycles-bcs: 7655183225 + * drm-cycles-vcs: 0 + * drm-total-cycles-vcs: 7655183225 + * drm-engine-capacity-vcs: 2 + * drm-cycles-vecs: 0 + * drm-total-cycles-vecs: 7655183225 + * drm-engine-capacity-vecs: 2 + * drm-cycles-ccs: 0 + * drm-total-cycles-ccs: 7655183225 + * drm-engine-capacity-ccs: 4 + * + * Possible `drm-cycles-` key names are: `rcs`, `ccs`, `bcs`, `vcs`, `vecs` and + * "other". + */ + +/** + * xe_drm_client_alloc() - Allocate drm client + * @void: No arg + * + * Allocate drm client struct to track client memory against + * same till client life. Call this API whenever new client + * has opened xe device. + * + * Return: pointer to client struct or NULL if can't allocate + */ +struct xe_drm_client *xe_drm_client_alloc(void) +{ + struct xe_drm_client *client; + + client = kzalloc_obj(*client); + if (!client) + return NULL; + + kref_init(&client->kref); + +#ifdef CONFIG_PROC_FS + spin_lock_init(&client->bos_lock); + INIT_LIST_HEAD(&client->bos_list); +#endif + return client; +} + +/** + * __xe_drm_client_free() - Free client struct + * @kref: The reference + * + * This frees client struct. Call this API when xe device is closed + * by drm client. + * + * Return: void + */ +void __xe_drm_client_free(struct kref *kref) +{ + struct xe_drm_client *client = + container_of(kref, typeof(*client), kref); + + kfree(client); +} + +#ifdef CONFIG_PROC_FS +/** + * xe_drm_client_add_bo() - Add BO for tracking client mem usage + * @client: The drm client ptr + * @bo: The xe BO ptr + * + * Add all BO created by individual drm client by calling this function. + * This helps in tracking client memory usage. + * + * Return: void + */ +void xe_drm_client_add_bo(struct xe_drm_client *client, + struct xe_bo *bo) +{ + XE_WARN_ON(bo->client); + XE_WARN_ON(!list_empty(&bo->client_link)); + + bo->client = xe_drm_client_get(client); + spin_lock(&client->bos_lock); + list_add_tail(&bo->client_link, &client->bos_list); + spin_unlock(&client->bos_lock); +} + +/** + * xe_drm_client_remove_bo() - Remove BO for tracking client mem usage + * @bo: The xe BO ptr + * + * Remove all BO removed by individual drm client by calling this function. + * This helps in tracking client memory usage. + * + * Return: void + */ +void xe_drm_client_remove_bo(struct xe_bo *bo) +{ + struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); + struct xe_drm_client *client = bo->client; + + xe_assert(xe, !kref_read(&bo->ttm.base.refcount)); + + spin_lock(&client->bos_lock); + list_del_init(&bo->client_link); + spin_unlock(&client->bos_lock); + + xe_drm_client_put(client); +} + +static void bo_meminfo(struct xe_bo *bo, + struct drm_memory_stats stats[TTM_NUM_MEM_TYPES]) +{ + u64 sz = xe_bo_size(bo); + u32 mem_type; + + xe_bo_assert_held(bo); + + /* + * The resource can be NULL if the BO has been purged, plus maybe some + * other cases. Either way there shouldn't be any memory to account for, + * or a current resource to account this against, so skip for now. + */ + if (!bo->ttm.resource) + return; + + mem_type = bo->ttm.resource->mem_type; + + if (drm_gem_object_is_shared_for_memory_stats(&bo->ttm.base)) + stats[mem_type].shared += sz; + else + stats[mem_type].private += sz; + + if (xe_bo_has_pages(bo)) { + stats[mem_type].resident += sz; + + if (!dma_resv_test_signaled(bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + stats[mem_type].active += sz; + else if (mem_type == XE_PL_SYSTEM || xe_bo_madv_is_dontneed(bo)) + stats[mem_type].purgeable += sz; + } +} + +static void show_meminfo(struct drm_printer *p, struct drm_file *file) +{ + struct drm_memory_stats stats[TTM_NUM_MEM_TYPES] = {}; + struct xe_file *xef = file->driver_priv; + struct ttm_device *bdev = &xef->xe->ttm; + struct ttm_resource_manager *man; + struct xe_drm_client *client; + struct drm_gem_object *obj; + struct xe_bo *bo; + LLIST_HEAD(deferred); + unsigned int id; + u32 mem_type; + + client = xef->client; + + /* Public objects. */ + spin_lock(&file->table_lock); + idr_for_each_entry(&file->object_idr, obj, id) { + struct xe_bo *bo = gem_to_xe_bo(obj); + + if (dma_resv_trylock(bo->ttm.base.resv)) { + bo_meminfo(bo, stats); + xe_bo_unlock(bo); + } else { + xe_bo_get(bo); + spin_unlock(&file->table_lock); + + xe_bo_lock(bo, false); + bo_meminfo(bo, stats); + xe_bo_unlock(bo); + + xe_bo_put(bo); + spin_lock(&file->table_lock); + } + } + spin_unlock(&file->table_lock); + + /* Internal objects. */ + spin_lock(&client->bos_lock); + list_for_each_entry(bo, &client->bos_list, client_link) { + if (!kref_get_unless_zero(&bo->ttm.base.refcount)) + continue; + + if (dma_resv_trylock(bo->ttm.base.resv)) { + bo_meminfo(bo, stats); + xe_bo_unlock(bo); + } else { + spin_unlock(&client->bos_lock); + + xe_bo_lock(bo, false); + bo_meminfo(bo, stats); + xe_bo_unlock(bo); + + spin_lock(&client->bos_lock); + /* The bo ref will prevent this bo from being removed from the list */ + xe_assert(xef->xe, !list_empty(&bo->client_link)); + } + + xe_bo_put_deferred(bo, &deferred); + } + spin_unlock(&client->bos_lock); + + xe_bo_put_commit(&deferred); + + for (mem_type = XE_PL_SYSTEM; mem_type < TTM_NUM_MEM_TYPES; ++mem_type) { + if (!xe_mem_type_to_name[mem_type]) + continue; + + man = ttm_manager_type(bdev, mem_type); + + if (man) { + drm_print_memory_stats(p, + &stats[mem_type], + DRM_GEM_OBJECT_ACTIVE | + DRM_GEM_OBJECT_RESIDENT | + DRM_GEM_OBJECT_PURGEABLE, + xe_mem_type_to_name[mem_type]); + } + } +} + +static struct xe_hw_engine *any_engine(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned long gt_id; + + for_each_gt(gt, xe, gt_id) { + struct xe_hw_engine *hwe = xe_gt_any_hw_engine(gt); + + if (hwe) + return hwe; + } + + return NULL; +} + +/* + * Pick any engine and grab its forcewake. On error phwe will be NULL and + * the returned forcewake reference will be invalid. Callers should check + * phwe against NULL. + */ +static struct xe_force_wake_ref force_wake_get_any_engine(struct xe_device *xe, + struct xe_hw_engine **phwe) +{ + enum xe_force_wake_domains domain; + struct xe_force_wake_ref fw_ref = {}; + struct xe_hw_engine *hwe; + + *phwe = NULL; + + hwe = any_engine(xe); + if (!hwe) + return fw_ref; /* will be invalid */ + + domain = xe_hw_engine_to_fw_domain(hwe); + + fw_ref = xe_force_wake_constructor(gt_to_fw(hwe->gt), domain); + if (xe_force_wake_ref_has_domain(fw_ref.domains, domain)) + *phwe = hwe; /* valid forcewake */ + + return fw_ref; +} + +static void show_run_ticks(struct drm_printer *p, struct drm_file *file) +{ + unsigned long class, i, gt_id, capacity[XE_ENGINE_CLASS_MAX] = { }; + struct xe_file *xef = file->driver_priv; + struct xe_device *xe = xef->xe; + struct xe_gt *gt; + struct xe_hw_engine *hwe; + struct xe_exec_queue *q; + u64 gpu_timestamp; + + /* + * RING_TIMESTAMP registers are inaccessible in VF mode. + * Without drm-total-cycles-*, other keys provide little value. + * Show all or none of the optional "run_ticks" keys in this case. + */ + if (IS_SRIOV_VF(xe)) + return; + + /* + * Wait for any exec queue going away: their cycles will get updated on + * context switch out, so wait for that to happen + */ + wait_var_event(&xef->exec_queue.pending_removal, + !atomic_read(&xef->exec_queue.pending_removal)); + + scoped_guard(xe_pm_runtime, xe) { + CLASS(xe_force_wake_release_only, fw_ref)(force_wake_get_any_engine(xe, &hwe)); + if (!hwe) + return; + + /* Accumulate all the exec queues from this client */ + mutex_lock(&xef->exec_queue.lock); + xa_for_each(&xef->exec_queue.xa, i, q) { + xe_exec_queue_get(q); + mutex_unlock(&xef->exec_queue.lock); + + xe_exec_queue_update_run_ticks(q); + + mutex_lock(&xef->exec_queue.lock); + xe_exec_queue_put(q); + } + mutex_unlock(&xef->exec_queue.lock); + + gpu_timestamp = xe_hw_engine_read_timestamp(hwe); + } + + for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) { + const char *class_name; + + for_each_gt(gt, xe, gt_id) + capacity[class] += gt->user_engines.instances_per_class[class]; + + /* + * Engines may be fused off or not exposed to userspace. Don't + * return anything if this entire class is not available + */ + if (!capacity[class]) + continue; + + class_name = xe_hw_engine_class_to_str(class); + drm_printf(p, "drm-cycles-%s:\t%llu\n", + class_name, xef->run_ticks[class]); + drm_printf(p, "drm-total-cycles-%s:\t%llu\n", + class_name, gpu_timestamp); + + if (capacity[class] > 1) + drm_printf(p, "drm-engine-capacity-%s:\t%lu\n", + class_name, capacity[class]); + } +} + +/** + * xe_drm_client_fdinfo() - Callback for fdinfo interface + * @p: The drm_printer ptr + * @file: The drm_file ptr + * + * This is callback for drm fdinfo interface. Register this callback + * in drm driver ops for show_fdinfo. + * + * Return: void + */ +void xe_drm_client_fdinfo(struct drm_printer *p, struct drm_file *file) +{ + show_meminfo(p, file); + show_run_ticks(p, file); +} +#endif diff --git a/drivers/gpu/drm/xe/xe_drm_client.h b/drivers/gpu/drm/xe/xe_drm_client.h new file mode 100644 index 000000000..a9649aa36 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drm_client.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_DRM_CLIENT_H_ +#define _XE_DRM_CLIENT_H_ + +#include +#include +#include +#include +#include +#include + +struct drm_file; +struct drm_printer; +struct xe_bo; + +struct xe_drm_client { + struct kref kref; + unsigned int id; +#ifdef CONFIG_PROC_FS + /** + * @bos_lock: lock protecting @bos_list + */ + spinlock_t bos_lock; + /** + * @bos_list: list of bos created by this client + * + * Protected by @bos_lock. + */ + struct list_head bos_list; +#endif +}; + + static inline struct xe_drm_client * +xe_drm_client_get(struct xe_drm_client *client) +{ + kref_get(&client->kref); + return client; +} + +void __xe_drm_client_free(struct kref *kref); + +static inline void xe_drm_client_put(struct xe_drm_client *client) +{ + kref_put(&client->kref, __xe_drm_client_free); +} + +struct xe_drm_client *xe_drm_client_alloc(void); +static inline struct xe_drm_client * +xe_drm_client_get(struct xe_drm_client *client); +static inline void xe_drm_client_put(struct xe_drm_client *client); +#ifdef CONFIG_PROC_FS +void xe_drm_client_fdinfo(struct drm_printer *p, struct drm_file *file); +void xe_drm_client_add_bo(struct xe_drm_client *client, + struct xe_bo *bo); +void xe_drm_client_remove_bo(struct xe_bo *bo); +#else +static inline void xe_drm_client_add_bo(struct xe_drm_client *client, + struct xe_bo *bo) +{ +} + +static inline void xe_drm_client_remove_bo(struct xe_bo *bo) +{ +} +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c new file mode 100644 index 000000000..984fa67b9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drm_ras.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include + +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_drm_ras.h" +#include "xe_ras.h" + +static const char * const error_components[] = DRM_XE_RAS_ERROR_COMPONENT_NAMES; +static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES; + +static int query_error_counter(struct xe_device *xe, + enum drm_xe_ras_error_severity severity, + u32 error_id, const char **name, u32 *val) +{ + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[severity]; + + if (!info || !info[error_id].name) + return -ENOENT; + + *name = info[error_id].name; + + /* Fetch counter from system controller if supported */ + if (xe->info.has_sysctrl) + return xe_ras_get_counter(xe, severity, error_id, val); + + *val = atomic_read(&info[error_id].counter); + + return 0; +} + +static int clear_error_counter(struct xe_device *xe, + enum drm_xe_ras_error_severity severity, + u32 error_id) +{ + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[severity]; + + if (!info || !info[error_id].name) + return -ENOENT; + + /* Clear counter from system controller if supported */ + if (xe->info.has_sysctrl) + return xe_ras_clear_counter(xe, severity, error_id); + + atomic_set(&info[error_id].counter, 0); + + return 0; +} + +static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_id, + const char **name, u32 *val) +{ + struct xe_device *xe = ep->priv; + + return query_error_counter(xe, DRM_XE_RAS_ERR_SEV_UNCORRECTABLE, error_id, name, val); +} + +static int clear_uncorrectable_error_counter(struct drm_ras_node *node, u32 error_id) +{ + struct xe_device *xe = node->priv; + + return clear_error_counter(xe, DRM_XE_RAS_ERR_SEV_UNCORRECTABLE, error_id); +} + +static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id, + const char **name, u32 *val) +{ + struct xe_device *xe = ep->priv; + + return query_error_counter(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id, name, val); +} + +static int clear_correctable_error_counter(struct drm_ras_node *node, u32 error_id) +{ + struct xe_device *xe = node->priv; + + return clear_error_counter(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id); +} + +static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe) +{ + struct xe_drm_ras_counter *counter; + int i; + + counter = drmm_kcalloc(&xe->drm, DRM_XE_RAS_ERR_COMP_MAX, sizeof(*counter), GFP_KERNEL); + if (!counter) + return ERR_PTR(-ENOMEM); + + for (i = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; i < DRM_XE_RAS_ERR_COMP_MAX; i++) { + if (!error_components[i]) + continue; + + counter[i].name = error_components[i]; + atomic_set(&counter[i].counter, 0); + } + + return counter; +} + +static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node, + const enum drm_xe_ras_error_severity severity) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct xe_drm_ras *ras = &xe->ras; + const char *device_name; + + device_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", + pci_domain_nr(pdev->bus), pdev->bus->number, + PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); + + if (!device_name) + return -ENOMEM; + + node->device_name = device_name; + node->node_name = error_severity[severity]; + node->type = DRM_RAS_NODE_TYPE_ERROR_COUNTER; + node->error_counter_range.first = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; + node->error_counter_range.last = DRM_XE_RAS_ERR_COMP_MAX - 1; + node->priv = xe; + + ras->info[severity] = allocate_and_copy_counters(xe); + if (IS_ERR(ras->info[severity])) + return PTR_ERR(ras->info[severity]); + + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) { + node->query_error_counter = query_correctable_error_counter; + node->clear_error_counter = clear_correctable_error_counter; + } else { + node->query_error_counter = query_uncorrectable_error_counter; + node->clear_error_counter = clear_uncorrectable_error_counter; + } + + return 0; +} + +static void cleanup_node_param(struct drm_ras_node *node) +{ + kfree(node->device_name); + node->device_name = NULL; +} + +static void cleanup_node(struct drm_device *drm, void *node) +{ + drm_ras_node_unregister(node); + cleanup_node_param(node); +} + +static int register_nodes(struct xe_device *xe) +{ + struct xe_drm_ras *ras = &xe->ras; + struct drm_ras_node *node; + int i, ret; + + for_each_error_severity(i) { + node = &ras->node[i]; + + ret = assign_node_params(xe, node, i); + if (ret) + goto free_param; + + ret = drm_ras_node_register(node); + if (ret) + goto free_param; + + ret = drmm_add_action_or_reset(&xe->drm, cleanup_node, node); + if (ret) + goto null_info; + } + + return 0; + +free_param: + cleanup_node_param(node); +null_info: + ras->info[i] = NULL; + return ret; +} + +/** + * xe_drm_ras_init() - Initialize DRM RAS + * @xe: xe device instance + * + * Allocate and register DRM RAS nodes per device + * + * Return: 0 on success, negative error code otherwise. + */ +int xe_drm_ras_init(struct xe_device *xe) +{ + struct xe_drm_ras *ras = &xe->ras; + struct drm_ras_node *node; + int err; + + if (!xe->info.has_drm_ras) + return 0; + + node = drmm_kcalloc(&xe->drm, DRM_XE_RAS_ERR_SEV_MAX, sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + ras->node = node; + + err = register_nodes(xe); + if (err) { + drm_err(&xe->drm, "Failed to register DRM RAS nodes (%pe)\n", ERR_PTR(err)); + return err; + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_drm_ras.h b/drivers/gpu/drm/xe/xe_drm_ras.h new file mode 100644 index 000000000..365c70e93 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drm_ras.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ +#ifndef _XE_DRM_RAS_H_ +#define _XE_DRM_RAS_H_ + +struct xe_device; + +#define for_each_error_severity(i) \ + for (i = 0; i < DRM_XE_RAS_ERR_SEV_MAX; i++) + +int xe_drm_ras_init(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_drm_ras_types.h b/drivers/gpu/drm/xe/xe_drm_ras_types.h new file mode 100644 index 000000000..8d729ad6a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drm_ras_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_DRM_RAS_TYPES_H_ +#define _XE_DRM_RAS_TYPES_H_ + +#include +#include + +struct drm_ras_node; + +/* Error categories reported by hardware */ +enum hardware_error { + HARDWARE_ERROR_CORRECTABLE = 0, + HARDWARE_ERROR_NONFATAL, + HARDWARE_ERROR_FATAL, + HARDWARE_ERROR_MAX +}; + +/** + * struct xe_drm_ras_counter - XE RAS counter + * + * This structure contains error component and counter information + */ +struct xe_drm_ras_counter { + /** @name: error component name */ + const char *name; + + /** @counter: count of error */ + atomic_t counter; +}; + +/** + * struct xe_drm_ras - XE DRM RAS structure + * + * This structure has details of error counters + */ +struct xe_drm_ras { + /** @node: DRM RAS node */ + struct drm_ras_node *node; + + /** @info: info array for all types of errors */ + struct xe_drm_ras_counter *info[DRM_XE_RAS_ERR_SEV_MAX]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_drv.h b/drivers/gpu/drm/xe/xe_drv.h new file mode 100644 index 000000000..95242a375 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drv.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_DRV_H_ +#define _XE_DRV_H_ + +#include + +#define DRIVER_NAME "xe" +#define DRIVER_DESC "Intel Xe2 Graphics" + +/* Interface history: + * + * 1.1: Original. + */ +#define DRIVER_MAJOR 1 +#define DRIVER_MINOR 1 +#define DRIVER_PATCHLEVEL 0 + +#endif diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c new file mode 100644 index 000000000..8a7c1b5d5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -0,0 +1,1043 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_eu_stall.h" +#include "xe_force_wake.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_gt_topology.h" +#include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_observation.h" +#include "xe_pm.h" +#include "xe_trace.h" +#include "xe_wa.h" + +#include "regs/xe_eu_stall_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" + +#define POLL_PERIOD_MS 5 +#define FW_WA_WAIT_TIMEOUT_US 10000 + +#define SWF_EUSTALL_MASK REG_GENMASK(6, 5) +#define REQ_EUSTALL_ENABLE REG_BIT(5) +#define ACK_EUSTALL_ENABLE REG_GENMASK(6, 5) +#define REQ_EUSTALL_DISABLE REG_BIT(6) +#define ACK_EUSTALL_DISABLE 0 + +static size_t per_xecore_buf_size = SZ_512K; + +struct per_xecore_buf { + /* Buffer vaddr */ + u8 *vaddr; + /* Write pointer */ + u32 write; + /* Read pointer */ + u32 read; +}; + +struct xe_eu_stall_data_stream { + bool pollin; + bool enabled; + bool reset_detected; + int wait_num_reports; + int sampling_rate_mult; + wait_queue_head_t poll_wq; + size_t data_record_size; + size_t per_xecore_buf_size; + unsigned int fw_ref; + + struct xe_gt *gt; + struct xe_bo *bo; + /* Lock to protect data buffer pointers */ + struct mutex xecore_buf_lock; + struct per_xecore_buf *xecore_buf; + struct { + bool reported_to_user; + xe_dss_mask_t mask; + } data_drop; + struct delayed_work buf_poll_work; +}; + +struct xe_eu_stall_gt { + /* Lock to protect stream */ + struct mutex stream_lock; + /* EU stall data stream */ + struct xe_eu_stall_data_stream *stream; + /* Workqueue to schedule buffer pointers polling work */ + struct workqueue_struct *buf_ptr_poll_wq; +}; + +/** + * struct eu_stall_open_properties - EU stall sampling properties received + * from user space at open. + * @sampling_rate_mult: EU stall sampling rate multiplier. + * HW will sample every (sampling_rate_mult x 251) cycles. + * @wait_num_reports: Minimum number of EU stall data reports to unblock poll(). + * @gt: GT on which EU stall data will be captured. + */ +struct eu_stall_open_properties { + int sampling_rate_mult; + int wait_num_reports; + struct xe_gt *gt; +}; + +/* + * EU stall data format for PVC + */ +struct xe_eu_stall_data_pvc { + __u64 ip_addr:29; /* Bits 0 to 28 */ + __u64 active_count:8; /* Bits 29 to 36 */ + __u64 other_count:8; /* Bits 37 to 44 */ + __u64 control_count:8; /* Bits 45 to 52 */ + __u64 pipestall_count:8; /* Bits 53 to 60 */ + __u64 send_count:8; /* Bits 61 to 68 */ + __u64 dist_acc_count:8; /* Bits 69 to 76 */ + __u64 sbid_count:8; /* Bits 77 to 84 */ + __u64 sync_count:8; /* Bits 85 to 92 */ + __u64 inst_fetch_count:8; /* Bits 93 to 100 */ + __u64 unused_bits:27; + __u64 unused[6]; +} __packed; + +/* + * EU stall data format for Xe2 arch GPUs (LNL, BMG). + */ +struct xe_eu_stall_data_xe2 { + __u64 ip_addr:29; /* Bits 0 to 28 */ + __u64 tdr_count:8; /* Bits 29 to 36 */ + __u64 other_count:8; /* Bits 37 to 44 */ + __u64 control_count:8; /* Bits 45 to 52 */ + __u64 pipestall_count:8; /* Bits 53 to 60 */ + __u64 send_count:8; /* Bits 61 to 68 */ + __u64 dist_acc_count:8; /* Bits 69 to 76 */ + __u64 sbid_count:8; /* Bits 77 to 84 */ + __u64 sync_count:8; /* Bits 85 to 92 */ + __u64 inst_fetch_count:8; /* Bits 93 to 100 */ + __u64 active_count:8; /* Bits 101 to 108 */ + __u64 ex_id:3; /* Bits 109 to 111 */ + __u64 end_flag:1; /* Bit 112 */ + __u64 unused_bits:15; + __u64 unused[6]; +} __packed; + +/* + * EU stall data format for Xe3p arch GPUs. + */ +struct xe_eu_stall_data_xe3p { + __u64 ip_addr:61; /* Bits 0 to 60 */ + __u64 tdr_count:8; /* Bits 61 to 68 */ + __u64 other_count:8; /* Bits 69 to 76 */ + __u64 control_count:8; /* Bits 77 to 84 */ + __u64 pipestall_count:8; /* Bits 85 to 92 */ + __u64 send_count:8; /* Bits 93 to 100 */ + __u64 dist_acc_count:8; /* Bits 101 to 108 */ + __u64 sbid_count:8; /* Bits 109 to 116 */ + __u64 sync_count:8; /* Bits 117 to 124 */ + __u64 inst_fetch_count:8; /* Bits 125 to 132 */ + __u64 active_count:8; /* Bits 133 to 140 */ + __u64 ex_id:3; /* Bits 141 to 143 */ + __u64 end_flag:1; /* Bit 144 */ + __u64 unused_bits:47; + __u64 unused[5]; +} __packed; + +const u64 eu_stall_sampling_rates[] = {251, 251 * 2, 251 * 3, 251 * 4, 251 * 5, 251 * 6, 251 * 7}; + +/** + * xe_eu_stall_get_sampling_rates - get EU stall sampling rates information. + * + * @num_rates: Pointer to a u32 to return the number of sampling rates. + * @rates: double u64 pointer to point to an array of sampling rates. + * + * Stores the number of sampling rates and pointer to the array of + * sampling rates in the input pointers. + * + * Returns: Size of the EU stall sampling rates array. + */ +size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates) +{ + *num_rates = ARRAY_SIZE(eu_stall_sampling_rates); + *rates = eu_stall_sampling_rates; + + return sizeof(eu_stall_sampling_rates); +} + +/** + * xe_eu_stall_get_per_xecore_buf_size - get per XeCore buffer size. + * + * Returns: The per XeCore buffer size used to allocate the per GT + * EU stall data buffer. + */ +size_t xe_eu_stall_get_per_xecore_buf_size(void) +{ + return per_xecore_buf_size; +} + +/** + * xe_eu_stall_data_record_size - get EU stall data record size. + * + * @xe: Pointer to a Xe device. + * + * Returns: EU stall data record size. + */ +size_t xe_eu_stall_data_record_size(struct xe_device *xe) +{ + size_t record_size = 0; + + if (GRAPHICS_VER(xe) >= 35) + record_size = sizeof(struct xe_eu_stall_data_xe3p); + else if (GRAPHICS_VER(xe) >= 20) + record_size = sizeof(struct xe_eu_stall_data_xe2); + else if (xe->info.platform == XE_PVC) + record_size = sizeof(struct xe_eu_stall_data_pvc); + + + xe_assert(xe, is_power_of_2(record_size)); + + return record_size; +} + +/** + * num_data_rows - Return the number of EU stall data rows of 64B each + * for a given data size. + * + * @data_size: EU stall data size + */ +static u32 num_data_rows(u32 data_size) +{ + return data_size >> 6; +} + +static void xe_eu_stall_fini(void *arg) +{ + struct xe_gt *gt = arg; + + destroy_workqueue(gt->eu_stall->buf_ptr_poll_wq); + mutex_destroy(>->eu_stall->stream_lock); + kfree(gt->eu_stall); +} + +/** + * xe_eu_stall_init() - Allocate and initialize GT level EU stall data + * structure xe_eu_stall_gt within struct xe_gt. + * + * @gt: GT being initialized. + * + * Returns: zero on success or a negative error code. + */ +int xe_eu_stall_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int ret; + + if (!xe_eu_stall_supported_on_platform(xe)) + return 0; + + gt->eu_stall = kzalloc_obj(*gt->eu_stall); + if (!gt->eu_stall) { + ret = -ENOMEM; + goto exit; + } + + mutex_init(>->eu_stall->stream_lock); + + gt->eu_stall->buf_ptr_poll_wq = alloc_ordered_workqueue("xe_eu_stall", 0); + if (!gt->eu_stall->buf_ptr_poll_wq) { + ret = -ENOMEM; + goto exit_free; + } + + return devm_add_action_or_reset(xe->drm.dev, xe_eu_stall_fini, gt); +exit_free: + mutex_destroy(>->eu_stall->stream_lock); + kfree(gt->eu_stall); +exit: + return ret; +} + +static int set_prop_eu_stall_sampling_rate(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + value = div_u64(value, 251); + if (value == 0 || value > 7) { + drm_dbg(&xe->drm, "Invalid EU stall sampling rate %llu\n", value); + return -EINVAL; + } + props->sampling_rate_mult = value; + return 0; +} + +static int set_prop_eu_stall_wait_num_reports(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + props->wait_num_reports = value; + + return 0; +} + +static int set_prop_eu_stall_gt_id(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + struct xe_gt *gt = xe_device_get_gt(xe, value); + + if (!gt) { + drm_dbg(&xe->drm, "Invalid GT ID %llu for EU stall sampling\n", value); + return -EINVAL; + } + props->gt = gt; + return 0; +} + +typedef int (*set_eu_stall_property_fn)(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props); + +static const set_eu_stall_property_fn xe_set_eu_stall_property_funcs[] = { + [DRM_XE_EU_STALL_PROP_SAMPLE_RATE] = set_prop_eu_stall_sampling_rate, + [DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS] = set_prop_eu_stall_wait_num_reports, + [DRM_XE_EU_STALL_PROP_GT_ID] = set_prop_eu_stall_gt_id, +}; + +static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension, + struct eu_stall_open_properties *props) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) || + XE_IOCTL_DBG(xe, !ext.property) || XE_IOCTL_DBG(xe, ext.pad)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs)); + return xe_set_eu_stall_property_funcs[idx](xe, ext.value, props); +} + +typedef int (*xe_eu_stall_user_extension_fn)(struct xe_device *xe, u64 extension, + struct eu_stall_open_properties *props); +static const xe_eu_stall_user_extension_fn xe_eu_stall_user_extension_funcs[] = { + [DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY] = xe_eu_stall_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 5 +static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension, + int ext_number, struct eu_stall_open_properties *props) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(xe_eu_stall_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_eu_stall_user_extension_funcs)); + err = xe_eu_stall_user_extension_funcs[idx](xe, extension, props); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (ext.next_extension) + return xe_eu_stall_user_extensions(xe, ext.next_extension, ++ext_number, props); + + return 0; +} + +/** + * buf_data_size - Calculate the number of bytes in a circular buffer + * given the read and write pointers and the size of + * the buffer. + * + * @buf_size: Size of the circular buffer + * @read_ptr: Read pointer with an additional overflow bit + * @write_ptr: Write pointer with an additional overflow bit + * + * Since the read and write pointers have an additional overflow bit, + * this function calculates the offsets from the pointers and use the + * offsets to calculate the data size in the buffer. + * + * Returns: number of bytes of data in the buffer + */ +static u32 buf_data_size(size_t buf_size, u32 read_ptr, u32 write_ptr) +{ + u32 read_offset, write_offset, size = 0; + + if (read_ptr == write_ptr) + goto exit; + + read_offset = read_ptr & (buf_size - 1); + write_offset = write_ptr & (buf_size - 1); + + if (write_offset > read_offset) + size = write_offset - read_offset; + else + size = buf_size - read_offset + write_offset; +exit: + return size; +} + +/** + * eu_stall_data_buf_poll - Poll for EU stall data in the buffer. + * + * @stream: xe EU stall data stream instance + * + * Returns: true if the EU stall buffer contains minimum stall data as + * specified by the event report count, else false. + */ +static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) +{ + u32 read_ptr, write_ptr_reg, write_ptr, total_data = 0; + u32 buf_size = stream->per_xecore_buf_size; + struct per_xecore_buf *xecore_buf; + struct xe_gt *gt = stream->gt; + bool min_data_present = false; + u16 group, instance; + unsigned int xecore; + + mutex_lock(&stream->xecore_buf_lock); + for_each_dss_steering(xecore, gt, group, instance) { + xecore_buf = &stream->xecore_buf[xecore]; + read_ptr = xecore_buf->read; + write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, + group, instance); + write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg); + write_ptr <<= 6; + write_ptr &= ((buf_size << 1) - 1); + if (!min_data_present) { + total_data += buf_data_size(buf_size, read_ptr, write_ptr); + if (num_data_rows(total_data) >= stream->wait_num_reports) + min_data_present = true; + } + if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP) + set_bit(xecore, stream->data_drop.mask); + xecore_buf->write = write_ptr; + } + /* If a GT or engine reset happens during EU stall sampling, + * all EU stall registers get reset to 0 and the cached values of + * the EU stall data buffers' read pointers are out of sync with + * the register values. This causes invalid data to be returned + * from read(). To prevent this, check the value of a EU stall base + * register. If it is zero, there has been a reset. + */ + if (unlikely(!xe_gt_mcr_unicast_read_any(gt, XEHPC_EUSTALL_BASE))) + stream->reset_detected = true; + + stream->pollin = min_data_present || stream->reset_detected; + mutex_unlock(&stream->xecore_buf_lock); + + return stream->pollin; +} + +static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 write_ptr_reg; + + /* On PVC, the overflow bit has to be cleared by writing 1 to it. + * On Xe2 and later GPUs, the bit has to be cleared by writing 0 to it. + */ + if (GRAPHICS_VER(xe) >= 20) + write_ptr_reg = REG_MASKED_FIELD_DISABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP); + else + write_ptr_reg = REG_MASKED_FIELD_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP); + + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT, write_ptr_reg, group, instance); +} + +static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream, + char __user *buf, size_t count, + size_t *total_data_size, struct xe_gt *gt, + u16 group, u16 instance, unsigned int xecore) +{ + size_t read_data_size, copy_size, buf_size; + u32 read_ptr_reg, read_ptr, write_ptr; + u8 *xecore_start_vaddr, *read_vaddr; + struct per_xecore_buf *xecore_buf; + u32 read_offset, write_offset; + + /* Hardware increments the read and write pointers such that they can + * overflow into one additional bit. For example, a 256KB size buffer + * offset pointer needs 18 bits. But HW uses 19 bits for the read and + * write pointers. This technique avoids wasting a slot in the buffer. + * Read and write offsets are calculated from the pointers in order to + * check if the write pointer has wrapped around the array. + */ + xecore_buf = &stream->xecore_buf[xecore]; + xecore_start_vaddr = xecore_buf->vaddr; + read_ptr = xecore_buf->read; + write_ptr = xecore_buf->write; + buf_size = stream->per_xecore_buf_size; + + read_data_size = buf_data_size(buf_size, read_ptr, write_ptr); + /* Read only the data that the user space buffer can accommodate */ + read_data_size = min_t(size_t, count - *total_data_size, read_data_size); + if (read_data_size == 0) + goto exit_drop; + + read_offset = read_ptr & (buf_size - 1); + write_offset = write_ptr & (buf_size - 1); + read_vaddr = xecore_start_vaddr + read_offset; + + if (write_offset > read_offset) { + if (copy_to_user(buf + *total_data_size, read_vaddr, read_data_size)) + return -EFAULT; + } else { + if (read_data_size >= buf_size - read_offset) + copy_size = buf_size - read_offset; + else + copy_size = read_data_size; + if (copy_to_user(buf + *total_data_size, read_vaddr, copy_size)) + return -EFAULT; + if (copy_to_user(buf + *total_data_size + copy_size, + xecore_start_vaddr, read_data_size - copy_size)) + return -EFAULT; + } + + *total_data_size += read_data_size; + read_ptr += read_data_size; + + /* Read pointer can overflow into one additional bit */ + read_ptr &= (buf_size << 1) - 1; + read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, (read_ptr >> 6)); + read_ptr_reg = REG_MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg); + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, group, instance); + xecore_buf->read = read_ptr; + trace_xe_eu_stall_data_read(group, instance, read_ptr, write_ptr, + read_data_size, *total_data_size); +exit_drop: + /* Clear drop bit (if set) after any data was read or if the buffer was empty. + * Drop bit can be set even if the buffer is empty as the buffer may have been emptied + * in the previous read() and the data drop bit was set during the previous read(). + */ + if (test_bit(xecore, stream->data_drop.mask)) { + clear_dropped_eviction_line_bit(gt, group, instance); + clear_bit(xecore, stream->data_drop.mask); + } + return 0; +} + +/** + * xe_eu_stall_stream_read_locked - copy EU stall counters data from the + * per xecore buffers to the userspace buffer + * @stream: A stream opened for EU stall count metrics + * @file: An xe EU stall data stream file + * @buf: destination buffer given by userspace + * @count: the number of bytes userspace wants to read + * + * Returns: Number of bytes copied or a negative error code + * If we've successfully copied any data then reporting that takes + * precedence over any internal error status, so the data isn't lost. + */ +static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *stream, + struct file *file, char __user *buf, + size_t count) +{ + struct xe_gt *gt = stream->gt; + size_t total_size = 0; + u16 group, instance; + unsigned int xecore; + int ret = 0; + + mutex_lock(&stream->xecore_buf_lock); + /* If EU stall registers got reset due to a GT/engine reset, + * continuing with the read() will return invalid data to + * the user space. Just return -ENODEV instead. + */ + if (unlikely(stream->reset_detected)) { + xe_gt_dbg(gt, "EU stall base register has been reset\n"); + mutex_unlock(&stream->xecore_buf_lock); + return -ENODEV; + } + if (bitmap_weight(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) { + if (!stream->data_drop.reported_to_user) { + stream->data_drop.reported_to_user = true; + xe_gt_dbg(gt, "EU stall data dropped in XeCores: %*pb\n", + XE_MAX_DSS_FUSE_BITS, stream->data_drop.mask); + mutex_unlock(&stream->xecore_buf_lock); + return -EIO; + } + stream->data_drop.reported_to_user = false; + } + for_each_dss_steering(xecore, gt, group, instance) { + ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size, + gt, group, instance, xecore); + if (ret || count == total_size) + break; + } + mutex_unlock(&stream->xecore_buf_lock); + return total_size ?: (ret ?: -EAGAIN); +} + +/* + * Userspace must enable the EU stall stream with DRM_XE_OBSERVATION_IOCTL_ENABLE + * before calling read(). + * + * Returns: The number of bytes copied or a negative error code on failure. + * -EIO if HW drops any EU stall data when the buffer is full. + */ +static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + ssize_t ret, aligned_count; + + aligned_count = ALIGN_DOWN(count, stream->data_record_size); + if (aligned_count == 0) + return -EINVAL; + + if (!stream->enabled) { + xe_gt_dbg(gt, "EU stall data stream not enabled to read\n"); + return -EINVAL; + } + + if (!(file->f_flags & O_NONBLOCK)) { + do { + ret = wait_event_interruptible(stream->poll_wq, stream->pollin); + if (ret) + return -EINTR; + + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_read_locked(stream, file, buf, aligned_count); + mutex_unlock(>->eu_stall->stream_lock); + } while (ret == -EAGAIN); + } else { + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_read_locked(stream, file, buf, aligned_count); + mutex_unlock(>->eu_stall->stream_lock); + } + + /* + * This may not work correctly if the user buffer is very small. + * We don't want to block the next read() when there is data in the buffer + * now, but couldn't be accommodated in the small user buffer. + */ + if (!stream->reset_detected) + stream->pollin = false; + + return ret; +} + +static void xe_eu_stall_stream_free(struct xe_eu_stall_data_stream *stream) +{ + struct xe_gt *gt = stream->gt; + + mutex_destroy(&stream->xecore_buf_lock); + gt->eu_stall->stream = NULL; + kfree(stream); +} + +static void xe_eu_stall_data_buf_destroy(struct xe_eu_stall_data_stream *stream) +{ + xe_bo_unpin_map_no_vm(stream->bo); + kfree(stream->xecore_buf); +} + +static int xe_eu_stall_data_buf_alloc(struct xe_eu_stall_data_stream *stream, + u16 last_xecore) +{ + struct xe_tile *tile = stream->gt->tile; + struct xe_bo *bo; + u32 size; + + stream->xecore_buf = kzalloc_objs(*stream->xecore_buf, last_xecore); + if (!stream->xecore_buf) + return -ENOMEM; + + size = stream->per_xecore_buf_size * last_xecore; + + bo = xe_bo_create_pin_map_at_novm(tile->xe, tile, size, ~0ull, ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, SZ_64, false); + if (IS_ERR(bo)) { + kfree(stream->xecore_buf); + return PTR_ERR(bo); + } + + XE_WARN_ON(!IS_ALIGNED(xe_bo_ggtt_addr(bo), SZ_64)); + stream->bo = bo; + + return 0; +} + +static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) +{ + u32 write_ptr_reg, write_ptr, read_ptr_reg, reg_value; + struct per_xecore_buf *xecore_buf; + struct xe_gt *gt = stream->gt; + u16 group, instance; + int xecore, ret = 0; + + /* Take runtime pm ref and forcewake to disable RC6 */ + xe_pm_runtime_get(gt_to_xe(gt)); + stream->fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_RENDER); + if (!xe_force_wake_ref_has_domain(stream->fw_ref, XE_FW_RENDER)) { + xe_gt_err(gt, "Failed to get RENDER forcewake\n"); + xe_pm_runtime_put(gt_to_xe(gt)); + return -ETIMEDOUT; + } + + if (XE_GT_WA(gt, 14027054324)) { + /* Request the firmware to apply the workaround and wait for an ACK */ + xe_mmio_write32(>->mmio, SWF_SCRATCHPAD(0), REQ_EUSTALL_ENABLE); + ret = xe_mmio_wait32(>->mmio, SWF_SCRATCHPAD(0), SWF_EUSTALL_MASK, + ACK_EUSTALL_ENABLE, FW_WA_WAIT_TIMEOUT_US, NULL, false); + if (ret) { + xe_gt_err(gt, "Timeout polling for EU stall enable ACK from firmware\n"); + xe_force_wake_put(gt_to_fw(gt), stream->fw_ref); + xe_pm_runtime_put(gt_to_xe(gt)); + return ret; + } + } + if (XE_GT_WA(gt, 22016596838)) + xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2, + REG_MASKED_FIELD_ENABLE(DISABLE_DOP_GATING)); + + for_each_dss_steering(xecore, gt, group, instance) { + write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, group, instance); + /* Clear any drop bits set and not cleared in the previous session. */ + if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP) + clear_dropped_eviction_line_bit(gt, group, instance); + write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg); + read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, write_ptr); + read_ptr_reg = REG_MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg); + /* Initialize the read pointer to the write pointer */ + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, group, instance); + write_ptr <<= 6; + write_ptr &= (stream->per_xecore_buf_size << 1) - 1; + xecore_buf = &stream->xecore_buf[xecore]; + xecore_buf->write = write_ptr; + xecore_buf->read = write_ptr; + } + stream->reset_detected = false; + stream->data_drop.reported_to_user = false; + bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS); + + reg_value = REG_MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE, + REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) | + REG_FIELD_PREP(EUSTALL_SAMPLE_RATE, + stream->sampling_rate_mult)); + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_CTRL, reg_value); + /* GGTT addresses can never be > 32 bits */ + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE_UPPER, 0); + reg_value = xe_bo_ggtt_addr(stream->bo); + reg_value |= REG_FIELD_PREP(XEHPC_EUSTALL_BASE_XECORE_BUF_SZ, + stream->per_xecore_buf_size / SZ_256K); + reg_value |= XEHPC_EUSTALL_BASE_ENABLE_SAMPLING; + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, reg_value); + + return ret; +} + +static void eu_stall_data_buf_poll_work_fn(struct work_struct *work) +{ + struct xe_eu_stall_data_stream *stream = + container_of(work, typeof(*stream), buf_poll_work.work); + struct xe_gt *gt = stream->gt; + + if (eu_stall_data_buf_poll(stream)) + wake_up(&stream->poll_wq); + + if (!stream->reset_detected) + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); +} + +static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, + struct eu_stall_open_properties *props) +{ + unsigned int max_wait_num_reports, xecore, last_xecore, num_xecores; + struct per_xecore_buf *xecore_buf; + struct xe_gt *gt = stream->gt; + xe_dss_mask_t all_xecores; + u16 group, instance; + u32 vaddr_offset; + int ret; + + bitmap_or(all_xecores, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS); + num_xecores = bitmap_weight(all_xecores, XE_MAX_DSS_FUSE_BITS); + last_xecore = xe_gt_topology_mask_last_dss(all_xecores) + 1; + + max_wait_num_reports = num_data_rows(per_xecore_buf_size * num_xecores); + if (props->wait_num_reports == 0 || props->wait_num_reports > max_wait_num_reports) { + xe_gt_dbg(gt, "Invalid EU stall event report count %u\n", + props->wait_num_reports); + xe_gt_dbg(gt, "Minimum event report count is 1, maximum is %u\n", + max_wait_num_reports); + return -EINVAL; + } + + init_waitqueue_head(&stream->poll_wq); + mutex_init(&stream->xecore_buf_lock); + INIT_DELAYED_WORK(&stream->buf_poll_work, eu_stall_data_buf_poll_work_fn); + stream->per_xecore_buf_size = per_xecore_buf_size; + stream->sampling_rate_mult = props->sampling_rate_mult; + stream->wait_num_reports = props->wait_num_reports; + stream->data_record_size = xe_eu_stall_data_record_size(gt_to_xe(gt)); + + ret = xe_eu_stall_data_buf_alloc(stream, last_xecore); + if (ret) + return ret; + + for_each_dss_steering(xecore, gt, group, instance) { + xecore_buf = &stream->xecore_buf[xecore]; + vaddr_offset = xecore * stream->per_xecore_buf_size; + xecore_buf->vaddr = stream->bo->vmap.vaddr + vaddr_offset; + } + return 0; +} + +static __poll_t xe_eu_stall_stream_poll_locked(struct xe_eu_stall_data_stream *stream, + struct file *file, poll_table *wait) +{ + __poll_t events = 0; + + poll_wait(file, &stream->poll_wq, wait); + + if (stream->pollin) + events |= EPOLLIN; + + return events; +} + +static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait) +{ + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + __poll_t ret; + + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_poll_locked(stream, file, wait); + mutex_unlock(>->eu_stall->stream_lock); + + return ret; +} + +static int xe_eu_stall_enable_locked(struct xe_eu_stall_data_stream *stream) +{ + struct xe_gt *gt = stream->gt; + int ret = 0; + + if (stream->enabled) + return ret; + + stream->enabled = true; + + ret = xe_eu_stall_stream_enable(stream); + + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); + return ret; +} + +static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream) +{ + struct xe_gt *gt = stream->gt; + int ret = 0; + + if (!stream->enabled) + return 0; + + stream->enabled = false; + + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, 0); + + cancel_delayed_work_sync(&stream->buf_poll_work); + + if (XE_GT_WA(gt, 22016596838)) + xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2, + REG_MASKED_FIELD_DISABLE(DISABLE_DOP_GATING)); + if (XE_GT_WA(gt, 14027054324)) { + /* Request the firmware to revert the workaround and wait for an ACK */ + xe_mmio_write32(>->mmio, SWF_SCRATCHPAD(0), REQ_EUSTALL_DISABLE); + ret = xe_mmio_wait32(>->mmio, SWF_SCRATCHPAD(0), SWF_EUSTALL_MASK, + ACK_EUSTALL_DISABLE, FW_WA_WAIT_TIMEOUT_US, NULL, false); + if (ret) + xe_gt_err(gt, "Timeout polling for EU stall disable ACK from firmware\n"); + } + + xe_force_wake_put(gt_to_fw(gt), stream->fw_ref); + xe_pm_runtime_put(gt_to_xe(gt)); + + return ret; +} + +static long xe_eu_stall_stream_ioctl_locked(struct xe_eu_stall_data_stream *stream, + unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case DRM_XE_OBSERVATION_IOCTL_ENABLE: + return xe_eu_stall_enable_locked(stream); + case DRM_XE_OBSERVATION_IOCTL_DISABLE: + return xe_eu_stall_disable_locked(stream); + } + + return -EINVAL; +} + +static long xe_eu_stall_stream_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + long ret; + + mutex_lock(>->eu_stall->stream_lock); + ret = xe_eu_stall_stream_ioctl_locked(stream, cmd, arg); + mutex_unlock(>->eu_stall->stream_lock); + + return ret; +} + +static int xe_eu_stall_stream_close(struct inode *inode, struct file *file) +{ + struct xe_eu_stall_data_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + + mutex_lock(>->eu_stall->stream_lock); + xe_eu_stall_disable_locked(stream); + xe_eu_stall_data_buf_destroy(stream); + xe_eu_stall_stream_free(stream); + mutex_unlock(>->eu_stall->stream_lock); + + drm_dev_put(>->tile->xe->drm); + + return 0; +} + +static const struct file_operations fops_eu_stall = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .release = xe_eu_stall_stream_close, + .poll = xe_eu_stall_stream_poll, + .read = xe_eu_stall_stream_read, + .unlocked_ioctl = xe_eu_stall_stream_ioctl, + .compat_ioctl = xe_eu_stall_stream_ioctl, +}; + +static int xe_eu_stall_stream_open_locked(struct drm_device *dev, + struct eu_stall_open_properties *props, + struct drm_file *file) +{ + struct xe_eu_stall_data_stream *stream; + struct xe_gt *gt = props->gt; + unsigned long f_flags = 0; + int ret, stream_fd; + + /* Only one session can be active at any time */ + if (gt->eu_stall->stream) { + xe_gt_dbg(gt, "EU stall sampling session already active\n"); + return -EBUSY; + } + + stream = kzalloc_obj(*stream); + if (!stream) + return -ENOMEM; + + gt->eu_stall->stream = stream; + stream->gt = gt; + + ret = xe_eu_stall_stream_init(stream, props); + if (ret) { + xe_gt_dbg(gt, "EU stall stream init failed : %d\n", ret); + goto err_free; + } + + stream_fd = anon_inode_getfd("[xe_eu_stall]", &fops_eu_stall, stream, f_flags); + if (stream_fd < 0) { + ret = stream_fd; + xe_gt_dbg(gt, "EU stall inode get fd failed : %d\n", ret); + goto err_destroy; + } + + /* Take a reference on the driver that will be kept with stream_fd + * until its release. + */ + drm_dev_get(>->tile->xe->drm); + + return stream_fd; + +err_destroy: + xe_eu_stall_data_buf_destroy(stream); +err_free: + xe_eu_stall_stream_free(stream); + return ret; +} + +/** + * xe_eu_stall_stream_open - Open a xe EU stall data stream fd + * + * @dev: DRM device pointer + * @data: pointer to first struct @drm_xe_ext_set_property in + * the chain of input properties from the user space. + * @file: DRM file pointer + * + * This function opens a EU stall data stream with input properties from + * the user space. + * + * Returns: EU stall data stream fd on success or a negative error code. + */ +int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct eu_stall_open_properties props = {}; + int ret; + + if (!xe_eu_stall_supported_on_platform(xe)) { + drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); + return -ENODEV; + } + + ret = xe_observation_paranoid_check(); + if (ret) { + drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n"); + return ret; + } + + /* Initialize and set default values */ + props.wait_num_reports = 1; + props.sampling_rate_mult = 4; + + ret = xe_eu_stall_user_extensions(xe, data, 0, &props); + if (ret) + return ret; + + if (!props.gt) { + drm_dbg(&xe->drm, "GT ID not provided for EU stall sampling\n"); + return -EINVAL; + } + + mutex_lock(&props.gt->eu_stall->stream_lock); + ret = xe_eu_stall_stream_open_locked(dev, &props, file); + mutex_unlock(&props.gt->eu_stall->stream_lock); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h new file mode 100644 index 000000000..842bef9f6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_EU_STALL_H_ +#define _XE_EU_STALL_H_ + +#include "xe_gt_types.h" +#include "xe_sriov.h" + +size_t xe_eu_stall_get_per_xecore_buf_size(void); +size_t xe_eu_stall_data_record_size(struct xe_device *xe); +size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates); + +int xe_eu_stall_init(struct xe_gt *gt); +int xe_eu_stall_stream_open(struct drm_device *dev, + u64 data, + struct drm_file *file); + +static inline bool xe_eu_stall_supported_on_platform(struct xe_device *xe) +{ + return !IS_SRIOV_VF(xe) && (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20); +} +#endif diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c new file mode 100644 index 000000000..d5293bc33 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_exec.h" + +#include +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_hw_engine_group.h" +#include "xe_macros.h" +#include "xe_pm.h" +#include "xe_ring_ops_types.h" +#include "xe_sched_job.h" +#include "xe_sync.h" +#include "xe_svm.h" +#include "xe_trace.h" +#include "xe_vm.h" + +/** + * DOC: Execbuf (User GPU command submission) + * + * Execs have historically been rather complicated in DRM drivers (at least in + * the i915) because a few things: + * + * - Passing in a list BO which are read / written to creating implicit syncs + * - Binding at exec time + * - Flow controlling the ring at exec time + * + * In Xe we avoid all of this complication by not allowing a BO list to be + * passed into an exec, using the dma-buf implicit sync uAPI, have binds as + * separate operations, and using the DRM scheduler to flow control the ring. + * Let's deep dive on each of these. + * + * We can get away from a BO list by forcing the user to use in / out fences on + * every exec rather than the kernel tracking dependencies of BO (e.g. if the + * user knows an exec writes to a BO and reads from the BO in the next exec, it + * is the user's responsibility to pass in / out fence between the two execs). + * + * We do not allow a user to trigger a bind at exec time rather we have a VM + * bind IOCTL which uses the same in / out fence interface as exec. In that + * sense, a VM bind is basically the same operation as an exec from the user + * perspective. e.g. If an exec depends on a VM bind use the in / out fence + * interface (struct drm_xe_sync) to synchronize like syncing between two + * dependent execs. + * + * Although a user cannot trigger a bind, we still have to rebind userptrs in + * the VM that have been invalidated since the last exec, likewise we also have + * to rebind BOs that have been evicted by the kernel. We schedule these rebinds + * behind any pending kernel operations on any external BOs in VM or any BOs + * private to the VM. This is accomplished by the rebinds waiting on BOs + * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs + * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and + * for external BOs). + * + * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute + * mode VMs we use preempt fences and a rebind worker (TODO: add link). + * + * There is no need to flow control the ring in the exec as we write the ring at + * submission time and set the DRM scheduler max job limit SIZE_OF_RING / + * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the + * ring is available. + * + * All of this results in a rather simple exec implementation. + * + * Flow + * ~~~~ + * + * .. code-block:: + * + * Parse input arguments + * Wait for any async VM bind passed as in-fences to start + * <----------------------------------------------------------------------| + * Lock global VM lock in read mode | + * Pin userptrs (also finds userptr invalidated since last exec) | + * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | + * Validate BOs that have been evicted | + * Create job | + * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | + * Add rebind fence dependency to job | + * Add job VM dma-resv bookkeeping slot (non-compute mode) | + * Add job to external BOs dma-resv write slots (non-compute mode) | + * Check if any userptrs invalidated since pin ------ Drop locks ---------| + * Install in / out fences for job + * Submit job + * Unlock all + */ + +/* + * Add validation and rebinding to the drm_exec locking loop, since both can + * trigger eviction which may require sleeping dma_resv locks. + */ +static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec) +{ + struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm); + int ret; + + /* The fence slot added here is intended for the exec sched job. */ + xe_vm_set_validation_exec(vm, &vm_exec->exec); + ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1); + xe_vm_set_validation_exec(vm, NULL); + return ret; +} + +int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec *args = data; + struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); + u64 __user *addresses_user = u64_to_user_ptr(args->address); + struct xe_exec_queue *q; + struct xe_sync_entry *syncs = NULL; + u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; + struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; + struct drm_exec *exec = &vm_exec.exec; + u32 i, num_syncs, num_in_sync = 0, num_ufence = 0; + struct xe_validation_ctx ctx; + struct xe_sched_job *job; + struct xe_vm *vm; + bool write_locked; + int err = 0; + struct xe_hw_engine_group *group; + enum xe_hw_engine_group_execution_mode mode, previous_mode; + + if (XE_IOCTL_DBG(xe, args->extensions) || + XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) || + XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) + return -EINVAL; + + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) + return -ENOENT; + + if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) { + err = -EINVAL; + goto err_exec_queue; + } + + if (XE_IOCTL_DBG(xe, args->num_batch_buffer && + q->width != args->num_batch_buffer)) { + err = -EINVAL; + goto err_exec_queue; + } + + if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) { + err = -ECANCELED; + goto err_exec_queue; + } + + if (atomic_read(&q->job_cnt) >= XE_MAX_JOB_COUNT_PER_EXEC_QUEUE) { + trace_xe_exec_queue_reach_max_job_count(q, XE_MAX_JOB_COUNT_PER_EXEC_QUEUE); + err = -EAGAIN; + goto err_exec_queue; + } + + if (args->num_syncs) { + syncs = kzalloc_objs(*syncs, args->num_syncs); + if (!syncs) { + err = -ENOMEM; + goto err_exec_queue; + } + } + + vm = q->vm; + + for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { + err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], + &syncs_user[num_syncs], NULL, 0, + SYNC_PARSE_FLAG_EXEC | + (xe_vm_in_lr_mode(vm) ? + SYNC_PARSE_FLAG_LR_MODE : 0)); + if (err) + goto err_syncs; + + if (xe_sync_is_ufence(&syncs[num_syncs])) + num_ufence++; + + if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs])) + num_in_sync++; + } + + if (XE_IOCTL_DBG(xe, num_ufence > 1)) { + err = -EINVAL; + goto err_syncs; + } + + if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) { + err = copy_from_user(addresses, addresses_user, + sizeof(u64) * q->width); + if (err) { + err = -EFAULT; + goto err_syncs; + } + } + + group = q->hwe->hw_engine_group; + mode = xe_hw_engine_group_find_exec_mode(q); + + if (mode == EXEC_MODE_DMA_FENCE) { + err = xe_hw_engine_group_get_mode(group, mode, &previous_mode, + syncs, num_in_sync ? + num_syncs : 0); + if (err) + goto err_syncs; + } + +retry: + if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) { + err = down_write_killable(&vm->lock); + write_locked = true; + } else { + /* We don't allow execs while the VM is in error state */ + err = down_read_interruptible(&vm->lock); + write_locked = false; + } + if (err) + goto err_hw_exec_mode; + + if (write_locked) { + err = xe_vm_userptr_pin(vm); + downgrade_write(&vm->lock); + write_locked = false; + if (err) + goto err_unlock_list; + } + + if (!args->num_batch_buffer) { + err = xe_vm_lock(vm, true); + if (err) + goto err_unlock_list; + + if (!xe_vm_in_lr_mode(vm)) { + struct dma_fence *fence; + + fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + xe_vm_unlock(vm); + goto err_unlock_list; + } + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], fence); + xe_exec_queue_last_fence_set(q, vm, fence); + dma_fence_put(fence); + } + + xe_vm_unlock(vm); + goto err_unlock_list; + } + + /* + * It's OK to block interruptible here with the vm lock held, since + * on task freezing during suspend / hibernate, the call will + * return -ERESTARTSYS and the IOCTL will be rerun. + */ + err = xe_pm_block_on_suspend(xe); + if (err) + goto err_unlock_list; + + if (!xe_vm_in_lr_mode(vm)) { + vm_exec.vm = &vm->gpuvm; + vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT; + err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val); + if (err) + goto err_unlock_list; + } + + if (xe_vm_is_closed_or_banned(q->vm)) { + drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); + err = -ECANCELED; + goto err_exec; + } + + if (xe_exec_queue_uses_pxp(q)) { + err = xe_vm_validate_protected(q->vm); + if (err) + goto err_exec; + } + + job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? + addresses : &args->address); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err_exec; + } + + /* + * Wait behind rebinds and any kernel operations (evictions, defrag + * moves, ...) on the VM and all external BOs. The VM's private BOs + * carry their kernel ops in the VM dma-resv KERNEL slot, while each + * external BO carries them in its own dma-resv KERNEL slot; both are + * covered by iterating every object locked by the exec, mirroring the + * drm_gpuvm_resv_add_fence() below. + */ + if (!xe_vm_in_lr_mode(vm)) { + struct drm_gem_object *obj; + + drm_exec_for_each_locked_object(exec, obj) { + err = xe_sched_job_add_deps(job, obj->resv, + DMA_RESV_USAGE_KERNEL); + if (err) + goto err_put_job; + } + } + + for (i = 0; i < num_syncs && !err; i++) + err = xe_sync_entry_add_deps(&syncs[i], job); + if (err) + goto err_put_job; + + if (!xe_vm_in_lr_mode(vm)) { + err = xe_svm_notifier_lock_interruptible(vm); + if (err) + goto err_put_job; + + err = __xe_vm_userptr_needs_repin(vm); + if (err) + goto err_repin; + } + + /* + * Point of no return, if we error after this point just set an error on + * the job and let the DRM scheduler / backend clean up the job. + */ + xe_sched_job_arm(job); + if (!xe_vm_in_lr_mode(vm)) + drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished, + DMA_RESV_USAGE_BOOKKEEP, + DMA_RESV_USAGE_BOOKKEEP); + + for (i = 0; i < num_syncs; i++) { + xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished); + xe_sched_job_init_user_fence(job, &syncs[i]); + } + + if (!xe_vm_in_lr_mode(vm)) + xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished); + xe_sched_job_push(job); + xe_vm_reactivate_rebind(vm); + + if (!err && !xe_vm_in_lr_mode(vm)) { + spin_lock(&xe->ttm.lru_lock); + ttm_lru_bulk_move_tail(&vm->lru_bulk_move); + spin_unlock(&xe->ttm.lru_lock); + } + + if (mode == EXEC_MODE_LR) + xe_hw_engine_group_resume_faulting_lr_jobs(group); + +err_repin: + if (!xe_vm_in_lr_mode(vm)) + xe_svm_notifier_unlock(vm); +err_put_job: + if (err) + xe_sched_job_put(job); +err_exec: + if (!xe_vm_in_lr_mode(vm)) + xe_validation_ctx_fini(&ctx); +err_unlock_list: + up_read(&vm->lock); + if (err == -EAGAIN) + goto retry; +err_hw_exec_mode: + if (mode == EXEC_MODE_DMA_FENCE) + xe_hw_engine_group_put(group); +err_syncs: + while (num_syncs--) + xe_sync_entry_cleanup(&syncs[num_syncs]); + kfree(syncs); +err_exec_queue: + xe_exec_queue_put(q); + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_exec.h b/drivers/gpu/drm/xe/xe_exec.h new file mode 100644 index 000000000..e4932494c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_EXEC_H_ +#define _XE_EXEC_H_ + +struct drm_device; +struct drm_file; + +int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c new file mode 100644 index 000000000..cd053de3c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -0,0 +1,1873 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_exec_queue.h" + +#include + +#include +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_dep_scheduler.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_sriov_pf.h" +#include "xe_gt_sriov_vf.h" +#include "xe_hw_engine_class_sysfs.h" +#include "xe_hw_engine_group.h" +#include "xe_irq.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_migrate.h" +#include "xe_pm.h" +#include "xe_trace.h" +#include "xe_vm.h" +#include "xe_pxp.h" + +/** + * DOC: Execution Queue + * + * An Execution queue is an interface for the HW context of execution. + * The user creates an execution queue, submits the GPU jobs through those + * queues and in the end destroys them. + * + * Execution queues can also be created by XeKMD itself for driver internal + * operations like object migration etc. + * + * An execution queue is associated with a specified HW engine or a group of + * engines (belonging to the same tile and engine class) and any GPU job + * submitted on the queue will be run on one of these engines. + * + * An execution queue is tied to an address space (VM). It holds a reference + * of the associated VM and the underlying Logical Ring Context/s (LRC/s) + * until the queue is destroyed. + * + * The execution queue sits on top of the submission backend. It opaquely + * handles the GuC and Execlist backends whichever the platform uses, and + * the ring operations the different engine classes support. + */ + +/** + * DOC: Multi Queue Group + * + * Multi Queue Group is another mode of execution supported by the compute + * and blitter copy command streamers (CCS and BCS, respectively). It is + * an enhancement of the existing hardware architecture and leverages the + * same submission model. It enables support for efficient, parallel + * execution of multiple queues within a single shared context. The multi + * queue group functionality is only supported with GuC submission backend. + * All the queues of a group must use the same address space (VM). + * + * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property + * supports creating a multi queue group and adding queues to a queue group. + * + * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field + * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with + * the queue being created as the primary queue (aka q0) of the group. To add + * secondary queues to the group, they need to be created with the above + * property with id of the primary queue as the value. The properties of + * the primary queue (like priority, time slice) applies to the whole group. + * So, these properties can't be set for secondary queues of a group. + * + * The hardware does not support removing a queue from a multi-queue group. + * However, queues can be dynamically added to the group. A group can have + * up to 64 queues. To support this, XeKMD holds references to LRCs of the + * queues even after the queues are destroyed by the user until the whole + * group is destroyed. The secondary queues hold a reference to the primary + * queue thus preventing the group from being destroyed when user destroys + * the primary queue. Once the primary queue is destroyed, secondary queues + * can't be added to the queue group and new job submissions on existing + * secondary queues are not allowed. + * + * The queues of a multi queue group can set their priority within the group + * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property. + * This multi queue priority can also be set dynamically through the + * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property + * supported by the secondary queues of a multi queue group, other than + * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE. + * + * When GuC reports an error on any of the queues of a multi queue group, + * the queue cleanup mechanism is invoked for all the queues of the group + * as hardware cannot make progress on the multi queue context. + * + * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC + * interface. + */ + +enum xe_exec_queue_sched_prop { + XE_EXEC_QUEUE_JOB_TIMEOUT = 0, + XE_EXEC_QUEUE_TIMESLICE = 1, + XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, + XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, +}; + +static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, + u64 extensions); + +static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_lrc *lrc; + unsigned long idx; + + if (xe_exec_queue_is_multi_queue_secondary(q)) { + /* + * Put pairs with get from xe_exec_queue_lookup() call + * in xe_exec_queue_group_validate(). + */ + xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q)); + return; + } + + if (!group) + return; + + /* Primary queue cleanup */ + xa_for_each(&group->xa, idx, lrc) + xe_lrc_put(lrc); + + xa_destroy(&group->xa); + mutex_destroy(&group->list_lock); + xe_bo_unpin_map_no_vm(group->cgp_bo); + kfree(group); +} + +static void __xe_exec_queue_free(struct xe_exec_queue *q) +{ + int i; + + for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) + if (q->tlb_inval[i].dep_scheduler) + xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); + + if (xe_exec_queue_uses_pxp(q)) + xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); + + if (xe_exec_queue_is_multi_queue(q)) + xe_exec_queue_group_cleanup(q); + + if (q->vm) { + xe_vm_remove_exec_queue(q->vm, q); + xe_vm_put(q->vm); + } + + if (q->xef) + xe_file_put(q->xef); + + kvfree(q->replay_state); + kfree(q); +} + +static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_tile *tile = gt_to_tile(q->gt); + int i; + + for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { + struct xe_dep_scheduler *dep_scheduler; + struct xe_gt *gt; + struct workqueue_struct *wq; + + if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) + gt = tile->primary_gt; + else + gt = tile->media_gt; + + if (!gt) + continue; + + wq = gt->tlb_inval.job_wq; + +#define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ + dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, + MAX_TLB_INVAL_JOBS); + if (IS_ERR(dep_scheduler)) + return PTR_ERR(dep_scheduler); + + q->tlb_inval[i].dep_scheduler = dep_scheduler; + } +#undef MAX_TLB_INVAL_JOBS + + return 0; +} + +static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, + struct xe_vm *vm, + u32 logical_mask, + u16 width, struct xe_hw_engine *hwe, + u32 flags, u64 extensions) +{ + struct xe_exec_queue *q; + struct xe_gt *gt = hwe->gt; + int err; + + /* only kernel queues can be permanent */ + XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); + + q = kzalloc_flex(*q, lrc, width); + if (!q) + return ERR_PTR(-ENOMEM); + + kref_init(&q->refcount); + q->flags = flags; + q->hwe = hwe; + q->gt = gt; + q->class = hwe->class; + q->width = width; + q->msix_vec = XE_IRQ_DEFAULT_MSIX; + q->logical_mask = logical_mask; + q->fence_irq = >->fence_irq[hwe->class]; + q->ring_ops = gt->ring_ops[hwe->class]; + q->ops = gt->exec_queue_ops; + INIT_LIST_HEAD(&q->lr.link); + INIT_LIST_HEAD(&q->vm_exec_queue_link); + INIT_LIST_HEAD(&q->multi_gt_link); + INIT_LIST_HEAD(&q->hw_engine_group_link); + INIT_LIST_HEAD(&q->pxp.link); + spin_lock_init(&q->multi_queue.lock); + spin_lock_init(&q->lrc_lookup_lock); + q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL; + + q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; + q->sched_props.preempt_timeout_us = + hwe->eclass->sched_props.preempt_timeout_us; + q->sched_props.job_timeout_ms = + hwe->eclass->sched_props.job_timeout_ms; + if (q->flags & EXEC_QUEUE_FLAG_KERNEL && + q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) + q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; + else + q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; + + if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { + err = alloc_dep_schedulers(xe, q); + if (err) { + __xe_exec_queue_free(q); + return ERR_PTR(err); + } + } + + if (vm) + q->vm = xe_vm_get(vm); + + if (extensions) { + /* + * may set q->usm, must come before xe_lrc_create(), + * may overwrite q->sched_props, must come before q->ops->init() + */ + err = exec_queue_user_extensions(xe, q, extensions); + if (err) { + __xe_exec_queue_free(q); + return ERR_PTR(err); + } + } + + return q; +} + +static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u16 idx) +{ + xe_assert(gt_to_xe(q->gt), idx < q->width); + + scoped_guard(spinlock, &q->lrc_lookup_lock) { + q->lrc[idx] = lrc; + if (xe_exec_queue_is_multi_queue(q)) + q->lrc[idx]->multi_queue.primary_lrc = + q->multi_queue.group->primary->lrc[0]; + } +} + +/** + * xe_exec_queue_get_lrc() - Get the LRC from exec queue. + * @q: The exec queue instance. + * @idx: Index within multi-LRC array. + * + * Retrieves LRC of given index for the exec queue under lock + * and takes reference. + * + * Return: Pointer to LRC on success, error on failure, NULL on + * lookup failure. + */ +struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx) +{ + struct xe_lrc *lrc; + + xe_assert(gt_to_xe(q->gt), idx < q->width); + + scoped_guard(spinlock, &q->lrc_lookup_lock) { + lrc = q->lrc[idx]; + if (lrc) + xe_lrc_get(lrc); + } + + return lrc; +} + +/** + * xe_exec_queue_lrc() - Get the LRC from exec queue. + * @q: The exec queue instance. + * + * Retrieves the primary LRC for the exec queue. Note that this function + * returns only the first LRC instance, even when multiple parallel LRCs + * are configured. This function does not increment reference count, + * so the reference can be just forgotten after use. + * + * Return: Pointer to LRC on success, error on failure + */ +struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) +{ + return q->lrc[0]; +} + +static void __xe_exec_queue_fini(struct xe_exec_queue *q) +{ + int i; + + q->ops->fini(q); + + for (i = 0; i < q->width; ++i) + xe_lrc_put(q->lrc[i]); +} + +static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) +{ + int i, err; + u32 flags = 0; + + /* + * PXP workloads executing on RCS or CCS must run in isolation (i.e. no + * other workload can use the EUs at the same time). On MTL this is done + * by setting the RUNALONE bit in the LRC, while starting on Xe2 there + * is a dedicated bit for it. + */ + if (xe_exec_queue_uses_pxp(q) && + (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { + if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) + flags |= XE_LRC_CREATE_PXP; + else + flags |= XE_LRC_CREATE_RUNALONE; + } + + if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL)) + flags |= XE_LRC_CREATE_USER_CTX; + + if (q->flags & EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX) + flags |= XE_LRC_DISABLE_STATE_CACHE_PERF_FIX; + + err = q->ops->init(q); + if (err) + return err; + + /* + * This must occur after q->ops->init to avoid race conditions during VF + * post-migration recovery, as the fixups for the LRC GGTT addresses + * depend on the queue being present in the backend tracking structure. + * + * In addition to above, we must wait on inflight GGTT changes to avoid + * writing out stale values here. Such wait provides a solid solution + * (without a race) only if the function can detect migration instantly + * from the moment vCPU resumes execution. + */ + for (i = 0; i < q->width; ++i) { + struct xe_lrc *__lrc = NULL; + int marker; + + do { + struct xe_lrc *lrc; + + marker = xe_gt_sriov_vf_wait_valid_ggtt(q->gt); + + lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state, + xe_lrc_ring_size(), q->msix_vec, flags); + if (IS_ERR(lrc)) { + err = PTR_ERR(lrc); + goto err_lrc; + } + + xe_exec_queue_set_lrc(q, lrc, i); + + if (__lrc) + xe_lrc_put(__lrc); + __lrc = lrc; + + } while (marker != xe_vf_migration_fixups_complete_count(q->gt)); + } + + return 0; + +err_lrc: + __xe_exec_queue_fini(q); + return err; +} + +/** + * xe_exec_queue_create() - Create an exec queue + * @xe: Xe device + * @vm: VM for the exec queue + * @logical_mask: Logical mask of HW engines + * @width: Width of the exec queue (number of LRCs) + * @hwe: Hardware engine + * @flags: Exec queue creation flags + * @extensions: Extensions for exec queue creation + * + * Create an exec queue (allocate and initialize) with the specified parameters + * + * Return: Pointer to the created exec queue on success, ERR_PTR on failure + */ +struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, + u32 logical_mask, u16 width, + struct xe_hw_engine *hwe, u32 flags, + u64 extensions) +{ + struct xe_exec_queue *q; + int err; + + /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ + xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); + + q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, + extensions); + if (IS_ERR(q)) + return q; + + err = __xe_exec_queue_init(q, flags); + if (err) + goto err_post_alloc; + + /* + * We can only add the queue to the PXP list after the init is complete, + * because the PXP termination can call exec_queue_kill and that will + * go bad if the queue is only half-initialized. This means that we + * can't do it when we handle the PXP extension in __xe_exec_queue_alloc + * and we need to do it here instead. + */ + if (xe_exec_queue_uses_pxp(q)) { + err = xe_pxp_exec_queue_add(xe->pxp, q); + if (err) + goto err_post_init; + } + + return q; + +err_post_init: + __xe_exec_queue_fini(q); +err_post_alloc: + __xe_exec_queue_free(q); + return ERR_PTR(err); +} +ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); + +/** + * xe_exec_queue_create_class() - Create an exec queue for a specific engine class + * @xe: Xe device + * @gt: GT for the exec queue + * @vm: VM for the exec queue + * @class: Engine class + * @flags: Exec queue creation flags + * @extensions: Extensions for exec queue creation + * + * Create an exec queue for the specified engine class. + * + * Return: Pointer to the created exec queue on success, ERR_PTR on failure + */ +struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, + enum xe_engine_class class, + u32 flags, u64 extensions) +{ + struct xe_hw_engine *hwe, *hwe0 = NULL; + enum xe_hw_engine_id id; + u32 logical_mask = 0; + + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + + if (hwe->class == class) { + logical_mask |= BIT(hwe->logical_instance); + if (!hwe0) + hwe0 = hwe; + } + } + + if (!logical_mask) + return ERR_PTR(-ENODEV); + + return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); +} + +/** + * xe_exec_queue_create_bind() - Create bind exec queue. + * @xe: Xe device. + * @tile: tile which bind exec queue belongs to. + * @flags: exec queue creation flags + * @user_vm: The user VM which this exec queue belongs to + * @extensions: exec queue creation extensions + * + * Normalize bind exec queue creation. Bind exec queue is tied to migration VM + * for access to physical memory required for page table programming. On a + * faulting devices the reserved copy engine instance must be used to avoid + * deadlocking (user binds cannot get stuck behind faults as kernel binds which + * resolve faults depend on user binds). On non-faulting devices any copy engine + * can be used. + * + * Returns exec queue on success, ERR_PTR on failure + */ +struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, + struct xe_tile *tile, + struct xe_vm *user_vm, + u32 flags, u64 extensions) +{ + struct xe_gt *gt = tile->primary_gt; + struct xe_exec_queue *q; + struct xe_vm *migrate_vm; + + migrate_vm = xe_migrate_get_vm(tile->migrate); + if (xe->info.has_usm) { + struct xe_hw_engine *hwe = gt->usm.paging_hwe0; + + if (!hwe) { + xe_vm_put(migrate_vm); + return ERR_PTR(-EINVAL); + } + + q = xe_exec_queue_create(xe, migrate_vm, + BIT(hwe->logical_instance), 1, hwe, + flags, extensions); + } else { + q = xe_exec_queue_create_class(xe, gt, migrate_vm, + XE_ENGINE_CLASS_COPY, flags, + extensions); + } + xe_vm_put(migrate_vm); + + if (!IS_ERR(q)) { + int err = drm_syncobj_create(&q->ufence_syncobj, + DRM_SYNCOBJ_CREATE_SIGNALED, + NULL); + if (err) { + xe_exec_queue_put(q); + return ERR_PTR(err); + } + + if (user_vm) + q->user_vm = xe_vm_get(user_vm); + } + + return q; +} +ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); + +/** + * xe_exec_queue_destroy() - Destroy an exec queue + * @ref: Reference count of the exec queue + * + * Called when the last reference to the exec queue is dropped. + * Cleans up all resources associated with the exec queue. + * This function should not be called directly; use xe_exec_queue_put() instead. + */ +void xe_exec_queue_destroy(struct kref *ref) +{ + struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); + struct xe_exec_queue *eq, *next; + int i; + + xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0); + + if (q->ufence_syncobj) + drm_syncobj_put(q->ufence_syncobj); + + if (xe_exec_queue_uses_pxp(q)) + xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); + + xe_exec_queue_last_fence_put_unlocked(q); + for_each_tlb_inval(i) + xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i); + + if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { + list_for_each_entry_safe(eq, next, &q->multi_gt_list, + multi_gt_link) + xe_exec_queue_put(eq); + } + + if (q->user_vm) { + xe_vm_put(q->user_vm); + q->user_vm = NULL; + } + + q->ops->destroy(q); +} + +/** + * xe_exec_queue_fini() - Finalize an exec queue + * @q: The exec queue + * + * Finalizes the exec queue by updating run ticks, releasing LRC references, + * and freeing the queue structure. This is called after the queue has been + * destroyed and all references have been dropped. + */ +void xe_exec_queue_fini(struct xe_exec_queue *q) +{ + /* + * Before releasing our ref to lrc and xef, accumulate our run ticks + * and wakeup any waiters. + */ + xe_exec_queue_update_run_ticks(q); + if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) + wake_up_var(&q->xef->exec_queue.pending_removal); + + __xe_exec_queue_fini(q); + __xe_exec_queue_free(q); +} + +/** + * xe_exec_queue_assign_name() - Assign a name to an exec queue + * @q: The exec queue + * @instance: Instance number for the engine + * + * Assigns a human-readable name to the exec queue based on its engine class + * and instance number (e.g., "rcs0", "vcs1", "bcs2"). + */ +void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) +{ + switch (q->class) { + case XE_ENGINE_CLASS_RENDER: + snprintf(q->name, sizeof(q->name), "rcs%d", instance); + break; + case XE_ENGINE_CLASS_VIDEO_DECODE: + snprintf(q->name, sizeof(q->name), "vcs%d", instance); + break; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + snprintf(q->name, sizeof(q->name), "vecs%d", instance); + break; + case XE_ENGINE_CLASS_COPY: + snprintf(q->name, sizeof(q->name), "bcs%d", instance); + break; + case XE_ENGINE_CLASS_COMPUTE: + snprintf(q->name, sizeof(q->name), "ccs%d", instance); + break; + case XE_ENGINE_CLASS_OTHER: + snprintf(q->name, sizeof(q->name), "gsccs%d", instance); + break; + default: + XE_WARN_ON(q->class); + } +} + +/** + * xe_exec_queue_lookup() - Look up an exec queue by ID + * @xef: Xe file private data + * @id: Exec queue ID + * + * Looks up an exec queue by its ID and increments its reference count. + * + * Return: Pointer to the exec queue if found, NULL otherwise + */ +struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) +{ + struct xe_exec_queue *q; + + mutex_lock(&xef->exec_queue.lock); + q = xa_load(&xef->exec_queue.xa, id); + if (q) + xe_exec_queue_get(q); + mutex_unlock(&xef->exec_queue.lock); + + return q; +} + +/** + * xe_exec_queue_device_get_max_priority() - Get maximum priority for an exec queues + * @xe: Xe device + * + * Returns the maximum priority level that can be assigned to an exec queues. + * + * Return: Maximum priority level (HIGH if CAP_SYS_NICE, NORMAL otherwise) + */ +enum xe_exec_queue_priority +xe_exec_queue_device_get_max_priority(struct xe_device *xe) +{ + return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : + XE_EXEC_QUEUE_PRIORITY_NORMAL; +} + +static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) + return -EPERM; + + q->sched_props.priority = value; + return 0; +} + +static bool xe_exec_queue_enforce_schedule_limit(void) +{ +#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) + return true; +#else + return !capable(CAP_SYS_NICE); +#endif +} + +static void +xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, + enum xe_exec_queue_sched_prop prop, + u32 *min, u32 *max) +{ + switch (prop) { + case XE_EXEC_QUEUE_JOB_TIMEOUT: + *min = eclass->sched_props.job_timeout_min; + *max = eclass->sched_props.job_timeout_max; + break; + case XE_EXEC_QUEUE_TIMESLICE: + *min = eclass->sched_props.timeslice_min; + *max = eclass->sched_props.timeslice_max; + break; + case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: + *min = eclass->sched_props.preempt_timeout_min; + *max = eclass->sched_props.preempt_timeout_max; + break; + default: + break; + } +#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) + if (capable(CAP_SYS_NICE)) { + switch (prop) { + case XE_EXEC_QUEUE_JOB_TIMEOUT: + *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; + *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; + break; + case XE_EXEC_QUEUE_TIMESLICE: + *min = XE_HW_ENGINE_TIMESLICE_MIN; + *max = XE_HW_ENGINE_TIMESLICE_MAX; + break; + case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: + *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; + *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; + break; + default: + break; + } + } +#endif +} + +static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + u32 min = 0, max = 0; + + xe_exec_queue_get_prop_minmax(q->hwe->eclass, + XE_EXEC_QUEUE_TIMESLICE, &min, &max); + + if (xe_exec_queue_enforce_schedule_limit() && + !xe_hw_engine_timeout_in_range(value, min, max)) + return -EINVAL; + + q->sched_props.timeslice_us = value; + return 0; +} + +static int +exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) +{ + if (value == DRM_XE_PXP_TYPE_NONE) + return 0; + + /* we only support HWDRM sessions right now */ + if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) + return -EINVAL; + + if (!xe_pxp_is_enabled(xe->pxp)) + return -ENODEV; + + return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); +} + +static int exec_queue_set_hang_replay_state(struct xe_device *xe, + struct xe_exec_queue *q, + u64 value) +{ + size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class); + u64 __user *address = u64_to_user_ptr(value); + void *ptr; + + if (q->replay_state) + return -EINVAL; + + ptr = vmemdup_user(address, size); + if (XE_IOCTL_DBG(xe, IS_ERR(ptr))) + return PTR_ERR(ptr); + + q->replay_state = ptr; + + return 0; +} + +static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_tile *tile = gt_to_tile(q->gt); + struct xe_exec_queue_group *group; + struct xe_bo *bo; + + group = kzalloc_obj(*group); + if (!group) + return -ENOMEM; + + bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_PINNED_LATE_RESTORE | + XE_BO_FLAG_FORCE_USER_VRAM | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_GGTT, false); + if (IS_ERR(bo)) { + drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n", + PTR_ERR(bo)); + kfree(group); + return PTR_ERR(bo); + } + + xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K); + + group->primary = q; + group->cgp_bo = bo; + INIT_LIST_HEAD(&group->list); + spin_lock_init(&group->suspend_lock); + xa_init_flags(&group->xa, XA_FLAGS_ALLOC1); + mutex_init(&group->list_lock); + q->multi_queue.group = group; + + /* group->list_lock is used in submission backend */ + if (IS_ENABLED(CONFIG_LOCKDEP)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&group->list_lock); + fs_reclaim_release(GFP_KERNEL); + } + + return 0; +} + +static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q, + u32 primary_id) +{ + struct xe_exec_queue_group *group; + struct xe_exec_queue *primary; + int ret; + + /* + * Get from below xe_exec_queue_lookup() pairs with put + * in xe_exec_queue_group_cleanup(). + */ + primary = xe_exec_queue_lookup(q->vm->xef, primary_id); + if (XE_IOCTL_DBG(xe, !primary)) + return -ENOENT; + + if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) || + XE_IOCTL_DBG(xe, q->vm != primary->vm) || + XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) { + ret = -EINVAL; + goto put_primary; + } + + group = primary->multi_queue.group; + q->multi_queue.valid = true; + q->multi_queue.group = group; + + return 0; +put_primary: + xe_exec_queue_put(primary); + return ret; +} + +#define XE_MAX_GROUP_SIZE 64 +static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 pos; + int err; + + xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q)); + + /* Primary queue holds a reference to LRCs of all secondary queues */ + err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]), + XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL); + if (XE_IOCTL_DBG(xe, err)) { + xe_lrc_put(q->lrc[0]); + + /* It is invalid if queue group limit is exceeded */ + if (err == -EBUSY) + err = -EINVAL; + + return err; + } + + q->multi_queue.pos = pos; + q->lrc[0]->multi_queue.pos = pos; + + return 0; +} + +static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_lrc *lrc; + + xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q)); + + lrc = xa_erase(&group->xa, q->multi_queue.pos); + xe_assert(xe, lrc); + xe_lrc_put(lrc); +} + +static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + if (XE_IOCTL_DBG(xe, !xe_gt_supports_multi_queue(q->gt, q->class))) + return -ENODEV; + + if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe))) + return -EOPNOTSUPP; + + if (XE_IOCTL_DBG(xe, !q->vm->xef)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q))) + return -EINVAL; + + if (value & DRM_XE_MULTI_GROUP_CREATE) { + if (XE_IOCTL_DBG(xe, value & ~DRM_XE_MULTI_GROUP_CREATE)) + return -EINVAL; + + q->multi_queue.valid = true; + q->multi_queue.is_primary = true; + q->multi_queue.pos = 0; + return 0; + } + + /* While adding secondary queues, the upper 32 bits must be 0 */ + if (XE_IOCTL_DBG(xe, value & (~0ull << 32))) + return -EINVAL; + + return xe_exec_queue_group_validate(xe, q, value); +} + +static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH)) + return -EINVAL; + + /* For queue creation time (!q->xef) setting, just store the priority value */ + if (!q->xef) { + q->multi_queue.priority = value; + return 0; + } + + if (!xe_exec_queue_is_multi_queue(q)) + return -EINVAL; + + return q->ops->set_multi_queue_priority(q, value); +} + +static int exec_queue_set_state_cache_perf_fix(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + if (XE_IOCTL_DBG(xe, q->class != XE_ENGINE_CLASS_RENDER)) + return -EOPNOTSUPP; + + q->flags |= value != 0 ? EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX : 0; + + return 0; +} + +typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, + struct xe_exec_queue *q, + u64 value); + +static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, + [DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] = + exec_queue_set_multi_queue_priority, + [DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX] = + exec_queue_set_state_cache_perf_fix, +}; + +/** + * xe_exec_queue_set_property_ioctl() - Set a property on an exec queue + * @dev: DRM device + * @data: IOCTL data + * @file: DRM file + * + * Allows setting properties on an existing exec queue. Currently only + * supports setting multi-queue priority. + * + * Return: 0 on success, negative error code on failure + */ +int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec_queue_set_property *args = data; + struct xe_exec_queue *q; + int ret; + u32 idx; + + if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->property != + DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) + return -EINVAL; + + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) + return -ENOENT; + + idx = array_index_nospec(args->property, + ARRAY_SIZE(exec_queue_set_property_funcs)); + ret = exec_queue_set_property_funcs[idx](xe, q, args->value); + if (XE_IOCTL_DBG(xe, ret)) + goto err_post_lookup; + + xe_exec_queue_put(q); + return 0; + + err_post_lookup: + xe_exec_queue_put(q); + return ret; +} + +static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties) +{ + struct xe_device *xe = gt_to_xe(q->gt); + u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) | + BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY); + + /* + * Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a + * multi-queue group. + */ + if (xe_exec_queue_is_multi_queue_secondary(q) && + properties & ~secondary_queue_valid_props) + return -EINVAL; + + /* + * HWDRM is the only supported PXP type today. It is display related and + * hence can't work with multi-queue. Reject the combination. The secondary + * queue path above already rejects any PXP property, so this also covers + * the multi-queue primary which would otherwise allow it. + */ + if (XE_IOCTL_DBG(xe, (properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)) && + (properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)))) + return -EINVAL; + + return 0; +} + +static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties) +{ + /* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */ + if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) && + !(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP))) + return -EINVAL; + + return 0; +} + +static int exec_queue_user_ext_set_property(struct xe_device *xe, + struct xe_exec_queue *q, + u64 extension, u64 *properties) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.property >= + ARRAY_SIZE(exec_queue_set_property_funcs)) || + XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE && + ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY && + ext.property != DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); + if (!exec_queue_set_property_funcs[idx]) + return -EINVAL; + + *properties |= BIT_ULL(idx); + err = exec_queue_user_ext_check(q, *properties); + if (XE_IOCTL_DBG(xe, err)) + return err; + + return exec_queue_set_property_funcs[idx](xe, q, ext.value); +} + +typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, + struct xe_exec_queue *q, + u64 extension, u64 *properties); + +static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { + [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 16 +static int __exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, + u64 extensions, int ext_number, u64 *properties) +{ + u64 __user *address = u64_to_user_ptr(extensions); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.name >= + ARRAY_SIZE(exec_queue_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, + ARRAY_SIZE(exec_queue_user_extension_funcs)); + err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (ext.next_extension) + return __exec_queue_user_extensions(xe, q, ext.next_extension, + ++ext_number, properties); + + return 0; +} + +static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, + u64 extensions) +{ + u64 properties = 0; + int err; + + err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties); + if (XE_IOCTL_DBG(xe, err)) + return err; + + err = exec_queue_user_ext_check_final(q, properties); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (xe_exec_queue_is_multi_queue_primary(q)) { + err = xe_exec_queue_group_init(xe, q); + if (XE_IOCTL_DBG(xe, err)) + return err; + } + + return 0; +} + +static u32 calc_validate_logical_mask(struct xe_device *xe, + struct drm_xe_engine_class_instance *eci, + u16 width, u16 num_placements) +{ + int len = width * num_placements; + int i, j, n; + u16 class; + u16 gt_id; + u32 return_mask = 0, prev_mask; + + if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && + len > 1)) + return 0; + + for (i = 0; i < width; ++i) { + u32 current_mask = 0; + + for (j = 0; j < num_placements; ++j) { + struct xe_hw_engine *hwe; + + n = j * width + i; + + hwe = xe_hw_engine_lookup(xe, eci[n]); + if (XE_IOCTL_DBG(xe, !hwe)) + return 0; + + if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) + return 0; + + if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || + XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) + return 0; + + class = eci[n].engine_class; + gt_id = eci[n].gt_id; + + if (width == 1 || !i) + return_mask |= BIT(eci[n].engine_instance); + current_mask |= BIT(eci[n].engine_instance); + } + + /* Parallel submissions must be logically contiguous */ + if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) + return 0; + + prev_mask = current_mask; + } + + return return_mask; +} + +static bool has_sched_groups(struct xe_gt *gt) +{ + if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_sriov_pf_sched_groups_enabled(gt)) + return true; + + if (IS_SRIOV_VF(gt_to_xe(gt)) && xe_gt_sriov_vf_sched_groups_enabled(gt)) + return true; + + return false; +} + +/** + * xe_exec_queue_create_ioctl() - Create an exec queue via IOCTL + * @dev: DRM device + * @data: IOCTL data + * @file: DRM file + * + * Creates a new exec queue based on user-provided parameters. Supports + * creating VM bind queues, regular exec queues, multi-lrc exec queues + * and multi-queue groups. + * + * Return: 0 on success with exec_queue_id filled in, negative error code on failure + */ +int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec_queue_create *args = data; + struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; + struct drm_xe_engine_class_instance __user *user_eci = + u64_to_user_ptr(args->instances); + struct xe_hw_engine *hwe; + struct xe_vm *vm; + struct xe_tile *tile; + struct xe_exec_queue *q = NULL; + u32 logical_mask; + u32 flags = 0; + u32 id; + u32 len; + int err; + + if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + len = args->width * args->num_placements; + if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) + return -EINVAL; + + err = copy_from_user(eci, user_eci, + sizeof(struct drm_xe_engine_class_instance) * len); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id))) + return -EINVAL; + + if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) + flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; + + if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { + if (XE_IOCTL_DBG(xe, args->width != 1) || + XE_IOCTL_DBG(xe, args->num_placements != 1) || + XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) + return -EINVAL; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -ENOENT; + + err = down_read_interruptible(&vm->lock); + if (err) { + xe_vm_put(vm); + return err; + } + + if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { + up_read(&vm->lock); + xe_vm_put(vm); + return -ENOENT; + } + + for_each_tile(tile, xe, id) { + struct xe_exec_queue *new; + + flags |= EXEC_QUEUE_FLAG_VM; + if (id) + flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; + + new = xe_exec_queue_create_bind(xe, tile, vm, flags, + args->extensions); + if (IS_ERR(new)) { + up_read(&vm->lock); + xe_vm_put(vm); + err = PTR_ERR(new); + if (q) + goto put_exec_queue; + return err; + } + if (id == 0) + q = new; + else + list_add_tail(&new->multi_gt_list, + &q->multi_gt_link); + } + up_read(&vm->lock); + xe_vm_put(vm); + } else { + logical_mask = calc_validate_logical_mask(xe, eci, + args->width, + args->num_placements); + if (XE_IOCTL_DBG(xe, !logical_mask)) + return -EINVAL; + + hwe = xe_hw_engine_lookup(xe, eci[0]); + if (XE_IOCTL_DBG(xe, !hwe)) + return -EINVAL; + + /* multi-lrc is only supported on select engine classes */ + if (XE_IOCTL_DBG(xe, args->width > 1 && + !(xe->info.multi_lrc_mask & BIT(hwe->class)))) + return -EOPNOTSUPP; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -ENOENT; + + err = down_read_interruptible(&vm->lock); + if (err) { + xe_vm_put(vm); + return err; + } + + if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { + up_read(&vm->lock); + xe_vm_put(vm); + return -ENOENT; + } + + /* SRIOV sched groups are not compatible with multi-lrc */ + if (XE_IOCTL_DBG(xe, args->width > 1 && has_sched_groups(hwe->gt))) { + up_read(&vm->lock); + xe_vm_put(vm); + return -EINVAL; + } + + q = xe_exec_queue_create(xe, vm, logical_mask, + args->width, hwe, flags, + args->extensions); + up_read(&vm->lock); + xe_vm_put(vm); + if (IS_ERR(q)) + return PTR_ERR(q); + + if (xe_exec_queue_is_multi_queue_secondary(q)) { + err = xe_exec_queue_group_add(xe, q); + if (XE_IOCTL_DBG(xe, err)) + goto put_exec_queue; + } + + if (xe_vm_in_preempt_fence_mode(vm)) { + q->lr.context = dma_fence_context_alloc(1); + + err = xe_vm_add_compute_exec_queue(vm, q); + if (XE_IOCTL_DBG(xe, err)) + goto delete_queue_group; + } + + if (q->vm && q->hwe->hw_engine_group) { + err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); + if (err) + goto kill_exec_queue; + } + } + + q->xef = xe_file_get(xef); + if (eci[0].engine_class != DRM_XE_ENGINE_CLASS_VM_BIND) + xe_vm_add_exec_queue(vm, q); + + /* user id alloc must always be last in ioctl to prevent UAF */ + err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); + if (err) + goto del_hw_engine_group; + + args->exec_queue_id = id; + + return 0; + +del_hw_engine_group: + if (q->vm && q->hwe && q->hwe->hw_engine_group) + xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); +kill_exec_queue: + xe_exec_queue_kill(q); +delete_queue_group: + if (xe_exec_queue_is_multi_queue_secondary(q)) + xe_exec_queue_group_delete(xe, q); +put_exec_queue: + xe_exec_queue_put(q); + return err; +} + +/** + * xe_exec_queue_get_property_ioctl() - Get a property from an exec queue + * @dev: DRM device + * @data: IOCTL data + * @file: DRM file + * + * Retrieves property values from an existing exec queue. Currently supports + * getting the ban/reset status. + * + * Return: 0 on success with value filled in, negative error code on failure + */ +int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec_queue_get_property *args = data; + struct xe_exec_queue *q; + int ret; + + if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) + return -ENOENT; + + switch (args->property) { + case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: + args->value = q->ops->reset_status(q); + ret = 0; + break; + default: + ret = -EINVAL; + } + + xe_exec_queue_put(q); + + return ret; +} + +/** + * xe_exec_queue_is_lr() - Whether an exec_queue is long-running + * @q: The exec_queue + * + * Return: True if the exec_queue is long-running, false otherwise. + */ +bool xe_exec_queue_is_lr(struct xe_exec_queue *q) +{ + return q->vm && xe_vm_in_lr_mode(q->vm) && + !(q->flags & EXEC_QUEUE_FLAG_VM); +} + +/** + * xe_exec_queue_is_idle() - Whether an exec_queue is idle. + * @q: The exec_queue + * + * FIXME: Need to determine what to use as the short-lived + * timeline lock for the exec_queues, so that the return value + * of this function becomes more than just an advisory + * snapshot in time. The timeline lock must protect the + * seqno from racing submissions on the same exec_queue. + * Typically vm->resv, but user-created timeline locks use the migrate vm + * and never grabs the migrate vm->resv so we have a race there. + * + * Return: True if the exec_queue is idle, false otherwise. + */ +bool xe_exec_queue_is_idle(struct xe_exec_queue *q) +{ + if (xe_exec_queue_is_parallel(q)) { + int i; + + for (i = 0; i < q->width; ++i) { + if (xe_lrc_seqno(q->lrc[i]) != + q->lrc[i]->fence_ctx.next_seqno - 1) + return false; + } + + return true; + } + + return xe_lrc_seqno(q->lrc[0]) == + q->lrc[0]->fence_ctx.next_seqno - 1; +} + +/** + * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue + * from hw + * @q: The exec queue + * + * Update the timestamp saved by HW for this exec queue and save run ticks + * calculated by using the delta from last update. + */ +void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) +{ + struct xe_device *xe = gt_to_xe(q->gt); + struct xe_lrc *lrc; + u64 old_ts, new_ts; + int idx; + + /* + * Jobs that are executed by kernel doesn't have a corresponding xe_file + * and thus are not accounted. + */ + if (!q->xef) + return; + + /* Synchronize with unbind while holding the xe file open */ + if (!drm_dev_enter(&xe->drm, &idx)) + return; + /* + * Only sample the first LRC. For parallel submission, all of them are + * scheduled together and we compensate that below by multiplying by + * width - this may introduce errors if that premise is not true and + * they don't exit 100% aligned. On the other hand, looping through + * the LRCs and reading them in different time could also introduce + * errors. + */ + lrc = q->lrc[0]; + new_ts = xe_lrc_update_timestamp(lrc, &old_ts); + q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; + + drm_dev_exit(idx); +} + +/** + * xe_exec_queue_kill - permanently stop all execution from an exec queue + * @q: The exec queue + * + * This function permanently stops all activity on an exec queue. If the queue + * is actively executing on the HW, it will be kicked off the engine; any + * pending jobs are discarded and all future submissions are rejected. + * This function is safe to call multiple times. + */ +void xe_exec_queue_kill(struct xe_exec_queue *q) +{ + struct xe_exec_queue *eq = q, *next; + + list_for_each_entry_safe(eq, next, &eq->multi_gt_list, + multi_gt_link) { + q->ops->kill(eq); + xe_vm_remove_compute_exec_queue(q->vm, eq); + } + + q->ops->kill(q); + xe_vm_remove_compute_exec_queue(q->vm, q); +} + +/** + * xe_exec_queue_destroy_ioctl() - Destroy an exec queue via IOCTL + * @dev: DRM device + * @data: IOCTL data + * @file: DRM file + * + * Destroys an existing exec queue and releases its reference. + * + * Return: 0 on success, negative error code on failure + */ +int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec_queue_destroy *args = data; + struct xe_exec_queue *q; + + if (XE_IOCTL_DBG(xe, args->pad) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + mutex_lock(&xef->exec_queue.lock); + q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); + if (q) + atomic_inc(&xef->exec_queue.pending_removal); + mutex_unlock(&xef->exec_queue.lock); + + if (XE_IOCTL_DBG(xe, !q)) + return -ENOENT; + + if (q->vm && q->hwe->hw_engine_group) + xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); + + xe_exec_queue_kill(q); + + trace_xe_exec_queue_close(q); + xe_exec_queue_put(q); + + return 0; +} + +static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, + struct xe_vm *vm) +{ + if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) { + xe_migrate_job_lock_assert(q); + } else if (q->flags & EXEC_QUEUE_FLAG_VM) { + lockdep_assert_held(&vm->lock); + } else { + xe_vm_assert_held(vm); + lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); + } +} + +/** + * xe_exec_queue_last_fence_put() - Drop ref to last fence + * @q: The exec queue + * @vm: The VM the engine does a bind or exec for + */ +void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) +{ + xe_exec_queue_last_fence_lockdep_assert(q, vm); + + xe_exec_queue_last_fence_put_unlocked(q); +} + +/** + * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked + * @q: The exec queue + * + * Only safe to be called from xe_exec_queue_destroy(). + */ +void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) +{ + if (q->last_fence) { + dma_fence_put(q->last_fence); + q->last_fence = NULL; + } +} + +/** + * xe_exec_queue_last_fence_get() - Get last fence + * @q: The exec queue + * @vm: The VM the engine does a bind or exec for + * + * Get last fence, takes a ref + * + * Returns: last fence if not signaled, dma fence stub if signaled + */ +struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, + struct xe_vm *vm) +{ + struct dma_fence *fence; + + xe_exec_queue_last_fence_lockdep_assert(q, vm); + + if (q->last_fence && + test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) + xe_exec_queue_last_fence_put(q, vm); + + fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); + dma_fence_get(fence); + return fence; +} + +/** + * xe_exec_queue_last_fence_get_for_resume() - Get last fence + * @q: The exec queue + * @vm: The VM the engine does a bind or exec for + * + * Get last fence, takes a ref. Only safe to be called in the context of + * resuming the hw engine group's long-running exec queue, when the group + * semaphore is held. + * + * Returns: last fence if not signaled, dma fence stub if signaled + */ +struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, + struct xe_vm *vm) +{ + struct dma_fence *fence; + + lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); + + if (q->last_fence && + test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) + xe_exec_queue_last_fence_put_unlocked(q); + + fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); + dma_fence_get(fence); + return fence; +} + +/** + * xe_exec_queue_last_fence_set() - Set last fence + * @q: The exec queue + * @vm: The VM the engine does a bind or exec for + * @fence: The fence + * + * Set the last fence for the engine. Increases reference count for fence, when + * closing engine xe_exec_queue_last_fence_put should be called. + */ +void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, + struct dma_fence *fence) +{ + xe_exec_queue_last_fence_lockdep_assert(q, vm); + xe_assert(vm->xe, !dma_fence_is_container(fence)); + + xe_exec_queue_last_fence_put(q, vm); + q->last_fence = dma_fence_get(fence); +} + +/** + * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence + * @q: The exec queue + * @vm: The VM the engine does a bind for + * @type: Either primary or media GT + */ +void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, + struct xe_vm *vm, + unsigned int type) +{ + xe_exec_queue_last_fence_lockdep_assert(q, vm); + xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + + xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type); +} + +/** + * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB + * invalidation fence unlocked + * @q: The exec queue + * @type: Either primary or media GT + * + * Only safe to be called from xe_exec_queue_destroy(). + */ +void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, + unsigned int type) +{ + xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + + dma_fence_put(q->tlb_inval[type].last_fence); + q->tlb_inval[type].last_fence = NULL; +} + +/** + * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation + * @q: The exec queue + * @vm: The VM the engine does a bind for + * @type: Either primary or media GT + * + * Get last fence, takes a ref + * + * Returns: last fence if not signaled, dma fence stub if signaled + */ +struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q, + struct xe_vm *vm, + unsigned int type) +{ + struct dma_fence *fence; + + xe_exec_queue_last_fence_lockdep_assert(q, vm); + xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | + EXEC_QUEUE_FLAG_MIGRATE)); + + if (q->tlb_inval[type].last_fence && + test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &q->tlb_inval[type].last_fence->flags)) + xe_exec_queue_tlb_inval_last_fence_put(q, vm, type); + + fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub(); + dma_fence_get(fence); + return fence; +} + +/** + * xe_exec_queue_tlb_inval_last_fence_set() - Set last fence for TLB invalidation + * @q: The exec queue + * @vm: The VM the engine does a bind for + * @fence: The fence + * @type: Either primary or media GT + * + * Set the last fence for the tlb invalidation type on the queue. Increases + * reference count for fence, when closing queue + * xe_exec_queue_tlb_inval_last_fence_put should be called. + */ +void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q, + struct xe_vm *vm, + struct dma_fence *fence, + unsigned int type) +{ + xe_exec_queue_last_fence_lockdep_assert(q, vm); + xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | + EXEC_QUEUE_FLAG_MIGRATE)); + xe_assert(vm->xe, !dma_fence_is_container(fence)); + + xe_exec_queue_tlb_inval_last_fence_put(q, vm, type); + q->tlb_inval[type].last_fence = dma_fence_get(fence); +} + +/** + * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references + * within all LRCs of a queue. + * @q: the &xe_exec_queue struct instance containing target LRCs + * @scratch: scratch buffer to be used as temporary storage + * + * Returns: zero on success, negative error code on failure + */ +int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) +{ + int i; + int err = 0; + + for (i = 0; i < q->width; ++i) { + struct xe_lrc *lrc; + + lrc = xe_exec_queue_get_lrc(q, i); + if (!lrc) + continue; + + xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch); + xe_lrc_update_hwctx_regs_with_address(lrc); + err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch); + xe_lrc_put(lrc); + if (err) + break; + } + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h new file mode 100644 index 000000000..0225426c5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_EXEC_QUEUE_H_ +#define _XE_EXEC_QUEUE_H_ + +#include "xe_exec_queue_types.h" +#include "xe_vm_types.h" + +struct drm_device; +struct drm_file; +struct xe_device; +struct xe_file; + +#define for_each_tlb_inval(__i) \ + for (__i = XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT; \ + __i <= XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT; ++__i) + +struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, + u32 logical_mask, u16 width, + struct xe_hw_engine *hw_engine, u32 flags, + u64 extensions); +struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, + enum xe_engine_class class, + u32 flags, u64 extensions); +struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, + struct xe_tile *tile, + struct xe_vm *user_vm, + u32 flags, u64 extensions); + +void xe_exec_queue_fini(struct xe_exec_queue *q); +void xe_exec_queue_destroy(struct kref *ref); +void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance); + +static inline struct xe_exec_queue * +xe_exec_queue_get_unless_zero(struct xe_exec_queue *q) +{ + if (kref_get_unless_zero(&q->refcount)) + return q; + + return NULL; +} + +struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id); + +static inline struct xe_exec_queue *xe_exec_queue_get(struct xe_exec_queue *q) +{ + kref_get(&q->refcount); + return q; +} + +static inline void xe_exec_queue_put(struct xe_exec_queue *q) +{ + kref_put(&q->refcount, xe_exec_queue_destroy); +} + +static inline bool xe_exec_queue_is_parallel(struct xe_exec_queue *q) +{ + return q->width > 1; +} + +static inline bool xe_exec_queue_uses_pxp(struct xe_exec_queue *q) +{ + return q->pxp.type; +} + +/** + * xe_exec_queue_is_multi_queue() - Whether an exec_queue is part of a queue group. + * @q: The exec_queue + * + * Return: True if the exec_queue is part of a queue group, false otherwise. + */ +static inline bool xe_exec_queue_is_multi_queue(struct xe_exec_queue *q) +{ + return q->multi_queue.valid; +} + +/** + * xe_exec_queue_is_multi_queue_primary() - Whether an exec_queue is primary queue + * of a multi queue group. + * @q: The exec_queue + * + * Return: True if @q is primary queue of a queue group, false otherwise. + */ +static inline bool xe_exec_queue_is_multi_queue_primary(struct xe_exec_queue *q) +{ + return q->multi_queue.is_primary; +} + +/** + * xe_exec_queue_is_multi_queue_secondary() - Whether an exec_queue is secondary queue + * of a multi queue group. + * @q: The exec_queue + * + * Return: True if @q is secondary queue of a queue group, false otherwise. + */ +static inline bool xe_exec_queue_is_multi_queue_secondary(struct xe_exec_queue *q) +{ + return xe_exec_queue_is_multi_queue(q) && !xe_exec_queue_is_multi_queue_primary(q); +} + +/** + * xe_exec_queue_multi_queue_primary() - Get multi queue group's primary queue + * @q: The exec_queue + * + * If @q belongs to a multi queue group, then the primary queue of the group will + * be returned. Otherwise, @q will be returned. + */ +static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_exec_queue *q) +{ + return xe_exec_queue_is_multi_queue(q) ? q->multi_queue.group->primary : q; +} + +bool xe_exec_queue_is_lr(struct xe_exec_queue *q); + +bool xe_exec_queue_is_idle(struct xe_exec_queue *q); + +void xe_exec_queue_kill(struct xe_exec_queue *q); + +int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe); + +void xe_exec_queue_last_fence_put(struct xe_exec_queue *e, struct xe_vm *vm); +void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *e); +struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *e, + struct xe_vm *vm); +struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *e, + struct xe_vm *vm); +void xe_exec_queue_last_fence_set(struct xe_exec_queue *e, struct xe_vm *vm, + struct dma_fence *fence); + +void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, + struct xe_vm *vm, + unsigned int type); + +void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, + unsigned int type); + +struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q, + struct xe_vm *vm, + unsigned int type); + +void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q, + struct xe_vm *vm, + struct dma_fence *fence, + unsigned int type); + +void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q); + +int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); + +struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q); +struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx); + +#endif diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h new file mode 100644 index 000000000..53b6c0bf4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -0,0 +1,350 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_EXEC_QUEUE_TYPES_H_ +#define _XE_EXEC_QUEUE_TYPES_H_ + +#include + +#include + +#include "xe_gpu_scheduler_types.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence_types.h" +#include "xe_lrc_types.h" + +struct drm_syncobj; +struct xe_execlist_exec_queue; +struct xe_gt; +struct xe_guc_exec_queue; +struct xe_hw_engine; +struct xe_vm; + +enum xe_exec_queue_priority { + XE_EXEC_QUEUE_PRIORITY_UNSET = -2, /* For execlist usage only */ + XE_EXEC_QUEUE_PRIORITY_LOW = 0, + XE_EXEC_QUEUE_PRIORITY_NORMAL, + XE_EXEC_QUEUE_PRIORITY_HIGH, + XE_EXEC_QUEUE_PRIORITY_KERNEL, + + XE_EXEC_QUEUE_PRIORITY_COUNT +}; + +/** + * enum xe_multi_queue_priority - Multi Queue priority values + * + * The priority values of the queues within the multi queue group. + */ +enum xe_multi_queue_priority { + /** @XE_MULTI_QUEUE_PRIORITY_LOW: Priority low */ + XE_MULTI_QUEUE_PRIORITY_LOW = 0, + /** @XE_MULTI_QUEUE_PRIORITY_NORMAL: Priority normal */ + XE_MULTI_QUEUE_PRIORITY_NORMAL, + /** @XE_MULTI_QUEUE_PRIORITY_HIGH: Priority high */ + XE_MULTI_QUEUE_PRIORITY_HIGH, +}; + +/** + * struct xe_exec_queue_group - Execution multi queue group + * + * Contains multi queue group information. + */ +struct xe_exec_queue_group { + /** @primary: Primary queue of this group */ + struct xe_exec_queue *primary; + /** @cgp_bo: BO for the Context Group Page */ + struct xe_bo *cgp_bo; + /** @xa: xarray to store LRCs */ + struct xarray xa; + /** @list: List of all secondary queues in the group */ + struct list_head list; + /** @list_lock: Secondary queue list lock */ + struct mutex list_lock; + /** + * @suspend_lock: Makes a secondary's suspend/resume and its forwarding + * to the primary atomic. Nested outside of the queue's message lock + * (@xe_guc_exec_queue.sched.msg_lock). + */ + spinlock_t suspend_lock; + /** @sync_pending: CGP_SYNC_DONE g2h response pending */ + bool sync_pending; + /** @banned: Group banned */ + bool banned; + /** @stopped: Group is stopped, protected by list_lock */ + bool stopped; +}; + +/** + * struct xe_exec_queue - Execution queue + * + * Contains all state necessary for submissions. Can either be a user object or + * a kernel object. + */ +struct xe_exec_queue { + /** @xef: Back pointer to xe file if this is user created exec queue */ + struct xe_file *xef; + + /** @gt: GT structure this exec queue can submit to */ + struct xe_gt *gt; + /** + * @hwe: A hardware of the same class. May (physical engine) or may not + * (virtual engine) be where jobs actual engine up running. Should never + * really be used for submissions. + */ + struct xe_hw_engine *hwe; + /** @refcount: ref count of this exec queue */ + struct kref refcount; + /** @vm: VM (address space) for this exec queue */ + struct xe_vm *vm; + /** + * @user_vm: User VM (address space) for this exec queue (bind queues + * only) + */ + struct xe_vm *user_vm; + + /** @class: class of this exec queue */ + enum xe_engine_class class; + /** + * @logical_mask: logical mask of where job submitted to exec queue can run + */ + u32 logical_mask; + /** @name: name of this exec queue */ + char name[MAX_FENCE_NAME_LEN]; + /** @width: width (number BB submitted per exec) of this exec queue */ + u16 width; + /** @msix_vec: MSI-X vector (for platforms that support it) */ + u16 msix_vec; + /** @fence_irq: fence IRQ used to signal job completion */ + struct xe_hw_fence_irq *fence_irq; + + /** + * @last_fence: last fence on exec queue, protected by vm->lock in write + * mode if bind exec queue, protected by dma resv lock if non-bind exec + * queue + */ + struct dma_fence *last_fence; + +/* queue used for kernel submission only */ +#define EXEC_QUEUE_FLAG_KERNEL BIT(0) +/* kernel engine only destroyed at driver unload */ +#define EXEC_QUEUE_FLAG_PERMANENT BIT(1) +/* for VM jobs. Caller needs to hold rpm ref when creating queue with this flag */ +#define EXEC_QUEUE_FLAG_VM BIT(2) +/* child of VM queue for multi-tile VM jobs */ +#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(3) +/* kernel exec_queue only, set priority to highest level */ +#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(4) +/* flag to indicate low latency hint to guc */ +#define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(5) +/* for migration (kernel copy, clear, bind) jobs */ +#define EXEC_QUEUE_FLAG_MIGRATE BIT(6) +/* for programming COMMON_SLICE_CHICKEN3 on first submission */ +#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX BIT(7) + + /** + * @flags: flags for this exec queue, should statically setup aside from ban + * bit + */ + unsigned long flags; + + union { + /** @multi_gt_list: list head for VM bind engines if multi-GT */ + struct list_head multi_gt_list; + /** @multi_gt_link: link for VM bind engines if multi-GT */ + struct list_head multi_gt_link; + }; + + union { + /** @execlist: execlist backend specific state for exec queue */ + struct xe_execlist_exec_queue *execlist; + /** @guc: GuC backend specific state for exec queue */ + struct xe_guc_exec_queue *guc; + }; + + /** @multi_queue: Multi queue information */ + struct { + /** @multi_queue.group: Queue group information */ + struct xe_exec_queue_group *group; + /** @multi_queue.link: Link into group's secondary queues list */ + struct list_head link; + /** + * @multi_queue.priority: Queue priority within the multi-queue group. + * It is protected by @multi_queue.lock. + */ + enum xe_multi_queue_priority priority; + /** @multi_queue.lock: Lock for protecting certain members */ + spinlock_t lock; + /** @multi_queue.pos: Position of queue within the multi-queue group */ + u8 pos; + /** @multi_queue.valid: Queue belongs to a multi queue group */ + u8 valid:1; + /** @multi_queue.is_primary: Is primary queue (Q0) of the group */ + u8 is_primary:1; + } multi_queue; + + /** @sched_props: scheduling properties */ + struct { + /** @sched_props.timeslice_us: timeslice period in micro-seconds */ + u32 timeslice_us; + /** @sched_props.preempt_timeout_us: preemption timeout in micro-seconds */ + u32 preempt_timeout_us; + /** @sched_props.job_timeout_ms: job timeout in milliseconds */ + u32 job_timeout_ms; + /** @sched_props.priority: priority of this exec queue */ + enum xe_exec_queue_priority priority; + } sched_props; + + /** @lr: long-running exec queue state */ + struct { + /** @lr.pfence: preemption fence */ + struct dma_fence *pfence; + /** @lr.context: preemption fence context */ + u64 context; + /** @lr.seqno: preemption fence seqno */ + u32 seqno; + /** @lr.link: link into VM's list of exec queues */ + struct list_head link; + /** + * @lr.suspended: Tracks whether the consumer-issued suspend() + * succeeded and a matching resume() is still owed. suspend() can + * fail (e.g. killed/banned/wedged), leaving the queue + * un-suspended, so consumers must only resume() queues that were + * actually suspended. Set by the suspend caller on success and + * cleared by the resume caller. A queue is only ever suspended by + * a single consumer at a time (preempt-fence mode and hw engine + * group fault mode are mutually exclusive), so a single flag is + * sufficient. + */ + bool suspended; + } lr; + +#define XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT 0 +#define XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT 1 +#define XE_EXEC_QUEUE_TLB_INVAL_COUNT (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1) + + /** @tlb_inval: TLB invalidations exec queue state */ + struct { + /** + * @tlb_inval.dep_scheduler: The TLB invalidation + * dependency scheduler + */ + struct xe_dep_scheduler *dep_scheduler; + /** + * @tlb_inval.last_fence: last fence for tlb invalidation, protected by + * vm->lock in write mode + */ + struct dma_fence *last_fence; + } tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_COUNT]; + + /** @vm_exec_queue_link: Link to track exec queue within a VM's list of exec queues. */ + struct list_head vm_exec_queue_link; + + /** @pxp: PXP info tracking */ + struct { + /** @pxp.type: PXP session type used by this queue */ + u8 type; + /** @pxp.link: link into the list of PXP exec queues */ + struct list_head link; + } pxp; + + /** @ufence_syncobj: User fence syncobj */ + struct drm_syncobj *ufence_syncobj; + + /** @ufence_timeline_value: User fence timeline value */ + u64 ufence_timeline_value; + + /** @replay_state: GPU hang replay state */ + void *replay_state; + + /** @ops: submission backend exec queue operations */ + const struct xe_exec_queue_ops *ops; + + /** @ring_ops: ring operations for this exec queue */ + const struct xe_ring_ops *ring_ops; + /** @entity: DRM sched entity for this exec queue (1 to 1 relationship) */ + struct drm_sched_entity *entity; + +#define XE_MAX_JOB_COUNT_PER_EXEC_QUEUE 1000 + /** @job_cnt: number of drm jobs in this exec queue */ + atomic_t job_cnt; + + /** + * @tlb_flush_seqno: The seqno of the last rebind tlb flush performed + * Protected by @vm's resv. Unused if @vm == NULL. + */ + u64 tlb_flush_seqno; + /** @hw_engine_group_link: link into exec queues in the same hw engine group */ + struct list_head hw_engine_group_link; + /** + * @lrc_lookup_lock: Lock for protecting lrc array access. Only used when + * running in parallel to queue creation is possible. + */ + spinlock_t lrc_lookup_lock; + /** @lrc: logical ring context for this exec queue */ + struct xe_lrc *lrc[] __counted_by(width); +}; + +/** + * struct xe_exec_queue_ops - Submission backend exec queue operations + */ +struct xe_exec_queue_ops { + /** @init: Initialize exec queue for submission backend */ + int (*init)(struct xe_exec_queue *q); + /** @kill: Kill inflight submissions for backend */ + void (*kill)(struct xe_exec_queue *q); + /** @fini: Undoes the init() for submission backend */ + void (*fini)(struct xe_exec_queue *q); + /** + * @destroy: Destroy exec queue for submission backend. The backend + * function must call xe_exec_queue_fini() (which will in turn call the + * fini() backend function) to ensure the queue is properly cleaned up. + */ + void (*destroy)(struct xe_exec_queue *q); + /** @set_priority: Set priority for exec queue */ + int (*set_priority)(struct xe_exec_queue *q, + enum xe_exec_queue_priority priority); + /** @set_timeslice: Set timeslice for exec queue */ + int (*set_timeslice)(struct xe_exec_queue *q, u32 timeslice_us); + /** @set_preempt_timeout: Set preemption timeout for exec queue */ + int (*set_preempt_timeout)(struct xe_exec_queue *q, u32 preempt_timeout_us); + /** @set_multi_queue_priority: Set multi queue priority */ + int (*set_multi_queue_priority)(struct xe_exec_queue *q, + enum xe_multi_queue_priority priority); + /** + * @suspend: Suspend exec queue from executing, allowed to be called + * multiple times in a row before resume with the caveat that + * suspend_wait returns before calling suspend again. + */ + int (*suspend)(struct xe_exec_queue *q); + /** + * @suspend_wait: Wait for an exec queue to suspend executing, should be + * call after suspend. In dma-fencing path thus must return within a + * reasonable amount of time. -ETIME return shall indicate an error + * waiting for suspend resulting in associated VM getting killed. + * -EAGAIN return indicates the wait should be tried again, if the wait + * is within a work item, the work item should be requeued as deadlock + * avoidance mechanism. + */ + int (*suspend_wait)(struct xe_exec_queue *q); + /** + * @suspend_wait_blocking: Like @suspend_wait, but waits uninterruptibly + * (does not abort on the calling task's signals). For cleanup/undo paths + * that must complete a suspend on behalf of a queue that may belong to a + * different process than the caller: a signal to the caller must not + * abandon the wait, which would leave the other process's queue + * suspended forever (cross-process DoS). A timeout bans like suspend_wait. + */ + int (*suspend_wait_blocking)(struct xe_exec_queue *q); + /** + * @resume: Resume exec queue execution, exec queue must be in a suspended + * state and dma fence returned from most recent suspend call must be + * signalled when this function is called. + */ + void (*resume)(struct xe_exec_queue *q); + /** @reset_status: check exec queue reset status */ + bool (*reset_status)(struct xe_exec_queue *q); +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c new file mode 100644 index 000000000..0d0db66c6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_execlist.h" + +#include + +#include "instructions/xe_mi_commands.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_lrc_layout.h" +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_gt_types.h" +#include "xe_irq.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_mocs.h" +#include "xe_ring_ops_types.h" +#include "xe_sched_job.h" + +#define XE_EXECLIST_HANG_LIMIT 1 + +#define SW_CTX_ID_SHIFT 37 +#define SW_CTX_ID_WIDTH 11 +#define XEHP_SW_CTX_ID_SHIFT 39 +#define XEHP_SW_CTX_ID_WIDTH 16 + +#define SW_CTX_ID \ + GENMASK_ULL(SW_CTX_ID_WIDTH + SW_CTX_ID_SHIFT - 1, \ + SW_CTX_ID_SHIFT) + +#define XEHP_SW_CTX_ID \ + GENMASK_ULL(XEHP_SW_CTX_ID_WIDTH + XEHP_SW_CTX_ID_SHIFT - 1, \ + XEHP_SW_CTX_ID_SHIFT) + + +static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, + u32 ctx_id) +{ + struct xe_gt *gt = hwe->gt; + struct xe_mmio *mmio = >->mmio; + struct xe_device *xe = gt_to_xe(gt); + u64 lrc_desc; + + lrc_desc = xe_lrc_descriptor(lrc); + + if (GRAPHICS_VERx100(xe) >= 1250) { + xe_gt_assert(hwe->gt, FIELD_FIT(XEHP_SW_CTX_ID, ctx_id)); + lrc_desc |= FIELD_PREP(XEHP_SW_CTX_ID, ctx_id); + } else { + xe_gt_assert(hwe->gt, FIELD_FIT(SW_CTX_ID, ctx_id)); + lrc_desc |= FIELD_PREP(SW_CTX_ID, ctx_id); + } + + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); + lrc->ring.old_tail = lrc->ring.tail; + + /* + * Make sure the context image is complete before we submit it to HW. + * + * Ostensibly, writes (including the WCB) should be flushed prior to + * an uncached write such as our mmio register access, the empirical + * evidence (esp. on Braswell) suggests that the WC write into memory + * may not be visible to the HW prior to the completion of the UC + * register write and that we may begin execution from the context + * before its image is complete leading to invalid PD chasing. + */ + wmb(); + + xe_mmio_write32(mmio, RING_HWS_PGA(hwe->mmio_base), + xe_bo_ggtt_addr(hwe->hwsp)); + xe_mmio_read32(mmio, RING_HWS_PGA(hwe->mmio_base)); + + xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_LO(hwe->mmio_base), + lower_32_bits(lrc_desc)); + xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_HI(hwe->mmio_base), + upper_32_bits(lrc_desc)); + xe_mmio_write32(mmio, RING_EXECLIST_CONTROL(hwe->mmio_base), + EL_CTRL_LOAD); +} + +static void __xe_execlist_port_start(struct xe_execlist_port *port, + struct xe_execlist_exec_queue *exl) +{ + struct xe_device *xe = gt_to_xe(port->hwe->gt); + int max_ctx = FIELD_MAX(SW_CTX_ID); + + if (GRAPHICS_VERx100(xe) >= 1250) + max_ctx = FIELD_MAX(XEHP_SW_CTX_ID); + + xe_execlist_port_assert_held(port); + + if (port->running_exl != exl || !exl->has_run) { + port->last_ctx_id++; + + /* 0 is reserved for the kernel context */ + if (port->last_ctx_id > max_ctx) + port->last_ctx_id = 1; + } + + __start_lrc(port->hwe, exl->q->lrc[0], port->last_ctx_id); + port->running_exl = exl; + exl->has_run = true; +} + +static void __xe_execlist_port_idle(struct xe_execlist_port *port) +{ + u32 noop[2] = { MI_NOOP, MI_NOOP }; + + xe_execlist_port_assert_held(port); + + if (!port->running_exl) + return; + + xe_lrc_write_ring(port->lrc, noop, sizeof(noop)); + __start_lrc(port->hwe, port->lrc, 0); + port->running_exl = NULL; +} + +static bool xe_execlist_is_idle(struct xe_execlist_exec_queue *exl) +{ + struct xe_lrc *lrc = exl->q->lrc[0]; + + return lrc->ring.tail == lrc->ring.old_tail; +} + +static void __xe_execlist_port_start_next_active(struct xe_execlist_port *port) +{ + struct xe_execlist_exec_queue *exl = NULL; + int i; + + xe_execlist_port_assert_held(port); + + for (i = ARRAY_SIZE(port->active) - 1; i >= 0; i--) { + while (!list_empty(&port->active[i])) { + exl = list_first_entry(&port->active[i], + struct xe_execlist_exec_queue, + active_link); + list_del(&exl->active_link); + + if (xe_execlist_is_idle(exl)) { + exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; + continue; + } + + list_add_tail(&exl->active_link, &port->active[i]); + __xe_execlist_port_start(port, exl); + return; + } + } + + __xe_execlist_port_idle(port); +} + +static u64 read_execlist_status(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + u32 hi, lo; + + lo = xe_mmio_read32(>->mmio, RING_EXECLIST_STATUS_LO(hwe->mmio_base)); + hi = xe_mmio_read32(>->mmio, RING_EXECLIST_STATUS_HI(hwe->mmio_base)); + + return lo | (u64)hi << 32; +} + +static void xe_execlist_port_irq_handler_locked(struct xe_execlist_port *port) +{ + u64 status; + + xe_execlist_port_assert_held(port); + + status = read_execlist_status(port->hwe); + if (status & BIT(7)) + return; + + __xe_execlist_port_start_next_active(port); +} + +static void xe_execlist_port_irq_handler(struct xe_hw_engine *hwe, + u16 intr_vec) +{ + struct xe_execlist_port *port = hwe->exl_port; + + spin_lock(&port->lock); + xe_execlist_port_irq_handler_locked(port); + spin_unlock(&port->lock); +} + +static void xe_execlist_port_wake_locked(struct xe_execlist_port *port, + enum xe_exec_queue_priority priority) +{ + xe_execlist_port_assert_held(port); + + if (port->running_exl && port->running_exl->active_priority >= priority) + return; + + __xe_execlist_port_start_next_active(port); +} + +static void xe_execlist_make_active(struct xe_execlist_exec_queue *exl) +{ + struct xe_execlist_port *port = exl->port; + enum xe_exec_queue_priority priority = exl->q->sched_props.priority; + + XE_WARN_ON(priority == XE_EXEC_QUEUE_PRIORITY_UNSET); + XE_WARN_ON(priority < 0); + XE_WARN_ON(priority >= ARRAY_SIZE(exl->port->active)); + + spin_lock_irq(&port->lock); + + if (exl->active_priority != priority && + exl->active_priority != XE_EXEC_QUEUE_PRIORITY_UNSET) { + /* Priority changed, move it to the right list */ + list_del(&exl->active_link); + exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; + } + + if (exl->active_priority == XE_EXEC_QUEUE_PRIORITY_UNSET) { + exl->active_priority = priority; + list_add_tail(&exl->active_link, &port->active[priority]); + } + + xe_execlist_port_wake_locked(exl->port, priority); + + spin_unlock_irq(&port->lock); +} + +static void xe_execlist_port_irq_fail_timer(struct timer_list *timer) +{ + struct xe_execlist_port *port = + container_of(timer, struct xe_execlist_port, irq_fail); + + spin_lock_irq(&port->lock); + xe_execlist_port_irq_handler_locked(port); + spin_unlock_irq(&port->lock); + + port->irq_fail.expires = jiffies + msecs_to_jiffies(1000); + add_timer(&port->irq_fail); +} + +struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, + struct xe_hw_engine *hwe) +{ + struct drm_device *drm = &xe->drm; + struct xe_execlist_port *port; + int i, err; + + port = drmm_kzalloc(drm, sizeof(*port), GFP_KERNEL); + if (!port) { + err = -ENOMEM; + goto err; + } + + port->hwe = hwe; + + port->lrc = xe_lrc_create(hwe, NULL, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0); + if (IS_ERR(port->lrc)) { + err = PTR_ERR(port->lrc); + goto err; + } + + spin_lock_init(&port->lock); + for (i = 0; i < ARRAY_SIZE(port->active); i++) + INIT_LIST_HEAD(&port->active[i]); + + port->last_ctx_id = 1; + port->running_exl = NULL; + + hwe->irq_handler = xe_execlist_port_irq_handler; + + /* TODO: Fix the interrupt code so it doesn't race like mad */ + timer_setup(&port->irq_fail, xe_execlist_port_irq_fail_timer, 0); + port->irq_fail.expires = jiffies + msecs_to_jiffies(1000); + add_timer(&port->irq_fail); + + return port; + +err: + return ERR_PTR(err); +} + +void xe_execlist_port_destroy(struct xe_execlist_port *port) +{ + timer_delete(&port->irq_fail); + + /* Prevent an interrupt while we're destroying */ + spin_lock_irq(>_to_xe(port->hwe->gt)->irq.lock); + port->hwe->irq_handler = NULL; + spin_unlock_irq(>_to_xe(port->hwe->gt)->irq.lock); + + xe_lrc_put(port->lrc); +} + +static struct dma_fence * +execlist_run_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + struct xe_exec_queue *q = job->q; + struct xe_execlist_exec_queue *exl = job->q->execlist; + + q->ring_ops->emit_job(job); + xe_execlist_make_active(exl); + + return job->fence; +} + +static void execlist_job_free(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + + xe_exec_queue_update_run_ticks(job->q); + xe_sched_job_put(job); +} + +static const struct drm_sched_backend_ops drm_sched_ops = { + .run_job = execlist_run_job, + .free_job = execlist_job_free, +}; + +static int execlist_exec_queue_init(struct xe_exec_queue *q) +{ + struct drm_gpu_scheduler *sched; + const struct drm_sched_init_args args = { + .ops = &drm_sched_ops, + .num_rqs = 1, + .credit_limit = xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, + .hang_limit = XE_SCHED_HANG_LIMIT, + .timeout = XE_SCHED_JOB_TIMEOUT, + .name = q->hwe->name, + .dev = gt_to_xe(q->gt)->drm.dev, + }; + struct xe_execlist_exec_queue *exl; + struct xe_device *xe = gt_to_xe(q->gt); + int err; + + xe_assert(xe, !xe_device_uc_enabled(xe)); + + drm_info(&xe->drm, "Enabling execlist submission (GuC submission disabled)\n"); + + exl = kzalloc_obj(*exl); + if (!exl) + return -ENOMEM; + + exl->q = q; + + err = drm_sched_init(&exl->sched, &args); + if (err) + goto err_free; + + sched = &exl->sched; + err = drm_sched_entity_init(&exl->entity, 0, &sched, 1, NULL); + if (err) + goto err_sched; + + exl->port = q->hwe->exl_port; + exl->has_run = false; + exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; + q->execlist = exl; + q->entity = &exl->entity; + + xe_exec_queue_assign_name(q, ffs(q->logical_mask) - 1); + + return 0; + +err_sched: + drm_sched_fini(&exl->sched); +err_free: + kfree(exl); + return err; +} + +static void execlist_exec_queue_fini(struct xe_exec_queue *q) +{ + struct xe_execlist_exec_queue *exl = q->execlist; + + drm_sched_entity_fini(&exl->entity); + drm_sched_fini(&exl->sched); + + kfree(exl); +} + +static void execlist_exec_queue_destroy_async(struct work_struct *w) +{ + struct xe_execlist_exec_queue *ee = + container_of(w, struct xe_execlist_exec_queue, destroy_async); + struct xe_exec_queue *q = ee->q; + struct xe_execlist_exec_queue *exl = q->execlist; + struct xe_device *xe = gt_to_xe(q->gt); + unsigned long flags; + + xe_assert(xe, !xe_device_uc_enabled(xe)); + + spin_lock_irqsave(&exl->port->lock, flags); + if (WARN_ON(exl->active_priority != XE_EXEC_QUEUE_PRIORITY_UNSET)) + list_del(&exl->active_link); + spin_unlock_irqrestore(&exl->port->lock, flags); + + xe_exec_queue_fini(q); +} + +static void execlist_exec_queue_kill(struct xe_exec_queue *q) +{ + /* NIY */ +} + +static void execlist_exec_queue_destroy(struct xe_exec_queue *q) +{ + INIT_WORK(&q->execlist->destroy_async, execlist_exec_queue_destroy_async); + queue_work(system_dfl_wq, &q->execlist->destroy_async); +} + +static int execlist_exec_queue_set_priority(struct xe_exec_queue *q, + enum xe_exec_queue_priority priority) +{ + /* NIY */ + return 0; +} + +static int execlist_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) +{ + /* NIY */ + return 0; +} + +static int execlist_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, + u32 preempt_timeout_us) +{ + /* NIY */ + return 0; +} + +static int execlist_exec_queue_suspend(struct xe_exec_queue *q) +{ + /* NIY */ + return 0; +} + +static int execlist_exec_queue_suspend_wait(struct xe_exec_queue *q) + +{ + /* NIY */ + return 0; +} + +static void execlist_exec_queue_resume(struct xe_exec_queue *q) +{ + /* NIY */ +} + +static bool execlist_exec_queue_reset_status(struct xe_exec_queue *q) +{ + /* NIY */ + return false; +} + +static const struct xe_exec_queue_ops execlist_exec_queue_ops = { + .init = execlist_exec_queue_init, + .kill = execlist_exec_queue_kill, + .fini = execlist_exec_queue_fini, + .destroy = execlist_exec_queue_destroy, + .set_priority = execlist_exec_queue_set_priority, + .set_timeslice = execlist_exec_queue_set_timeslice, + .set_preempt_timeout = execlist_exec_queue_set_preempt_timeout, + .suspend = execlist_exec_queue_suspend, + .suspend_wait = execlist_exec_queue_suspend_wait, + .suspend_wait_blocking = execlist_exec_queue_suspend_wait, + .resume = execlist_exec_queue_resume, + .reset_status = execlist_exec_queue_reset_status, +}; + +int xe_execlist_init(struct xe_gt *gt) +{ + /* GuC submission enabled, nothing to do */ + if (xe_device_uc_enabled(gt_to_xe(gt))) + return 0; + + gt->exec_queue_ops = &execlist_exec_queue_ops; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_execlist.h b/drivers/gpu/drm/xe/xe_execlist.h new file mode 100644 index 000000000..26f600ac8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_execlist.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_EXECLIST_H_ +#define _XE_EXECLIST_H_ + +#include "xe_execlist_types.h" + +struct xe_device; +struct xe_gt; + +#define xe_execlist_port_assert_held(port) lockdep_assert_held(&(port)->lock) + +int xe_execlist_init(struct xe_gt *gt); +struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, + struct xe_hw_engine *hwe); +void xe_execlist_port_destroy(struct xe_execlist_port *port); + +#endif diff --git a/drivers/gpu/drm/xe/xe_execlist_types.h b/drivers/gpu/drm/xe/xe_execlist_types.h new file mode 100644 index 000000000..92c4ba52d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_execlist_types.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_EXECLIST_TYPES_H_ +#define _XE_EXECLIST_TYPES_H_ + +#include +#include +#include + +#include "xe_exec_queue_types.h" + +struct xe_hw_engine; +struct xe_execlist_exec_queue; + +struct xe_execlist_port { + struct xe_hw_engine *hwe; + + spinlock_t lock; + + struct list_head active[XE_EXEC_QUEUE_PRIORITY_COUNT]; + + u32 last_ctx_id; + + struct xe_execlist_exec_queue *running_exl; + + struct timer_list irq_fail; + + struct xe_lrc *lrc; +}; + +struct xe_execlist_exec_queue { + struct xe_exec_queue *q; + + struct drm_gpu_scheduler sched; + + struct drm_sched_entity entity; + + struct xe_execlist_port *port; + + bool has_run; + + struct work_struct destroy_async; + + enum xe_exec_queue_priority active_priority; + struct list_head active_link; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_force_wake.c b/drivers/gpu/drm/xe/xe_force_wake.c new file mode 100644 index 000000000..197e2197b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_force_wake.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_force_wake.h" + +#include + +#include "regs/xe_gt_regs.h" +#include "regs/xe_reg_defs.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_mmio.h" +#include "xe_sriov.h" + +#define XE_FORCE_WAKE_ACK_TIMEOUT_MS 50 + +static const char *str_wake_sleep(bool wake) +{ + return wake ? "wake" : "sleep"; +} + +static void mark_domain_initialized(struct xe_force_wake *fw, + enum xe_force_wake_domain_id id) +{ + fw->initialized_domains |= BIT(id); +} + +static void init_domain(struct xe_force_wake *fw, + enum xe_force_wake_domain_id id, + struct xe_reg reg, struct xe_reg ack) +{ + struct xe_force_wake_domain *domain = &fw->domains[id]; + + domain->id = id; + domain->reg_ctl = reg; + domain->reg_ack = ack; + domain->val = FORCEWAKE_MT(FORCEWAKE_KERNEL); + domain->mask = FORCEWAKE_MT_MASK(FORCEWAKE_KERNEL); + + mark_domain_initialized(fw, id); +} + +void xe_force_wake_init_gt(struct xe_gt *gt, struct xe_force_wake *fw) +{ + struct xe_device *xe = gt_to_xe(gt); + + fw->gt = gt; + spin_lock_init(&fw->lock); + + if (xe->info.graphics_verx100 >= 1270) { + init_domain(fw, XE_FW_DOMAIN_ID_GT, + FORCEWAKE_GT, + FORCEWAKE_ACK_GT_MTL); + } else { + init_domain(fw, XE_FW_DOMAIN_ID_GT, + FORCEWAKE_GT, + FORCEWAKE_ACK_GT); + } +} + +void xe_force_wake_init_engines(struct xe_gt *gt, struct xe_force_wake *fw) +{ + int i, j; + + if (xe_gt_is_main_type(gt)) + init_domain(fw, XE_FW_DOMAIN_ID_RENDER, + FORCEWAKE_RENDER, + FORCEWAKE_ACK_RENDER); + + for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + init_domain(fw, XE_FW_DOMAIN_ID_MEDIA_VDBOX0 + j, + FORCEWAKE_MEDIA_VDBOX(j), + FORCEWAKE_ACK_MEDIA_VDBOX(j)); + } + + for (i = XE_HW_ENGINE_VECS0, j = 0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + init_domain(fw, XE_FW_DOMAIN_ID_MEDIA_VEBOX0 + j, + FORCEWAKE_MEDIA_VEBOX(j), + FORCEWAKE_ACK_MEDIA_VEBOX(j)); + } + + if (gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)) + init_domain(fw, XE_FW_DOMAIN_ID_GSC, + FORCEWAKE_GSC, + FORCEWAKE_ACK_GSC); +} + +static void __domain_ctl(struct xe_gt *gt, struct xe_force_wake_domain *domain, bool wake) +{ + if (IS_SRIOV_VF(gt_to_xe(gt))) + return; + + xe_mmio_write32(>->mmio, domain->reg_ctl, domain->mask | (wake ? domain->val : 0)); +} + +static int __domain_wait(struct xe_gt *gt, struct xe_force_wake_domain *domain, bool wake) +{ + u32 value; + int ret; + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return 0; + + ret = xe_mmio_wait32(>->mmio, domain->reg_ack, domain->val, wake ? domain->val : 0, + XE_FORCE_WAKE_ACK_TIMEOUT_MS * USEC_PER_MSEC, + &value, true); + if (ret) + xe_gt_err(gt, "Force wake domain %d failed to ack %s (%pe) reg[%#x] = %#x\n", + domain->id, str_wake_sleep(wake), ERR_PTR(ret), + domain->reg_ack.addr, value); + if (value == ~0) { + xe_gt_err(gt, + "Force wake domain %d: %s. MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n", + domain->id, str_wake_sleep(wake)); + ret = -EIO; + } + + return ret; +} + +static void domain_wake(struct xe_gt *gt, struct xe_force_wake_domain *domain) +{ + __domain_ctl(gt, domain, true); +} + +static int domain_wake_wait(struct xe_gt *gt, + struct xe_force_wake_domain *domain) +{ + return __domain_wait(gt, domain, true); +} + +static void domain_sleep(struct xe_gt *gt, struct xe_force_wake_domain *domain) +{ + __domain_ctl(gt, domain, false); +} + +static int domain_sleep_wait(struct xe_gt *gt, + struct xe_force_wake_domain *domain) +{ + return __domain_wait(gt, domain, false); +} + +/** + * xe_force_wake_get() : Increase the domain refcount + * @fw: struct xe_force_wake + * @domains: forcewake domains to get refcount on + * + * This function wakes up @domains if they are asleep and takes references. + * If requested domain is XE_FORCEWAKE_ALL then only applicable/initialized + * domains will be considered for refcount and it is a caller responsibility + * to check returned ref if it includes any specific domain by using + * xe_force_wake_ref_has_domain() function. Caller must call + * xe_force_wake_put() function to decrease incremented refcounts. + * + * When possible, scope-based forcewake (through CLASS(xe_force_wake, ...) or + * xe_with_force_wake()) should be used instead of direct calls to this + * function. Direct usage of get/put should only be used when the function + * has goto-based flows that can interfere with scope-based cleanup, or when + * the lifetime of the forcewake reference does not match a specific scope + * (e.g., forcewake obtained in one function and released in a different one). + * + * Return: opaque reference to woken domains or zero if none of requested + * domains were awake. + */ +unsigned int __must_check xe_force_wake_get(struct xe_force_wake *fw, + enum xe_force_wake_domains domains) +{ + struct xe_gt *gt = fw->gt; + struct xe_force_wake_domain *domain; + unsigned int ref_incr = 0, awake_rqst = 0, awake_failed = 0; + unsigned int tmp, ref_rqst; + unsigned long flags; + + xe_gt_assert(gt, is_power_of_2(domains)); + xe_gt_assert(gt, domains <= XE_FORCEWAKE_ALL); + xe_gt_assert(gt, domains == XE_FORCEWAKE_ALL || fw->initialized_domains & domains); + + ref_rqst = (domains == XE_FORCEWAKE_ALL) ? fw->initialized_domains : domains; + spin_lock_irqsave(&fw->lock, flags); + for_each_fw_domain_masked(domain, ref_rqst, fw, tmp) { + if (!domain->ref++) { + awake_rqst |= BIT(domain->id); + domain_wake(gt, domain); + } + ref_incr |= BIT(domain->id); + } + for_each_fw_domain_masked(domain, awake_rqst, fw, tmp) { + if (domain_wake_wait(gt, domain) == 0) { + fw->awake_domains |= BIT(domain->id); + } else { + awake_failed |= BIT(domain->id); + --domain->ref; + } + } + ref_incr &= ~awake_failed; + spin_unlock_irqrestore(&fw->lock, flags); + + xe_gt_WARN(gt, awake_failed, "Forcewake domain%s %#x failed to acknowledge awake request\n", + str_plural(hweight_long(awake_failed)), awake_failed); + + if (domains == XE_FORCEWAKE_ALL && ref_incr == fw->initialized_domains) + ref_incr |= XE_FORCEWAKE_ALL; + + return ref_incr; +} + +/** + * xe_force_wake_put - Decrement the refcount and put domain to sleep if refcount becomes 0 + * @fw: Pointer to the force wake structure + * @fw_ref: return of xe_force_wake_get() + * + * This function reduces the reference counts for domains in fw_ref. If + * refcount for any of the specified domain reaches 0, it puts the domain to sleep + * and waits for acknowledgment for domain to sleep within 50 milisec timeout. + * Warns in case of timeout of ack from domain. + */ +void xe_force_wake_put(struct xe_force_wake *fw, unsigned int fw_ref) +{ + struct xe_gt *gt = fw->gt; + struct xe_force_wake_domain *domain; + unsigned int tmp, sleep = 0; + unsigned long flags; + int ack_fail = 0; + + /* + * Avoid unnecessary lock and unlock when the function is called + * in error path of individual domains. + */ + if (!fw_ref) + return; + + if (xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) + fw_ref = fw->initialized_domains; + + spin_lock_irqsave(&fw->lock, flags); + for_each_fw_domain_masked(domain, fw_ref, fw, tmp) { + xe_gt_assert(gt, domain->ref); + + if (!--domain->ref) { + sleep |= BIT(domain->id); + domain_sleep(gt, domain); + } + } + for_each_fw_domain_masked(domain, sleep, fw, tmp) { + if (domain_sleep_wait(gt, domain) == 0) + fw->awake_domains &= ~BIT(domain->id); + else + ack_fail |= BIT(domain->id); + } + spin_unlock_irqrestore(&fw->lock, flags); + + xe_gt_WARN(gt, ack_fail, "Forcewake domain%s %#x failed to acknowledge sleep request\n", + str_plural(hweight_long(ack_fail)), ack_fail); +} + +const char *xe_force_wake_domain_to_str(enum xe_force_wake_domain_id id) +{ + switch (id) { + case XE_FW_DOMAIN_ID_GT: + return "GT"; + case XE_FW_DOMAIN_ID_RENDER: + return "Render"; + case XE_FW_DOMAIN_ID_MEDIA: + return "Media"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX0: + return "VDBox0"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX1: + return "VDBox1"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX2: + return "VDBox2"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX3: + return "VDBox3"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX4: + return "VDBox4"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX5: + return "VDBox5"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX6: + return "VDBox6"; + case XE_FW_DOMAIN_ID_MEDIA_VDBOX7: + return "VDBox7"; + case XE_FW_DOMAIN_ID_MEDIA_VEBOX0: + return "VEBox0"; + case XE_FW_DOMAIN_ID_MEDIA_VEBOX1: + return "VEBox1"; + case XE_FW_DOMAIN_ID_MEDIA_VEBOX2: + return "VEBox2"; + case XE_FW_DOMAIN_ID_MEDIA_VEBOX3: + return "VEBox3"; + case XE_FW_DOMAIN_ID_GSC: + return "GSC"; + default: + return "Unknown"; + } +} diff --git a/drivers/gpu/drm/xe/xe_force_wake.h b/drivers/gpu/drm/xe/xe_force_wake.h new file mode 100644 index 000000000..e2721f205 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_force_wake.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_FORCE_WAKE_H_ +#define _XE_FORCE_WAKE_H_ + +#include "xe_assert.h" +#include "xe_force_wake_types.h" + +struct xe_gt; + +void xe_force_wake_init_gt(struct xe_gt *gt, + struct xe_force_wake *fw); +void xe_force_wake_init_engines(struct xe_gt *gt, + struct xe_force_wake *fw); +unsigned int __must_check xe_force_wake_get(struct xe_force_wake *fw, + enum xe_force_wake_domains domains); +void xe_force_wake_put(struct xe_force_wake *fw, unsigned int fw_ref); + +const char *xe_force_wake_domain_to_str(enum xe_force_wake_domain_id id); + +#define for_each_fw_domain_masked(domain__, mask__, fw__, tmp__) \ + for (tmp__ = (mask__); tmp__; tmp__ &= ~BIT(ffs(tmp__) - 1)) \ + for_each_if(((domain__) = ((fw__)->domains + \ + (ffs(tmp__) - 1))) && \ + (domain__)->reg_ctl.addr) + +#define for_each_fw_domain(domain__, fw__, tmp__) \ + for_each_fw_domain_masked((domain__), (fw__)->initialized_domains, (fw__), (tmp__)) + +static inline int +xe_force_wake_ref(struct xe_force_wake *fw, + enum xe_force_wake_domains domain) +{ + xe_gt_assert(fw->gt, domain != XE_FORCEWAKE_ALL); + return fw->domains[ffs(domain) - 1].ref; +} + +/** + * xe_force_wake_assert_held - asserts domain is awake + * @fw : xe_force_wake structure + * @domain: xe_force_wake_domains apart from XE_FORCEWAKE_ALL + * + * xe_force_wake_assert_held() is designed to confirm a particular + * forcewake domain's wakefulness; it doesn't verify the wakefulness of + * multiple domains. Make sure the caller doesn't input multiple + * domains(XE_FORCEWAKE_ALL) as a parameter. + */ +static inline void +xe_force_wake_assert_held(struct xe_force_wake *fw, + enum xe_force_wake_domains domain) +{ + xe_gt_assert(fw->gt, domain != XE_FORCEWAKE_ALL); + xe_gt_assert(fw->gt, fw->awake_domains & domain); +} + +/** + * xe_force_wake_ref_has_domain - verifies if the domains are in fw_ref + * @fw_ref : the force_wake reference + * @domain : forcewake domain to verify + * + * This function confirms whether the @fw_ref includes a reference to the + * specified @domain. + * + * Return: true if domain is refcounted. + */ +static inline bool +xe_force_wake_ref_has_domain(unsigned int fw_ref, enum xe_force_wake_domains domain) +{ + return fw_ref & domain; +} + +struct xe_force_wake_ref { + struct xe_force_wake *fw; + unsigned int domains; +}; + +static struct xe_force_wake_ref +xe_force_wake_constructor(struct xe_force_wake *fw, unsigned int domains) +{ + struct xe_force_wake_ref fw_ref = { .fw = fw }; + + fw_ref.domains = xe_force_wake_get(fw, domains); + + return fw_ref; +} + +DEFINE_CLASS(xe_force_wake, struct xe_force_wake_ref, + xe_force_wake_put(_T.fw, _T.domains), + xe_force_wake_constructor(fw, domains), + struct xe_force_wake *fw, unsigned int domains); + +/* + * Scoped helper for the forcewake class, using the same trick as scoped_guard() + * to bind the lifetime to the next statement/block. + */ +#define __xe_with_force_wake(ref, fw, domains, done) \ + for (CLASS(xe_force_wake, ref)(fw, domains), *(done) = NULL; \ + !(done); (done) = (void *)1) + +#define xe_with_force_wake(ref, fw, domains) \ + __xe_with_force_wake(ref, fw, domains, __UNIQUE_ID(done)) + +/* + * Used when xe_force_wake_constructor() has already been called by another + * function and the current function is responsible for releasing the forcewake + * reference in all possible cases and error paths. + */ +DEFINE_CLASS(xe_force_wake_release_only, struct xe_force_wake_ref, + if (_T.fw) xe_force_wake_put(_T.fw, _T.domains), fw_ref, + struct xe_force_wake_ref fw_ref); + +#endif diff --git a/drivers/gpu/drm/xe/xe_force_wake_types.h b/drivers/gpu/drm/xe/xe_force_wake_types.h new file mode 100644 index 000000000..14b7b86e8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_force_wake_types.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_FORCE_WAKE_TYPES_H_ +#define _XE_FORCE_WAKE_TYPES_H_ + +#include +#include + +#include "regs/xe_reg_defs.h" + +enum xe_force_wake_domain_id { + XE_FW_DOMAIN_ID_GT = 0, + XE_FW_DOMAIN_ID_RENDER, + XE_FW_DOMAIN_ID_MEDIA, + XE_FW_DOMAIN_ID_MEDIA_VDBOX0, + XE_FW_DOMAIN_ID_MEDIA_VDBOX1, + XE_FW_DOMAIN_ID_MEDIA_VDBOX2, + XE_FW_DOMAIN_ID_MEDIA_VDBOX3, + XE_FW_DOMAIN_ID_MEDIA_VDBOX4, + XE_FW_DOMAIN_ID_MEDIA_VDBOX5, + XE_FW_DOMAIN_ID_MEDIA_VDBOX6, + XE_FW_DOMAIN_ID_MEDIA_VDBOX7, + XE_FW_DOMAIN_ID_MEDIA_VEBOX0, + XE_FW_DOMAIN_ID_MEDIA_VEBOX1, + XE_FW_DOMAIN_ID_MEDIA_VEBOX2, + XE_FW_DOMAIN_ID_MEDIA_VEBOX3, + XE_FW_DOMAIN_ID_GSC, + XE_FW_DOMAIN_ID_COUNT +}; + +enum xe_force_wake_domains { + XE_FW_GT = BIT(XE_FW_DOMAIN_ID_GT), + XE_FW_RENDER = BIT(XE_FW_DOMAIN_ID_RENDER), + XE_FW_MEDIA = BIT(XE_FW_DOMAIN_ID_MEDIA), + XE_FW_MEDIA_VDBOX0 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX0), + XE_FW_MEDIA_VDBOX1 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX1), + XE_FW_MEDIA_VDBOX2 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX2), + XE_FW_MEDIA_VDBOX3 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX3), + XE_FW_MEDIA_VDBOX4 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX4), + XE_FW_MEDIA_VDBOX5 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX5), + XE_FW_MEDIA_VDBOX6 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX6), + XE_FW_MEDIA_VDBOX7 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX7), + XE_FW_MEDIA_VEBOX0 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX0), + XE_FW_MEDIA_VEBOX1 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX1), + XE_FW_MEDIA_VEBOX2 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX2), + XE_FW_MEDIA_VEBOX3 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX3), + XE_FW_GSC = BIT(XE_FW_DOMAIN_ID_GSC), + XE_FORCEWAKE_ALL = BIT(XE_FW_DOMAIN_ID_COUNT) +}; + +/** + * struct xe_force_wake_domain - Xe force wake power domain + * + * Represents an individual device-internal power domain. The driver must + * ensure the power domain is awake before accessing registers or other + * hardware functionality that is part of the power domain. Since different + * driver threads may access hardware units simultaneously, a reference count + * is used to ensure that the domain remains awake as long as any software + * is using the part of the hardware covered by the power domain. + * + * Hardware provides a register interface to allow the driver to request + * wake/sleep of power domains, although in most cases the actual action of + * powering the hardware up/down is handled by firmware (and may be subject to + * requirements and constraints outside of the driver's visibility) so the + * driver needs to wait for an acknowledgment that a wake request has been + * acted upon before accessing the parts of the hardware that reside within the + * power domain. + */ +struct xe_force_wake_domain { + /** @id: domain force wake id */ + enum xe_force_wake_domain_id id; + /** @reg_ctl: domain wake control register address */ + struct xe_reg reg_ctl; + /** @reg_ack: domain ack register address */ + struct xe_reg reg_ack; + /** @val: domain wake write value */ + u32 val; + /** @mask: domain mask */ + u32 mask; + /** @ref: domain reference */ + u32 ref; +}; + +/** + * struct xe_force_wake - Xe force wake collection + * + * Represents a collection of related power domains (struct + * xe_force_wake_domain) associated with a subunit of the device. + * + * Currently only used for GT power domains (where the term "forcewake" is used + * in the hardware documentation), although the interface could be extended to + * power wells in other parts of the hardware in the future. + */ +struct xe_force_wake { + /** @gt: back pointers to GT */ + struct xe_gt *gt; + /** @lock: protects everything force wake struct */ + spinlock_t lock; + /** @awake_domains: mask of all domains awake */ + unsigned int awake_domains; + /** @initialized_domains: mask of all initialized domains */ + unsigned int initialized_domains; + /** @domains: force wake domains */ + struct xe_force_wake_domain domains[XE_FW_DOMAIN_ID_COUNT]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gen_wa_oob.c b/drivers/gpu/drm/xe/xe_gen_wa_oob.c new file mode 100644 index 000000000..e7a50b134 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gen_wa_oob.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#define HEADER \ + "// SPDX-License-Identifier: MIT\n" \ + "\n" \ + "/*\n" \ + " * DO NOT MODIFY.\n" \ + " *\n" \ + " * This file was generated from rules: %s\n" \ + " */\n" \ + "#ifndef _GENERATED_%s_\n" \ + "#define _GENERATED_%s_\n" \ + "\n" \ + "enum {\n" + +#define FOOTER \ + "};\n" \ + "\n" \ + "#endif\n" + +static void print_usage(FILE *f, const char *progname) +{ + fprintf(f, "usage: %s \n", + progname); +} + +static void print_parse_error(const char *err_msg, const char *line, + unsigned int lineno) +{ + fprintf(stderr, "ERROR: %s\nERROR: %u: %.60s\n", + err_msg, lineno, line); +} + +static char *strip(char *line, size_t linelen) +{ + while (isspace(*(line + linelen))) + linelen--; + + line[linelen - 1] = '\0'; + + return line + strspn(line, " \f\n\r\t\v"); +} + +#define MAX_LINE_LEN 4096 +static int parse(FILE *input, FILE *csource, FILE *cheader, char *prefix) +{ + char line[MAX_LINE_LEN + 1]; + char *name, *prev_name = NULL, *rules; + unsigned int lineno = 0, idx = 0; + + while (fgets(line, sizeof(line), input)) { + size_t linelen; + bool is_continuation; + + if (line[0] == '\0' || line[0] == '#' || line[0] == '\n') { + lineno++; + continue; + } + + linelen = strlen(line); + if (linelen == MAX_LINE_LEN) { + print_parse_error("line too long", line, lineno); + return -EINVAL; + } + + is_continuation = isspace(line[0]); + name = strip(line, linelen); + + if (!is_continuation) { + name = strtok(name, " \t"); + rules = strtok(NULL, ""); + } else { + if (!prev_name) { + print_parse_error("invalid rule continuation", + line, lineno); + return -EINVAL; + } + + rules = name; + name = NULL; + } + + if (rules[0] == '\0') { + print_parse_error("invalid empty rule\n", line, lineno); + return -EINVAL; + } + + if (name) { + fprintf(cheader, "\t%s_%s = %u,\n", prefix, name, idx); + + /* Close previous entry before starting a new one */ + if (idx) + fprintf(csource, ") },\n"); + + fprintf(csource, "{ XE_RTP_NAME(\"%s\"),\n XE_RTP_RULES(%s", + name, rules); + idx++; + } else { + fprintf(csource, ", OR,\n\t%s", rules); + } + + lineno++; + if (!is_continuation) + prev_name = name; + } + + /* Close last entry */ + if (idx) + fprintf(csource, ") },\n"); + + fprintf(cheader, "\t_%s_COUNT = %u\n", prefix, idx); + + return 0; +} + +/* Avoid GNU vs POSIX basename() discrepancy, just use our own */ +static const char *xbasename(const char *s) +{ + const char *p = strrchr(s, '/'); + + return p ? p + 1 : s; +} + +static int fn_to_prefix(const char *fn, char *prefix, size_t size) +{ + size_t len; + + fn = xbasename(fn); + len = strlen(fn); + + if (len > size - 1) + return -ENAMETOOLONG; + + memcpy(prefix, fn, len + 1); + + for (char *p = prefix; *p; p++) { + switch (*p) { + case '.': + *p = '\0'; + return 0; + default: + *p = toupper(*p); + break; + } + } + + return 0; +} + +int main(int argc, const char *argv[]) +{ + enum { + ARGS_INPUT, + ARGS_CSOURCE, + ARGS_CHEADER, + _ARGS_COUNT + }; + struct { + const char *fn; + const char *mode; + FILE *f; + } args[] = { + [ARGS_INPUT] = { .fn = argv[1], .mode = "r" }, + [ARGS_CSOURCE] = { .fn = argv[2], .mode = "w" }, + [ARGS_CHEADER] = { .fn = argv[3], .mode = "w" }, + }; + int ret = 1; + char prefix[128]; + + if (argc < 3) { + fprintf(stderr, "ERROR: wrong arguments\n"); + print_usage(stderr, argv[0]); + return 1; + } + + if (fn_to_prefix(args[ARGS_CHEADER].fn, prefix, sizeof(prefix)) < 0) + return 1; + + for (int i = 0; i < _ARGS_COUNT; i++) { + args[i].f = fopen(args[i].fn, args[i].mode); + if (!args[i].f) { + fprintf(stderr, "ERROR: Can't open %s: %m\n", + args[i].fn); + goto err; + } + } + + fprintf(args[ARGS_CHEADER].f, HEADER, xbasename(args[ARGS_INPUT].fn), + prefix, prefix); + + ret = parse(args[ARGS_INPUT].f, args[ARGS_CSOURCE].f, + args[ARGS_CHEADER].f, prefix); + if (!ret) + fprintf(args[ARGS_CHEADER].f, FOOTER); + +err: + for (int i = 0; i < _ARGS_COUNT; i++) { + if (args[i].f) + fclose(args[i].f); + } + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c new file mode 100644 index 000000000..8ec238624 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -0,0 +1,1167 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_ggtt.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "regs/xe_gt_regs.h" +#include "regs/xe_gtt_defs.h" +#include "regs/xe_regs.h" +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_pat.h" +#include "xe_pm.h" +#include "xe_res_cursor.h" +#include "xe_sriov.h" +#include "xe_tile_printk.h" +#include "xe_tile_sriov_vf.h" +#include "xe_tlb_inval.h" +#include "xe_wa.h" +#include "xe_wopcm.h" + +/** + * DOC: Global Graphics Translation Table (GGTT) + * + * Xe GGTT implements the support for a Global Virtual Address space that is used + * for resources that are accessible to privileged (i.e. kernel-mode) processes, + * and not tied to a specific user-level process. For example, the Graphics + * micro-Controller (GuC) and Display Engine (if present) utilize this Global + * address space. + * + * The Global GTT (GGTT) translates from the Global virtual address to a physical + * address that can be accessed by HW. The GGTT is a flat, single-level table. + * + * Xe implements a simplified version of the GGTT specifically managing only a + * certain range of it that goes from the Write Once Protected Content Memory (WOPCM) + * Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to + * the GuC (Graphics Microcontroller) hardware limitations. The GuC address space + * is limited on both ends of the GGTT, because the GuC shim HW redirects + * accesses to those addresses to other HW areas instead of going through the + * GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size, + * while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things + * simple, instead of checking each object to see if they are accessed by GuC or + * not, we just exclude those areas from the allocator. Additionally, to simplify + * the driver load, we use the maximum WOPCM size in this logic instead of the + * programmed one, so we don't need to wait until the actual size to be + * programmed is determined (which requires FW fetch) before initializing the + * GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs + * depending on the platform) but we can live with this. Another benefit of this + * is the GuC bootrom can't access anything below the WOPCM max size so anything + * the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT + * above the WOPCM max size. Starting the GGTT allocations above the WOPCM max + * give us the correct placement for free. + */ + +#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_ONLINE BIT(1) + +/** + * struct xe_ggtt_node - A node in GGTT. + * + * This struct is allocated with xe_ggtt_insert_node(,_transform) or xe_ggtt_insert_bo(,_at). + * It will be deallocated using xe_ggtt_node_remove(). + */ +struct xe_ggtt_node { + /** @ggtt: Back pointer to xe_ggtt where this region will be inserted at */ + struct xe_ggtt *ggtt; + /** @base: A drm_mm_node */ + struct drm_mm_node base; + /** @delayed_removal_work: The work struct for the delayed removal */ + struct work_struct delayed_removal_work; + /** @invalidate_on_remove: If it needs invalidation upon removal */ + bool invalidate_on_remove; +}; + +/** + * struct xe_ggtt_pt_ops - GGTT Page table operations + * Which can vary from platform to platform. + */ +struct xe_ggtt_pt_ops { + /** @pte_encode_flags: Encode PTE flags for a given BO */ + u64 (*pte_encode_flags)(struct xe_bo *bo, u16 pat_index); + + /** @ggtt_set_pte: Directly write into GGTT's PTE */ + xe_ggtt_set_pte_fn ggtt_set_pte; + + /** @ggtt_get_pte: Directly read from GGTT's PTE */ + u64 (*ggtt_get_pte)(struct xe_ggtt *ggtt, u64 addr); +}; + +/** + * struct xe_ggtt - Main GGTT struct + * + * In general, each tile can contains its own Global Graphics Translation Table + * (GGTT) instance. + */ +struct xe_ggtt { + /** @tile: Back pointer to tile where this GGTT belongs */ + struct xe_tile *tile; + /** @start: Start offset of GGTT */ + u64 start; + /** @size: Total usable size of this GGTT */ + u64 size; + /** + * @flags: Flags for this GGTT. + * Acceptable flags: + * + * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K. + * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock + * after init + */ + unsigned int flags; + /** @scratch: Internal object allocation used as a scratch page */ + struct xe_bo *scratch; + /** @lock: Mutex lock to protect GGTT data */ + struct mutex lock; + /** + * @gsm: The iomem pointer to the actual location of the translation + * table located in the GSM for easy PTE manipulation + */ + u64 __iomem *gsm; + /** @pt_ops: Page Table operations per platform */ + const struct xe_ggtt_pt_ops *pt_ops; + /** @mm: The memory manager used to manage individual GGTT allocations */ + struct drm_mm mm; + /** @access_count: counts GGTT writes */ + unsigned int access_count; + /** @wq: Dedicated unordered work queue to process node removals */ + struct workqueue_struct *wq; +}; + +static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index) +{ + u64 pte = XE_PAGE_PRESENT; + + if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)) + pte |= XE_GGTT_PTE_DM; + + return pte; +} + +static u64 xelpg_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index) +{ + struct xe_device *xe = xe_bo_device(bo); + u64 pte; + + pte = xelp_ggtt_pte_flags(bo, pat_index); + + xe_assert(xe, pat_index <= 3); + + if (pat_index & BIT(0)) + pte |= XELPG_GGTT_PTE_PAT0; + + if (pat_index & BIT(1)) + pte |= XELPG_GGTT_PTE_PAT1; + + return pte; +} + +static unsigned int probe_gsm_size(struct pci_dev *pdev) +{ + u16 gmch_ctl, ggms; + + pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl); + ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK; + return ggms ? SZ_1M << ggms : 0; +} + +static void ggtt_update_access_counter(struct xe_ggtt *ggtt) +{ + struct xe_tile *tile = ggtt->tile; + struct xe_gt *affected_gt; + u32 max_gtt_writes; + + if (tile->primary_gt && XE_GT_WA(tile->primary_gt, 22019338487)) { + affected_gt = tile->primary_gt; + max_gtt_writes = 1100; + + /* Only expected to apply to primary GT on dgpu platforms */ + xe_tile_assert(tile, IS_DGFX(tile_to_xe(tile))); + } else { + affected_gt = tile->media_gt; + max_gtt_writes = 63; + + /* Only expected to apply to media GT on igpu platforms */ + xe_tile_assert(tile, !IS_DGFX(tile_to_xe(tile))); + } + + /* + * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit + * to wait for completion of prior GTT writes before letting this through. + * This needs to be done for all GGTT writes originating from the CPU. + */ + lockdep_assert_held(&ggtt->lock); + + if ((++ggtt->access_count % max_gtt_writes) == 0) { + xe_mmio_write32(&affected_gt->mmio, GMD_ID, 0x0); + ggtt->access_count = 0; + } +} + +/** + * xe_ggtt_start - Get starting offset of GGTT. + * @ggtt: &xe_ggtt + * + * Returns: Starting offset for this &xe_ggtt. + */ +u64 xe_ggtt_start(struct xe_ggtt *ggtt) +{ + return ggtt->start; +} + +/** + * xe_ggtt_size - Get size of GGTT. + * @ggtt: &xe_ggtt + * + * Returns: Total usable size of this &xe_ggtt. + */ +u64 xe_ggtt_size(struct xe_ggtt *ggtt) +{ + return ggtt->size; +} + +static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte) +{ + xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK)); + xe_tile_assert(ggtt->tile, addr < ggtt->start + ggtt->size); + + writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]); +} + +static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte) +{ + xe_ggtt_set_pte(ggtt, addr, pte); + ggtt_update_access_counter(ggtt); +} + +static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr) +{ + xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK)); + xe_tile_assert(ggtt->tile, addr < ggtt->start + ggtt->size); + + return readq(&ggtt->gsm[addr >> XE_PTE_SHIFT]); +} + +static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size) +{ + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), XE_CACHE_WB); + u64 end = start + size - 1; + u64 scratch_pte; + + xe_tile_assert(ggtt->tile, start < end); + + if (ggtt->scratch) + scratch_pte = xe_bo_addr(ggtt->scratch, 0, XE_PAGE_SIZE) | + ggtt->pt_ops->pte_encode_flags(ggtt->scratch, + pat_index); + else + scratch_pte = 0; + + while (start < end) { + ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte); + start += XE_PAGE_SIZE; + } +} + +static void primelockdep(struct xe_ggtt *ggtt) +{ + if (!IS_ENABLED(CONFIG_LOCKDEP)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&ggtt->lock); + fs_reclaim_release(GFP_KERNEL); +} + +/** + * xe_ggtt_alloc - Allocate a GGTT for a given &xe_tile + * @tile: &xe_tile + * + * Allocates a &xe_ggtt for a given tile. + * + * Return: &xe_ggtt on success, or NULL when out of memory. + */ +struct xe_ggtt *xe_ggtt_alloc(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_ggtt *ggtt; + + ggtt = drmm_kzalloc(&xe->drm, sizeof(*ggtt), GFP_KERNEL); + if (!ggtt) + return NULL; + + if (drmm_mutex_init(&xe->drm, &ggtt->lock)) + return NULL; + + primelockdep(ggtt); + ggtt->tile = tile; + + return ggtt; +} + +static void ggtt_fini_early(struct drm_device *drm, void *arg) +{ + struct xe_ggtt *ggtt = arg; + + destroy_workqueue(ggtt->wq); + drm_mm_takedown(&ggtt->mm); +} + +static void ggtt_fini(void *arg) +{ + struct xe_ggtt *ggtt = arg; + + ggtt->scratch = NULL; +} + +#ifdef CONFIG_LOCKDEP +void xe_ggtt_might_lock(struct xe_ggtt *ggtt) +{ + might_lock(&ggtt->lock); +} +#endif + +static const struct xe_ggtt_pt_ops xelp_pt_ops = { + .pte_encode_flags = xelp_ggtt_pte_flags, + .ggtt_set_pte = xe_ggtt_set_pte, + .ggtt_get_pte = xe_ggtt_get_pte, +}; + +static const struct xe_ggtt_pt_ops xelpg_pt_ops = { + .pte_encode_flags = xelpg_ggtt_pte_flags, + .ggtt_set_pte = xe_ggtt_set_pte, + .ggtt_get_pte = xe_ggtt_get_pte, +}; + +static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = { + .pte_encode_flags = xelpg_ggtt_pte_flags, + .ggtt_set_pte = xe_ggtt_set_pte_and_flush, + .ggtt_get_pte = xe_ggtt_get_pte, +}; + +static void __xe_ggtt_init_early(struct xe_ggtt *ggtt, u64 start, u64 size) +{ + ggtt->start = start; + ggtt->size = size; + drm_mm_init(&ggtt->mm, 0, size); +} + +int xe_ggtt_init_kunit(struct xe_ggtt *ggtt, u32 start, u32 size) +{ + __xe_ggtt_init_early(ggtt, start, size); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(xe_ggtt_init_kunit); + +static void dev_fini_ggtt(void *arg) +{ + struct xe_ggtt *ggtt = arg; + + scoped_guard(mutex, &ggtt->lock) + ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE; + drain_workqueue(ggtt->wq); +} + +/** + * xe_ggtt_init_early - Early GGTT initialization + * @ggtt: the &xe_ggtt to be initialized + * + * It allows to create new mappings usable by the GuC. + * Mappings are not usable by the HW engines, as it doesn't have scratch nor + * initial clear done to it yet. That will happen in the regular, non-early + * GGTT initialization. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_init_early(struct xe_ggtt *ggtt) +{ + struct xe_device *xe = tile_to_xe(ggtt->tile); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + unsigned int gsm_size; + u64 ggtt_start, wopcm = xe_wopcm_size(xe), ggtt_size; + int err; + + if (!IS_SRIOV_VF(xe)) { + if (GRAPHICS_VERx100(xe) >= 1250) + gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */ + else + gsm_size = probe_gsm_size(pdev); + if (gsm_size == 0) { + xe_tile_err(ggtt->tile, "Hardware reported no preallocated GSM\n"); + return -ENOMEM; + } + ggtt_start = wopcm; + ggtt_size = (gsm_size / 8) * (u64)XE_PAGE_SIZE - ggtt_start; + } else { + ggtt_start = xe_tile_sriov_vf_ggtt_base(ggtt->tile); + ggtt_size = xe_tile_sriov_vf_ggtt(ggtt->tile); + + if (ggtt_start < wopcm || + ggtt_start + ggtt_size > GUC_GGTT_TOP) { + xe_tile_err(ggtt->tile, "Invalid GGTT configuration: %#llx-%#llx\n", + ggtt_start, ggtt_start + ggtt_size - 1); + return -ERANGE; + } + } + + ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M; + if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + ggtt->flags |= XE_GGTT_FLAGS_64K; + + if (ggtt_size + ggtt_start > GUC_GGTT_TOP) + ggtt_size = GUC_GGTT_TOP - ggtt_start; + + if (GRAPHICS_VERx100(xe) >= 1270) + ggtt->pt_ops = + (ggtt->tile->media_gt && XE_GT_WA(ggtt->tile->media_gt, 22019338487)) || + (ggtt->tile->primary_gt && XE_GT_WA(ggtt->tile->primary_gt, 22019338487)) ? + &xelpg_pt_wa_ops : &xelpg_pt_ops; + else + ggtt->pt_ops = &xelp_pt_ops; + + ggtt->wq = alloc_workqueue("xe-ggtt-wq", WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!ggtt->wq) + return -ENOMEM; + + __xe_ggtt_init_early(ggtt, ggtt_start, ggtt_size); + + err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt); + if (err) + return err; + + ggtt->flags |= XE_GGTT_FLAGS_ONLINE; + return devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt); +} +ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */ + +static void xe_ggtt_invalidate(struct xe_ggtt *ggtt); + +static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) +{ + struct drm_mm_node *hole; + u64 start, end; + + /* Display may have allocated inside ggtt, so be careful with clearing here */ + mutex_lock(&ggtt->lock); + drm_mm_for_each_hole(hole, &ggtt->mm, start, end) + xe_ggtt_clear(ggtt, ggtt->start + start, end - start); + + xe_ggtt_invalidate(ggtt); + mutex_unlock(&ggtt->lock); +} + +static void ggtt_node_fini(struct xe_ggtt_node *node) +{ + kfree(node); +} + +static void ggtt_node_remove(struct xe_ggtt_node *node) +{ + struct xe_ggtt *ggtt = node->ggtt; + bool bound; + + mutex_lock(&ggtt->lock); + bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE; + if (bound) + xe_ggtt_clear(ggtt, xe_ggtt_node_addr(node), xe_ggtt_node_size(node)); + drm_mm_remove_node(&node->base); + node->base.size = 0; + if (bound && node->invalidate_on_remove) + xe_ggtt_invalidate(ggtt); + mutex_unlock(&ggtt->lock); + + ggtt_node_fini(node); +} + +static void ggtt_node_remove_work_func(struct work_struct *work) +{ + struct xe_ggtt_node *node = container_of(work, typeof(*node), + delayed_removal_work); + struct xe_device *xe = tile_to_xe(node->ggtt->tile); + + guard(xe_pm_runtime)(xe); + ggtt_node_remove(node); +} + +/** + * xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT + * @node: the &xe_ggtt_node to be removed + * @invalidate: if node needs invalidation upon removal + */ +void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate) +{ + struct xe_ggtt *ggtt; + struct xe_device *xe; + + if (!node || !node->ggtt) + return; + + ggtt = node->ggtt; + xe = tile_to_xe(ggtt->tile); + + node->invalidate_on_remove = invalidate; + + if (xe_pm_runtime_get_if_active(xe)) { + ggtt_node_remove(node); + xe_pm_runtime_put(xe); + } else { + queue_work(ggtt->wq, &node->delayed_removal_work); + } +} + +/** + * xe_ggtt_init - Regular non-early GGTT initialization + * @ggtt: the &xe_ggtt to be initialized + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_init(struct xe_ggtt *ggtt) +{ + struct xe_device *xe = tile_to_xe(ggtt->tile); + unsigned int flags; + int err; + + /* + * So we don't need to worry about 64K GGTT layout when dealing with + * scratch entries, rather keep the scratch page in system memory on + * platforms where 64K pages are needed for VRAM. + */ + flags = 0; + if (ggtt->flags & XE_GGTT_FLAGS_64K) + flags |= XE_BO_FLAG_SYSTEM; + else + flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile); + + ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags); + if (IS_ERR(ggtt->scratch)) { + err = PTR_ERR(ggtt->scratch); + goto err; + } + + xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, xe_bo_size(ggtt->scratch)); + + xe_ggtt_initial_clear(ggtt); + + return devm_add_action_or_reset(xe->drm.dev, ggtt_fini, ggtt); +err: + ggtt->scratch = NULL; + return err; +} + +static void ggtt_invalidate_gt_tlb(struct xe_gt *gt) +{ + int err; + + if (!gt) + return; + + err = xe_tlb_inval_ggtt(>->tlb_inval); + xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err)); +} + +static void xe_ggtt_invalidate(struct xe_ggtt *ggtt) +{ + struct xe_device *xe = tile_to_xe(ggtt->tile); + + /* + * XXX: Barrier for GGTT pages. Unsure exactly why this required but + * without this LNL is having issues with the GuC reading scratch page + * vs. correct GGTT page. Not particularly a hot code path so blindly + * do a mmio read here which results in GuC reading correct GGTT page. + */ + xe_mmio_read32(xe_root_tile_mmio(xe), VF_CAP_REG); + + /* Each GT in a tile has its own TLB to cache GGTT lookups */ + ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt); + ggtt_invalidate_gt_tlb(ggtt->tile->media_gt); +} + +/** + * xe_ggtt_shift_nodes() - Shift GGTT nodes to adjust for a change in usable address range. + * @ggtt: the &xe_ggtt struct instance + * @new_start: new location of area provisioned for current VF + * + * Ensure that all struct &xe_ggtt_node are moved to the @new_start base address + * by changing the base offset of the GGTT. + * + * This function may be called multiple times during recovery, but if + * @new_start is unchanged from the current base, it's a noop. + * + * @new_start should be a value between xe_wopcm_size() and #GUC_GGTT_TOP. + */ +void xe_ggtt_shift_nodes(struct xe_ggtt *ggtt, u64 new_start) +{ + guard(mutex)(&ggtt->lock); + + xe_tile_assert(ggtt->tile, new_start >= xe_wopcm_size(tile_to_xe(ggtt->tile))); + xe_tile_assert(ggtt->tile, new_start + ggtt->size <= GUC_GGTT_TOP); + + /* pairs with READ_ONCE in xe_ggtt_node_addr() */ + WRITE_ONCE(ggtt->start, new_start); +} + +static int xe_ggtt_insert_node_locked(struct xe_ggtt_node *node, + u32 size, u32 align, u32 mm_flags) +{ + return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0, + mm_flags); +} + +static struct xe_ggtt_node *ggtt_node_init(struct xe_ggtt *ggtt) +{ + struct xe_ggtt_node *node = kzalloc_obj(*node, GFP_NOFS); + + if (!node) + return ERR_PTR(-ENOMEM); + + INIT_WORK(&node->delayed_removal_work, ggtt_node_remove_work_func); + node->ggtt = ggtt; + + return node; +} + +/** + * xe_ggtt_insert_node - Insert a &xe_ggtt_node into the GGTT + * @ggtt: the &xe_ggtt into which the node should be inserted. + * @size: size of the node + * @align: alignment constrain of the node + * + * Return: &xe_ggtt_node on success or a ERR_PTR on failure. + */ +struct xe_ggtt_node *xe_ggtt_insert_node(struct xe_ggtt *ggtt, u32 size, u32 align) +{ + struct xe_ggtt_node *node; + int ret; + + node = ggtt_node_init(ggtt); + if (IS_ERR(node)) + return node; + + guard(mutex)(&ggtt->lock); + ret = xe_ggtt_insert_node_locked(node, size, align, + DRM_MM_INSERT_HIGH); + if (ret) { + ggtt_node_fini(node); + return ERR_PTR(ret); + } + + return node; +} + +/** + * xe_ggtt_node_pt_size() - Get the size of page table entries needed to map a GGTT node. + * @node: the &xe_ggtt_node + * + * Return: GGTT node page table entries size in bytes. + */ +size_t xe_ggtt_node_pt_size(const struct xe_ggtt_node *node) +{ + if (!node) + return 0; + + return node->base.size / XE_PAGE_SIZE * sizeof(u64); +} + +/** + * xe_ggtt_map_bo - Map the BO into GGTT + * @ggtt: the &xe_ggtt where node will be mapped + * @node: the &xe_ggtt_node where this BO is mapped + * @bo: the &xe_bo to be mapped + * @pte: The pte flags to append. + */ +static void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, + struct xe_bo *bo, u64 pte) +{ + u64 start, end; + struct xe_res_cursor cur; + + if (XE_WARN_ON(!node)) + return; + + start = xe_ggtt_node_addr(node); + end = start + xe_bo_size(bo); + + if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) { + xe_assert(xe_bo_device(bo), bo->ttm.ttm); + + for (xe_res_first_sg(xe_bo_sg(bo), 0, xe_bo_size(bo), &cur); + cur.remaining; xe_res_next(&cur, XE_PAGE_SIZE)) + ggtt->pt_ops->ggtt_set_pte(ggtt, end - cur.remaining, + pte | xe_res_dma(&cur)); + } else { + /* Prepend GPU offset */ + pte |= vram_region_gpu_offset(bo->ttm.resource); + + for (xe_res_first(bo->ttm.resource, 0, xe_bo_size(bo), &cur); + cur.remaining; xe_res_next(&cur, XE_PAGE_SIZE)) + ggtt->pt_ops->ggtt_set_pte(ggtt, end - cur.remaining, + pte + cur.start); + } +} + +/** + * xe_ggtt_map_bo_unlocked - Restore a mapping of a BO into GGTT + * @ggtt: the &xe_ggtt where node will be mapped + * @bo: the &xe_bo to be mapped + * + * This is used to restore a GGTT mapping after suspend. + */ +void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo) +{ + u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode); + u64 pte; + + mutex_lock(&ggtt->lock); + pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index); + xe_ggtt_map_bo(ggtt, bo->ggtt_node[ggtt->tile->id], bo, pte); + mutex_unlock(&ggtt->lock); +} + +/** + * xe_ggtt_insert_node_transform - Insert a newly allocated &xe_ggtt_node into the GGTT + * @ggtt: the &xe_ggtt where the node will inserted/reserved. + * @bo: The bo to be transformed + * @pte_flags: The extra GGTT flags to add to mapping. + * @size: size of the node + * @align: required alignment for node + * @transform: transformation function that will populate the GGTT node, or NULL for linear mapping. + * @arg: Extra argument to pass to the transformation function. + * + * This function allows inserting a GGTT node with a custom transformation function. + * This is useful for display to allow inserting rotated framebuffers to GGTT. + * + * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise. + */ +struct xe_ggtt_node *xe_ggtt_insert_node_transform(struct xe_ggtt *ggtt, + struct xe_bo *bo, u64 pte_flags, + u64 size, u32 align, + xe_ggtt_transform_cb transform, void *arg) +{ + struct xe_ggtt_node *node; + int ret; + + node = ggtt_node_init(ggtt); + if (IS_ERR(node)) + return ERR_CAST(node); + + if (mutex_lock_interruptible(&ggtt->lock) < 0) { + ret = -ERESTARTSYS; + goto err; + } + + ret = xe_ggtt_insert_node_locked(node, size, align, 0); + if (ret) + goto err_unlock; + + if (transform) + transform(ggtt, node, pte_flags, ggtt->pt_ops->ggtt_set_pte, arg); + else + xe_ggtt_map_bo(ggtt, node, bo, pte_flags); + + mutex_unlock(&ggtt->lock); + return node; + +err_unlock: + mutex_unlock(&ggtt->lock); +err: + ggtt_node_fini(node); + return ERR_PTR(ret); +} + +static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, + u64 start, u64 end, struct drm_exec *exec) +{ + u64 alignment = bo->min_align > 0 ? bo->min_align : XE_PAGE_SIZE; + u8 tile_id = ggtt->tile->id; + int err; + + if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) + alignment = SZ_64K; + + if (XE_WARN_ON(bo->ggtt_node[tile_id])) { + /* Someone's already inserted this BO in the GGTT */ + xe_tile_assert(ggtt->tile, bo->ggtt_node[tile_id]->base.size == xe_bo_size(bo)); + return 0; + } + + err = xe_bo_validate(bo, NULL, false, exec); + if (err) + return err; + + xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile)); + + bo->ggtt_node[tile_id] = ggtt_node_init(ggtt); + if (IS_ERR(bo->ggtt_node[tile_id])) { + err = PTR_ERR(bo->ggtt_node[tile_id]); + bo->ggtt_node[tile_id] = NULL; + goto out; + } + + mutex_lock(&ggtt->lock); + /* + * When inheriting the initial framebuffer, the framebuffer is + * physically located at VRAM address 0, and usually at GGTT address 0 too. + * + * The display code will ask for a GGTT allocation between end of BO and + * remainder of GGTT, unaware that the start is reserved by WOPCM. + */ + if (start >= ggtt->start) + start -= ggtt->start; + else + start = 0; + + /* Should never happen, but since we handle start, fail graciously for end */ + if (end >= ggtt->start) + end -= ggtt->start; + else + end = 0; + + xe_tile_assert(ggtt->tile, end >= start + xe_bo_size(bo)); + + err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node[tile_id]->base, + xe_bo_size(bo), alignment, 0, start, end, 0); + if (err) { + ggtt_node_fini(bo->ggtt_node[tile_id]); + bo->ggtt_node[tile_id] = NULL; + } else { + u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode); + u64 pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index); + + xe_ggtt_map_bo(ggtt, bo->ggtt_node[tile_id], bo, pte); + } + mutex_unlock(&ggtt->lock); + + if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE) + xe_ggtt_invalidate(ggtt); + +out: + xe_pm_runtime_put(tile_to_xe(ggtt->tile)); + + return err; +} + +/** + * xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space + * @ggtt: the &xe_ggtt where bo will be inserted + * @bo: the &xe_bo to be inserted + * @start: address where it will be inserted + * @end: end of the range where it will be inserted + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, + u64 start, u64 end, struct drm_exec *exec) +{ + return __xe_ggtt_insert_bo_at(ggtt, bo, start, end, exec); +} + +/** + * xe_ggtt_insert_bo - Insert BO into GGTT + * @ggtt: the &xe_ggtt where bo will be inserted + * @bo: the &xe_bo to be inserted + * @exec: The drm_exec transaction to use for exhaustive eviction. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo, + struct drm_exec *exec) +{ + return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX, exec); +} + +/** + * xe_ggtt_remove_bo - Remove a BO from the GGTT + * @ggtt: the &xe_ggtt where node will be removed + * @bo: the &xe_bo to be removed + */ +void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) +{ + u8 tile_id = ggtt->tile->id; + + if (XE_WARN_ON(!bo->ggtt_node[tile_id])) + return; + + /* This BO is not currently in the GGTT */ + xe_tile_assert(ggtt->tile, bo->ggtt_node[tile_id]->base.size == xe_bo_size(bo)); + + xe_ggtt_node_remove(bo->ggtt_node[tile_id], + bo->flags & XE_BO_FLAG_GGTT_INVALIDATE); +} + +/** + * xe_ggtt_largest_hole - Largest GGTT hole + * @ggtt: the &xe_ggtt that will be inspected + * @alignment: minimum alignment + * @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare + * + * Return: size of the largest continuous GGTT region + */ +u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare) +{ + const struct drm_mm *mm = &ggtt->mm; + const struct drm_mm_node *entry; + u64 hole_start, hole_end, hole_size; + u64 max_hole = 0; + + mutex_lock(&ggtt->lock); + drm_mm_for_each_hole(entry, mm, hole_start, hole_end) { + hole_start = max(hole_start, ggtt->start); + hole_start = ALIGN(hole_start, alignment); + hole_end = ALIGN_DOWN(hole_end, alignment); + if (hole_start >= hole_end) + continue; + hole_size = hole_end - hole_start; + if (spare) + *spare -= min3(*spare, hole_size, max_hole); + max_hole = max(max_hole, hole_size); + } + + mutex_unlock(&ggtt->lock); + + return max_hole; +} + +#ifdef CONFIG_PCI_IOV +static u64 xe_encode_vfid_pte(u16 vfid) +{ + return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT; +} + +static void xe_ggtt_assign_locked(const struct xe_ggtt_node *node, u16 vfid) +{ + struct xe_ggtt *ggtt = node->ggtt; + u64 start = xe_ggtt_node_addr(node); + u64 size = xe_ggtt_node_size(node); + u64 end = start + size - 1; + u64 pte = xe_encode_vfid_pte(vfid); + + lockdep_assert_held(&ggtt->lock); + + while (start < end) { + ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte); + start += XE_PAGE_SIZE; + } + + xe_ggtt_invalidate(ggtt); +} + +/** + * xe_ggtt_assign - assign a GGTT region to the VF + * @node: the &xe_ggtt_node to update + * @vfid: the VF identifier + * + * This function is used by the PF driver to assign a GGTT region to the VF. + * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some + * platforms VFs can't modify that either. + */ +void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid) +{ + guard(mutex)(&node->ggtt->lock); + xe_ggtt_assign_locked(node, vfid); +} + +/** + * xe_ggtt_node_save() - Save a &xe_ggtt_node to a buffer. + * @node: the &xe_ggtt_node to be saved + * @dst: destination buffer + * @size: destination buffer size in bytes + * @vfid: VF identifier + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_node_save(struct xe_ggtt_node *node, void *dst, size_t size, u16 vfid) +{ + struct xe_ggtt *ggtt; + u64 start, end; + u64 *buf = dst; + u64 pte; + + if (!node) + return -ENOENT; + + ggtt = node->ggtt; + guard(mutex)(&ggtt->lock); + + if (xe_ggtt_node_pt_size(node) != size) + return -EINVAL; + + start = xe_ggtt_node_addr(node); + end = start + xe_ggtt_node_size(node) - 1; + + while (start < end) { + pte = ggtt->pt_ops->ggtt_get_pte(ggtt, start); + if (vfid != u64_get_bits(pte, GGTT_PTE_VFID)) + return -EPERM; + + *buf++ = u64_replace_bits(pte, 0, GGTT_PTE_VFID); + start += XE_PAGE_SIZE; + } + + return 0; +} + +/** + * xe_ggtt_node_load() - Load a &xe_ggtt_node from a buffer. + * @node: the &xe_ggtt_node to be loaded + * @src: source buffer + * @size: source buffer size in bytes + * @vfid: VF identifier + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_node_load(struct xe_ggtt_node *node, const void *src, size_t size, u16 vfid) +{ + u64 vfid_pte = xe_encode_vfid_pte(vfid); + const u64 *buf = src; + struct xe_ggtt *ggtt; + u64 start, end; + + if (!node) + return -ENOENT; + + ggtt = node->ggtt; + guard(mutex)(&ggtt->lock); + + if (xe_ggtt_node_pt_size(node) != size) + return -EINVAL; + + start = xe_ggtt_node_addr(node); + end = start + xe_ggtt_node_size(node) - 1; + + while (start < end) { + vfid_pte = u64_replace_bits(*buf++, vfid, GGTT_PTE_VFID); + ggtt->pt_ops->ggtt_set_pte(ggtt, start, vfid_pte); + start += XE_PAGE_SIZE; + } + xe_ggtt_invalidate(ggtt); + + return 0; +} + +#endif + +/** + * xe_ggtt_dump - Dump GGTT for debug + * @ggtt: the &xe_ggtt to be dumped + * @p: the &drm_mm_printer helper handle to be used to dump the information + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p) +{ + int err; + + err = mutex_lock_interruptible(&ggtt->lock); + if (err) + return err; + + drm_mm_print(&ggtt->mm, p); + mutex_unlock(&ggtt->lock); + return err; +} + +/** + * xe_ggtt_print_holes - Print holes + * @ggtt: the &xe_ggtt to be inspected + * @alignment: min alignment + * @p: the &drm_printer + * + * Print GGTT ranges that are available and return total size available. + * + * Return: Total available size. + */ +u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p) +{ + const struct drm_mm *mm = &ggtt->mm; + const struct drm_mm_node *entry; + u64 hole_start, hole_end, hole_size; + u64 total = 0; + char buf[10]; + + mutex_lock(&ggtt->lock); + drm_mm_for_each_hole(entry, mm, hole_start, hole_end) { + hole_start = max(hole_start, ggtt->start); + hole_start = ALIGN(hole_start, alignment); + hole_end = ALIGN_DOWN(hole_end, alignment); + if (hole_start >= hole_end) + continue; + hole_size = hole_end - hole_start; + total += hole_size; + + string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf)); + drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n", + hole_start, hole_end - 1, buf); + } + + mutex_unlock(&ggtt->lock); + + return total; +} + +/** + * xe_ggtt_encode_pte_flags - Get PTE encoding flags for BO + * @ggtt: &xe_ggtt + * @bo: &xe_bo + * @pat_index: The pat_index for the PTE. + * + * This function returns the pte_flags for a given BO, without address. + * It's used for DPT to fill a GGTT mapped BO with a linear lookup table. + */ +u64 xe_ggtt_encode_pte_flags(struct xe_ggtt *ggtt, + struct xe_bo *bo, u16 pat_index) +{ + return ggtt->pt_ops->pte_encode_flags(bo, pat_index); +} + +/** + * xe_ggtt_read_pte - Read a PTE from the GGTT + * @ggtt: &xe_ggtt + * @offset: the offset for which the mapping should be read. + * + * Used by testcases, and by display reading out an inherited bios FB. + */ +u64 xe_ggtt_read_pte(struct xe_ggtt *ggtt, u64 offset) +{ + return ioread64(ggtt->gsm + (offset / XE_PAGE_SIZE)); +} + +/** + * xe_ggtt_node_addr - Get @node offset in GGTT. + * @node: &xe_ggtt_node + * + * Get the GGTT offset for allocated node. + */ +u64 xe_ggtt_node_addr(const struct xe_ggtt_node *node) +{ + /* pairs with WRITE_ONCE in xe_ggtt_shift_nodes() */ + return node->base.start + READ_ONCE(node->ggtt->start); +} + +/** + * xe_ggtt_node_size - Get @node allocation size. + * @node: &xe_ggtt_node + * + * Get the allocated node's size. + */ +u64 xe_ggtt_node_size(const struct xe_ggtt_node *node) +{ + return node->base.size; +} diff --git a/drivers/gpu/drm/xe/xe_ggtt.h b/drivers/gpu/drm/xe/xe_ggtt.h new file mode 100644 index 000000000..c864cc975 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ggtt.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_GGTT_H_ +#define _XE_GGTT_H_ + +#include "xe_ggtt_types.h" + +struct drm_printer; +struct xe_bo; +struct xe_tile; +struct drm_exec; + +struct xe_ggtt *xe_ggtt_alloc(struct xe_tile *tile); +int xe_ggtt_init_early(struct xe_ggtt *ggtt); +int xe_ggtt_init_kunit(struct xe_ggtt *ggtt, u32 reserved, u32 size); +int xe_ggtt_init(struct xe_ggtt *ggtt); + +void xe_ggtt_shift_nodes(struct xe_ggtt *ggtt, u64 new_base); +u64 xe_ggtt_start(struct xe_ggtt *ggtt); +u64 xe_ggtt_size(struct xe_ggtt *ggtt); + +struct xe_ggtt_node * +xe_ggtt_insert_node(struct xe_ggtt *ggtt, u32 size, u32 align); +struct xe_ggtt_node * +xe_ggtt_insert_node_transform(struct xe_ggtt *ggtt, + struct xe_bo *bo, u64 pte, + u64 size, u32 align, + xe_ggtt_transform_cb transform, void *arg); +void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate); +size_t xe_ggtt_node_pt_size(const struct xe_ggtt_node *node); +void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo); +int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo, struct drm_exec *exec); +int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, + u64 start, u64 end, struct drm_exec *exec); +void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo); +u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare); + +int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p); +u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p); + +#ifdef CONFIG_PCI_IOV +void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid); +int xe_ggtt_node_save(struct xe_ggtt_node *node, void *dst, size_t size, u16 vfid); +int xe_ggtt_node_load(struct xe_ggtt_node *node, const void *src, size_t size, u16 vfid); +#endif + +#ifndef CONFIG_LOCKDEP +static inline void xe_ggtt_might_lock(struct xe_ggtt *ggtt) +{ } +#else +void xe_ggtt_might_lock(struct xe_ggtt *ggtt); +#endif + +u64 xe_ggtt_encode_pte_flags(struct xe_ggtt *ggtt, struct xe_bo *bo, u16 pat_index); +u64 xe_ggtt_read_pte(struct xe_ggtt *ggtt, u64 offset); + +u64 xe_ggtt_node_addr(const struct xe_ggtt_node *node); +u64 xe_ggtt_node_size(const struct xe_ggtt_node *node); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ggtt_types.h b/drivers/gpu/drm/xe/xe_ggtt_types.h new file mode 100644 index 000000000..cf754e4d5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ggtt_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GGTT_TYPES_H_ +#define _XE_GGTT_TYPES_H_ + +#include +#include + +struct xe_ggtt; +struct xe_ggtt_node; + +typedef void (*xe_ggtt_set_pte_fn)(struct xe_ggtt *ggtt, u64 addr, u64 pte); +typedef void (*xe_ggtt_transform_cb)(struct xe_ggtt *ggtt, + struct xe_ggtt_node *node, + u64 pte_flags, + xe_ggtt_set_pte_fn set_pte, void *arg); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c new file mode 100644 index 000000000..9c8004d5d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gpu_scheduler.h" + +static void xe_sched_process_msg_queue(struct xe_gpu_scheduler *sched) +{ + if (!drm_sched_is_stopped(&sched->base)) + queue_work(sched->base.submit_wq, &sched->work_process_msg); +} + +static void xe_sched_process_msg_queue_if_ready(struct xe_gpu_scheduler *sched) +{ + struct xe_sched_msg *msg; + + xe_sched_msg_lock(sched); + msg = list_first_entry_or_null(&sched->msgs, struct xe_sched_msg, link); + if (msg) + xe_sched_process_msg_queue(sched); + xe_sched_msg_unlock(sched); +} + +static struct xe_sched_msg * +xe_sched_get_msg(struct xe_gpu_scheduler *sched) +{ + struct xe_sched_msg *msg; + + xe_sched_msg_lock(sched); + msg = list_first_entry_or_null(&sched->msgs, + struct xe_sched_msg, link); + if (msg) + list_del_init(&msg->link); + xe_sched_msg_unlock(sched); + + return msg; +} + +static void xe_sched_process_msg_work(struct work_struct *w) +{ + struct xe_gpu_scheduler *sched = + container_of(w, struct xe_gpu_scheduler, work_process_msg); + struct xe_sched_msg *msg; + + if (drm_sched_is_stopped(&sched->base)) + return; + + msg = xe_sched_get_msg(sched); + if (msg) { + sched->ops->process_msg(msg); + + xe_sched_process_msg_queue_if_ready(sched); + } +} + +int xe_sched_init(struct xe_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, + const struct xe_sched_backend_ops *xe_ops, + struct workqueue_struct *submit_wq, + uint32_t hw_submission, unsigned hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, + struct device *dev) +{ + const struct drm_sched_init_args args = { + .ops = ops, + .submit_wq = submit_wq, + .num_rqs = 1, + .credit_limit = hw_submission, + .hang_limit = hang_limit, + .timeout = timeout, + .timeout_wq = timeout_wq, + .score = score, + .name = name, + .dev = dev, + }; + + sched->ops = xe_ops; + spin_lock_init(&sched->msg_lock); + INIT_LIST_HEAD(&sched->msgs); + INIT_WORK(&sched->work_process_msg, xe_sched_process_msg_work); + + return drm_sched_init(&sched->base, &args); +} + +void xe_sched_fini(struct xe_gpu_scheduler *sched) +{ + xe_sched_submission_stop(sched); + drm_sched_fini(&sched->base); +} + +void xe_sched_submission_start(struct xe_gpu_scheduler *sched) +{ + drm_sched_wqueue_start(&sched->base); + queue_work(sched->base.submit_wq, &sched->work_process_msg); +} + +void xe_sched_submission_stop(struct xe_gpu_scheduler *sched) +{ + drm_sched_wqueue_stop(&sched->base); + cancel_work_sync(&sched->work_process_msg); +} + +void xe_sched_submission_resume_tdr(struct xe_gpu_scheduler *sched) +{ + drm_sched_resume_timeout(&sched->base, sched->base.timeout); +} + +void xe_sched_add_msg(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg) +{ + xe_sched_msg_lock(sched); + xe_sched_add_msg_locked(sched, msg); + xe_sched_msg_unlock(sched); +} + +void xe_sched_add_msg_locked(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg) +{ + lockdep_assert_held(&sched->msg_lock); + + list_add_tail(&msg->link, &sched->msgs); + xe_sched_process_msg_queue(sched); +} + +/** + * xe_sched_add_msg_head() - Xe GPU scheduler add message to head of list + * @sched: Xe GPU scheduler + * @msg: Message to add + */ +void xe_sched_add_msg_head(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg) +{ + lockdep_assert_held(&sched->msg_lock); + + list_add(&msg->link, &sched->msgs); + xe_sched_process_msg_queue(sched); +} diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.h b/drivers/gpu/drm/xe/xe_gpu_scheduler.h new file mode 100644 index 000000000..664c2db56 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GPU_SCHEDULER_H_ +#define _XE_GPU_SCHEDULER_H_ + +#include "xe_gpu_scheduler_types.h" +#include "xe_sched_job.h" + +int xe_sched_init(struct xe_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, + const struct xe_sched_backend_ops *xe_ops, + struct workqueue_struct *submit_wq, + uint32_t hw_submission, unsigned hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, + struct device *dev); +void xe_sched_fini(struct xe_gpu_scheduler *sched); + +void xe_sched_submission_start(struct xe_gpu_scheduler *sched); +void xe_sched_submission_stop(struct xe_gpu_scheduler *sched); + +void xe_sched_submission_resume_tdr(struct xe_gpu_scheduler *sched); + +void xe_sched_add_msg(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg); +void xe_sched_add_msg_locked(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg); +void xe_sched_add_msg_head(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg); + +static inline void xe_sched_msg_lock(struct xe_gpu_scheduler *sched) +{ + spin_lock(&sched->msg_lock); +} + +static inline void xe_sched_msg_unlock(struct xe_gpu_scheduler *sched) +{ + spin_unlock(&sched->msg_lock); +} + +static inline void xe_sched_stop(struct xe_gpu_scheduler *sched) +{ + drm_sched_stop(&sched->base, NULL); +} + +static inline void xe_sched_tdr_queue_imm(struct xe_gpu_scheduler *sched) +{ + drm_sched_tdr_queue_imm(&sched->base); +} + +static inline void xe_sched_resubmit_jobs(struct xe_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job; + bool restore_replay = false; + + drm_sched_for_each_pending_job(s_job, &sched->base, NULL) { + restore_replay |= to_xe_sched_job(s_job)->restore_replay; + if (restore_replay || !drm_sched_job_is_signaled(s_job)) + sched->base.ops->run_job(s_job); + } +} + +static inline bool +xe_sched_invalidate_job(struct xe_sched_job *job, int threshold) +{ + return drm_sched_invalidate_job(&job->drm, threshold); +} + +/** + * xe_sched_first_pending_job() - Find first pending job which is unsignaled + * @sched: Xe GPU scheduler + * + * Return first unsignaled job in pending list or NULL + */ +static inline +struct xe_sched_job *xe_sched_first_pending_job(struct xe_gpu_scheduler *sched) +{ + struct drm_sched_job *job; + + drm_sched_for_each_pending_job(job, &sched->base, NULL) + if (!drm_sched_job_is_signaled(job)) + return to_xe_sched_job(job); + + return NULL; +} + +static inline int +xe_sched_entity_init(struct xe_sched_entity *entity, + struct xe_gpu_scheduler *sched) +{ + return drm_sched_entity_init(entity, 0, + (struct drm_gpu_scheduler **)&sched, + 1, NULL); +} + +#define xe_sched_entity_fini drm_sched_entity_fini + +#endif diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler_types.h b/drivers/gpu/drm/xe/xe_gpu_scheduler_types.h new file mode 100644 index 000000000..63d9bf925 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler_types.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GPU_SCHEDULER_TYPES_H_ +#define _XE_GPU_SCHEDULER_TYPES_H_ + +#include + +/** + * struct xe_sched_msg - an in-band (relative to GPU scheduler run queue) + * message + * + * Generic enough for backend defined messages, backend can expand if needed. + */ +struct xe_sched_msg { + /** @link: list link into the gpu scheduler list of messages */ + struct list_head link; + /** + * @private_data: opaque pointer to message private data (backend defined) + */ + void *private_data; + /** @opcode: opcode of message (backend defined) */ + unsigned int opcode; +}; + +/** + * struct xe_sched_backend_ops - Define the backend operations called by the + * scheduler + */ +struct xe_sched_backend_ops { + /** + * @process_msg: Process a message. Allowed to block, it is this + * function's responsibility to free message if dynamically allocated. + */ + void (*process_msg)(struct xe_sched_msg *msg); +}; + +/** + * struct xe_gpu_scheduler - Xe GPU scheduler + */ +struct xe_gpu_scheduler { + /** @base: DRM GPU scheduler */ + struct drm_gpu_scheduler base; + /** @ops: Xe scheduler ops */ + const struct xe_sched_backend_ops *ops; + /** @msgs: list of messages to be processed in @work_process_msg */ + struct list_head msgs; + /** @msg_lock: Message lock */ + spinlock_t msg_lock; + /** @work_process_msg: processes messages */ + struct work_struct work_process_msg; +}; + +#define xe_sched_entity drm_sched_entity +#define xe_sched_policy drm_sched_policy + +#endif diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c new file mode 100644 index 000000000..aab59dc64 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gsc.h" + +#include + +#include +#include + +#include + +#include "abi/gsc_mkhi_commands_abi.h" +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_gsc_proxy.h" +#include "xe_gsc_submit.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_guc_pc.h" +#include "xe_huc.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_sched_job.h" +#include "xe_uc_fw.h" +#include "xe_wa.h" +#include "instructions/xe_gsc_commands.h" +#include "regs/xe_gsc_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_irq_regs.h" + +static struct xe_gt * +gsc_to_gt(struct xe_gsc *gsc) +{ + return container_of(gsc, struct xe_gt, uc.gsc); +} + +static int memcpy_fw(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + u32 fw_size = gsc->fw.size; + void *storage; + + /* + * FIXME: xe_migrate_copy does not work with stolen mem yet, so we use + * a memcpy for now. + */ + storage = kmalloc(fw_size, GFP_KERNEL); + if (!storage) + return -ENOMEM; + + xe_map_memcpy_from(xe, storage, &gsc->fw.bo->vmap, 0, fw_size); + xe_map_memcpy_to(xe, &gsc->private->vmap, 0, storage, fw_size); + xe_map_memset(xe, &gsc->private->vmap, fw_size, 0, + xe_bo_size(gsc->private) - fw_size); + + kfree(storage); + + return 0; +} + +static int emit_gsc_upload(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + u64 offset = xe_bo_ggtt_addr(gsc->private); + struct xe_bb *bb; + struct xe_sched_job *job; + struct dma_fence *fence; + long timeout; + + bb = xe_bb_new(gt, 4, false); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + bb->cs[bb->len++] = GSC_FW_LOAD; + bb->cs[bb->len++] = lower_32_bits(offset); + bb->cs[bb->len++] = upper_32_bits(offset); + bb->cs[bb->len++] = (xe_bo_size(gsc->private) / SZ_4K) | + GSC_FW_LOAD_LIMIT_VALID; + + job = xe_bb_create_job(gsc->q, bb); + if (IS_ERR(job)) { + xe_bb_free(bb, NULL); + return PTR_ERR(job); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, HZ); + dma_fence_put(fence); + xe_bb_free(bb, NULL); + if (timeout < 0) + return timeout; + else if (!timeout) + return -ETIME; + + return 0; +} + +#define version_query_wr(xe_, map_, offset_, field_, val_) \ + xe_map_wr_field(xe_, map_, offset_, struct gsc_get_compatibility_version_in, field_, val_) +#define version_query_rd(xe_, map_, offset_, field_) \ + xe_map_rd_field(xe_, map_, offset_, struct gsc_get_compatibility_version_out, field_) + +static u32 emit_version_query_msg(struct xe_device *xe, struct iosys_map *map, u32 wr_offset) +{ + xe_map_memset(xe, map, wr_offset, 0, sizeof(struct gsc_get_compatibility_version_in)); + + version_query_wr(xe, map, wr_offset, header.group_id, MKHI_GROUP_ID_GFX_SRV); + version_query_wr(xe, map, wr_offset, header.command, + MKHI_GFX_SRV_GET_HOST_COMPATIBILITY_VERSION); + + return wr_offset + sizeof(struct gsc_get_compatibility_version_in); +} + +#define GSC_VER_PKT_SZ SZ_4K /* 4K each for input and output */ +static int query_compatibility_version(struct xe_gsc *gsc) +{ + struct xe_uc_fw_version *compat = &gsc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]; + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *bo; + u32 wr_offset; + u32 rd_offset; + u64 ggtt_offset; + int err; + + bo = xe_bo_create_pin_map_novm(xe, tile, GSC_VER_PKT_SZ * 2, + ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT, false); + if (IS_ERR(bo)) { + xe_gt_err(gt, "failed to allocate bo for GSC version query\n"); + return PTR_ERR(bo); + } + + ggtt_offset = xe_bo_ggtt_addr(bo); + + wr_offset = xe_gsc_emit_header(xe, &bo->vmap, 0, HECI_MEADDRESS_MKHI, 0, + sizeof(struct gsc_get_compatibility_version_in)); + wr_offset = emit_version_query_msg(xe, &bo->vmap, wr_offset); + + err = xe_gsc_pkt_submit_kernel(gsc, ggtt_offset, wr_offset, + ggtt_offset + GSC_VER_PKT_SZ, + GSC_VER_PKT_SZ); + if (err) { + xe_gt_err(gt, + "failed to submit GSC request for compatibility version: %d\n", + err); + goto out_bo; + } + + err = xe_gsc_read_out_header(xe, &bo->vmap, GSC_VER_PKT_SZ, + sizeof(struct gsc_get_compatibility_version_out), + &rd_offset); + if (err) { + xe_gt_err(gt, "HuC: invalid GSC reply for version query (err=%d)\n", err); + goto out_bo; + } + + compat->major = version_query_rd(xe, &bo->vmap, rd_offset, proj_major); + compat->minor = version_query_rd(xe, &bo->vmap, rd_offset, compat_major); + compat->patch = version_query_rd(xe, &bo->vmap, rd_offset, compat_minor); + + xe_gt_info(gt, "found GSC cv%u.%u.%u\n", compat->major, compat->minor, compat->patch); + +out_bo: + xe_bo_unpin_map_no_vm(bo); + return err; +} + +static int gsc_fw_is_loaded(struct xe_gt *gt) +{ + return xe_mmio_read32(>->mmio, HECI_FWSTS1(MTL_GSC_HECI1_BASE)) & + HECI1_FWSTS1_INIT_COMPLETE; +} + +static int gsc_fw_wait(struct xe_gt *gt) +{ + /* + * GSC load can take up to 250ms from the moment the instruction is + * executed by the GSCCS. To account for possible submission delays or + * other issues, we use a 500ms timeout in the wait here. + */ + return xe_mmio_wait32(>->mmio, HECI_FWSTS1(MTL_GSC_HECI1_BASE), + HECI1_FWSTS1_INIT_COMPLETE, + HECI1_FWSTS1_INIT_COMPLETE, + 500 * USEC_PER_MSEC, NULL, false); +} + +static int gsc_upload(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + int err; + + /* we should only be here if the init step were successful */ + xe_assert(xe, xe_uc_fw_is_loadable(&gsc->fw) && gsc->q); + + if (gsc_fw_is_loaded(gt)) { + xe_gt_err(gt, "GSC already loaded at upload time\n"); + return -EEXIST; + } + + err = memcpy_fw(gsc); + if (err) { + xe_gt_err(gt, "Failed to memcpy GSC FW\n"); + return err; + } + + /* + * GSC is only killed by an FLR, so we need to trigger one on unload to + * make sure we stop it. This is because we assign a chunk of memory to + * the GSC as part of the FW load, so we need to make sure it stops + * using it when we release it to the system on driver unload. Note that + * this is not a problem of the unload per-se, because the GSC will not + * touch that memory unless there are requests for it coming from the + * driver; therefore, no accesses will happen while Xe is not loaded, + * but if we re-load the driver then the GSC might wake up and try to + * access that old memory location again. + * Given that an FLR is a very disruptive action (see the FLR function + * for details), we want to do it as the last action before releasing + * the access to the MMIO bar, which means we need to do it as part of + * mmio cleanup. + */ + xe->needs_flr_on_fini = true; + + err = emit_gsc_upload(gsc); + if (err) { + xe_gt_err(gt, "Failed to emit GSC FW upload (%pe)\n", ERR_PTR(err)); + return err; + } + + err = gsc_fw_wait(gt); + if (err) { + xe_gt_err(gt, "Failed to wait for GSC load (%pe)\n", ERR_PTR(err)); + return err; + } + + err = query_compatibility_version(gsc); + if (err) + return err; + + err = xe_uc_fw_check_version_requirements(&gsc->fw); + if (err) + return err; + + return 0; +} + +static int gsc_upload_and_init(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_tile *tile = gt_to_tile(gt); + unsigned int fw_ref; + int ret; + + if (tile->primary_gt && XE_GT_WA(tile->primary_gt, 14018094691)) { + fw_ref = xe_force_wake_get(gt_to_fw(tile->primary_gt), XE_FORCEWAKE_ALL); + + /* + * If the forcewake fails we want to keep going, because the worst + * case outcome in failing to apply the WA is that PXP won't work, + * which is not fatal. Forcewake get warns implicitly in case of failure + */ + xe_gt_mcr_multicast_write(tile->primary_gt, + EU_SYSTOLIC_LIC_THROTTLE_CTL_WITH_LOCK, + EU_SYSTOLIC_LIC_THROTTLE_CTL_LOCK_BIT); + } + + ret = gsc_upload(gsc); + + if (tile->primary_gt && XE_GT_WA(tile->primary_gt, 14018094691)) + xe_force_wake_put(gt_to_fw(tile->primary_gt), fw_ref); + + if (ret) + return ret; + + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_TRANSFERRED); + + /* GSC load is done, restore expected GT frequencies */ + xe_gt_sanitize_freq(gt); + + xe_gt_dbg(gt, "GSC FW async load completed\n"); + + /* HuC auth failure is not fatal */ + if (xe_huc_is_authenticated(>->uc.huc, XE_HUC_AUTH_VIA_GUC)) + xe_huc_auth(>->uc.huc, XE_HUC_AUTH_VIA_GSC); + + ret = xe_gsc_proxy_start(gsc); + if (ret) + return ret; + + xe_gt_dbg(gt, "GSC proxy init completed\n"); + + return 0; +} + +static int gsc_er_complete(struct xe_gt *gt) +{ + u32 er_status; + + if (!gsc_fw_is_loaded(gt)) + return 0; + + /* + * Starting on Xe2, the GSCCS engine reset is a 2-step process. When the + * driver or the GuC hit the GDRST register, the CS is immediately reset + * and a success is reported, but the GSC shim keeps resetting in the + * background. While the shim reset is ongoing, the CS is able to accept + * new context submission, but any commands that require the shim will + * be stalled until the reset is completed. This means that we can keep + * submitting to the GSCCS as long as we make sure that the preemption + * timeout is big enough to cover any delay introduced by the reset. + * When the shim reset completes, a specific CS interrupt is triggered, + * in response to which we need to check the GSCI_TIMER_STATUS register + * to see if the reset was successful or not. + * Note that the GSCI_TIMER_STATUS register is not power save/restored, + * so it gets reset on MC6 entry. However, a reset failure stops MC6, + * so in that scenario we're always guaranteed to find the correct + * value. + */ + er_status = xe_mmio_read32(>->mmio, GSCI_TIMER_STATUS) & GSCI_TIMER_STATUS_VALUE; + + if (er_status == GSCI_TIMER_STATUS_TIMER_EXPIRED) { + /* + * XXX: we should trigger an FLR here, but we don't have support + * for that yet. Since we can't recover from the error, we + * declare the device as wedged. + */ + xe_gt_err(gt, "GSC ER timed out!\n"); + xe_device_declare_wedged(gt_to_xe(gt)); + return -EIO; + } + + return 0; +} + +static void gsc_work(struct work_struct *work) +{ + struct xe_gsc *gsc = container_of(work, typeof(*gsc), work); + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + u32 actions; + int ret; + + spin_lock_irq(&gsc->lock); + actions = gsc->work_actions; + gsc->work_actions = 0; + spin_unlock_irq(&gsc->lock); + + guard(xe_pm_runtime)(xe); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + + if (actions & GSC_ACTION_ER_COMPLETE) { + if (gsc_er_complete(gt)) + return; + } + + if (actions & GSC_ACTION_FW_LOAD) { + ret = gsc_upload_and_init(gsc); + if (ret && ret != -EEXIST) + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_LOAD_FAIL); + else + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_RUNNING); + } + + if (actions & GSC_ACTION_SW_PROXY) + xe_gsc_proxy_request_handler(gsc); +} + +void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec) +{ + struct xe_gt *gt = hwe->gt; + struct xe_gsc *gsc = >->uc.gsc; + + if (unlikely(!intr_vec)) + return; + + if (intr_vec & GSC_ER_COMPLETE) { + spin_lock(&gsc->lock); + gsc->work_actions |= GSC_ACTION_ER_COMPLETE; + spin_unlock(&gsc->lock); + + queue_work(gsc->wq, &gsc->work); + } +} + +int xe_gsc_init(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_tile *tile = gt_to_tile(gt); + int ret; + + gsc->fw.type = XE_UC_FW_TYPE_GSC; + INIT_WORK(&gsc->work, gsc_work); + spin_lock_init(&gsc->lock); + + /* The GSC uC is only available on the media GT */ + if (tile->media_gt && (gt != tile->media_gt)) { + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_NOT_SUPPORTED); + return 0; + } + + /* + * Starting from BMG the GSC is no longer needed for MC6 entry, so the + * only missing features if the FW is lacking would be the content + * protection ones. This is acceptable, so we allow the driver load to + * continue if the GSC FW is missing. + */ + ret = xe_uc_fw_init(&gsc->fw); + if (!xe_uc_fw_is_enabled(&gsc->fw)) + return 0; + else if (gt_to_xe(gt)->info.platform >= XE_BATTLEMAGE && !xe_uc_fw_is_available(&gsc->fw)) + return 0; + else if (ret) + goto out; + + ret = xe_gsc_proxy_init(gsc); + if (ret && ret != -ENODEV) + goto out; + + return 0; + +out: + xe_gt_err(gt, "GSC init failed with %d", ret); + return ret; +} + +static void free_resources(void *arg) +{ + struct xe_gsc *gsc = arg; + + if (gsc->wq) { + destroy_workqueue(gsc->wq); + gsc->wq = NULL; + } + + if (gsc->q) { + xe_exec_queue_put(gsc->q); + gsc->q = NULL; + } +} + +int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, XE_ENGINE_CLASS_OTHER, 0, true); + struct xe_exec_queue *q; + struct workqueue_struct *wq; + struct xe_bo *bo; + int err; + + if (!xe_uc_fw_is_available(&gsc->fw)) + return 0; + + if (!hwe) + return -ENODEV; + + bo = xe_managed_bo_create_pin_map(xe, tile, SZ_4M, + XE_BO_FLAG_STOLEN | + XE_BO_FLAG_GGTT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + q = xe_exec_queue_create(xe, NULL, + BIT(hwe->logical_instance), 1, hwe, + EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_PERMANENT, 0); + if (IS_ERR(q)) { + xe_gt_err(gt, "Failed to create queue for GSC submission\n"); + return PTR_ERR(q); + } + + wq = alloc_ordered_workqueue("gsc-ordered-wq", 0); + if (!wq) { + err = -ENOMEM; + goto out_q; + } + + gsc->private = bo; + gsc->q = q; + gsc->wq = wq; + + err = devm_add_action_or_reset(xe->drm.dev, free_resources, gsc); + if (err) + return err; + + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_LOADABLE); + + return 0; + +out_q: + xe_exec_queue_put(q); + return err; +} + +void xe_gsc_load_start(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + + if (!xe_uc_fw_is_loadable(&gsc->fw) || !gsc->q) + return; + + /* + * The GSC HW is only reset by driver FLR or D3cold entry. We don't + * support the former at runtime, while the latter is only supported on + * DGFX, for which we don't support GSC. Therefore, if GSC failed to + * load previously there is no need to try again because the HW is + * stuck in the error state. + */ + xe_assert(xe, !IS_DGFX(xe)); + if (xe_uc_fw_is_in_error_state(&gsc->fw)) + return; + + /* GSC FW survives GT reset and D3Hot */ + if (gsc_fw_is_loaded(gt)) { + if (xe_gsc_proxy_init_done(gsc)) + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_RUNNING); + else + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_TRANSFERRED); + return; + } + + spin_lock_irq(&gsc->lock); + gsc->work_actions |= GSC_ACTION_FW_LOAD; + spin_unlock_irq(&gsc->lock); + + queue_work(gsc->wq, &gsc->work); +} + +void xe_gsc_wait_for_worker_completion(struct xe_gsc *gsc) +{ + if (xe_uc_fw_is_loadable(&gsc->fw) && gsc->wq) + flush_work(&gsc->work); +} + +void xe_gsc_stop_prepare(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + int ret; + + if (!xe_uc_fw_is_loadable(&gsc->fw) || xe_uc_fw_is_in_error_state(&gsc->fw)) + return; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GSC); + + /* + * If the GSC FW load or the proxy init are interrupted, the only way + * to recover it is to do an FLR and reload the GSC from scratch. + * Therefore, let's wait for the init to complete before stopping + * operations. The proxy init is the last step, so we can just wait on + * that + */ + ret = xe_gsc_wait_for_proxy_init_done(gsc); + if (ret) + xe_gt_err(gt, "failed to wait for GSC init completion before uc stop\n"); +} + +/* + * wa_14015076503: if the GSC FW is loaded, we need to alert it before doing a + * GSC engine reset by writing a notification bit in the GS1 register and then + * triggering an interrupt to GSC; from the interrupt it will take up to 200ms + * for the FW to get prepare for the reset, so we need to wait for that amount + * of time. + * After the reset is complete we need to then clear the GS1 register. + */ +void xe_gsc_wa_14015076503(struct xe_gt *gt, bool prep) +{ + u32 gs1_set = prep ? HECI_H_GS1_ER_PREP : 0; + u32 gs1_clr = prep ? 0 : HECI_H_GS1_ER_PREP; + + /* WA only applies if the GSC is loaded */ + if (!XE_GT_WA(gt, 14015076503) || !gsc_fw_is_loaded(gt)) + return; + + xe_mmio_rmw32(>->mmio, HECI_H_GS1(MTL_GSC_HECI2_BASE), gs1_clr, gs1_set); + + if (prep) { + /* make sure the reset bit is clear when writing the CSR reg */ + xe_mmio_rmw32(>->mmio, HECI_H_CSR(MTL_GSC_HECI2_BASE), + HECI_H_CSR_RST, HECI_H_CSR_IG); + msleep(200); + } +} + +/** + * xe_gsc_print_info - print info about GSC FW status + * @gsc: the GSC structure + * @p: the printer to be used to print the info + */ +void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_mmio *mmio = >->mmio; + + xe_uc_fw_print(&gsc->fw, p); + + drm_printf(p, "\tfound security version %u\n", gsc->security_version); + + if (!xe_uc_fw_is_available(&gsc->fw)) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + if (!fw_ref.domains) + return; + + drm_printf(p, "\nHECI1 FWSTS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", + xe_mmio_read32(mmio, HECI_FWSTS1(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(mmio, HECI_FWSTS2(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(mmio, HECI_FWSTS3(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(mmio, HECI_FWSTS4(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(mmio, HECI_FWSTS5(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(mmio, HECI_FWSTS6(MTL_GSC_HECI1_BASE))); +} diff --git a/drivers/gpu/drm/xe/xe_gsc.h b/drivers/gpu/drm/xe/xe_gsc.h new file mode 100644 index 000000000..b8b8e0810 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GSC_H_ +#define _XE_GSC_H_ + +#include + +struct drm_printer; +struct xe_gsc; +struct xe_gt; +struct xe_hw_engine; + +int xe_gsc_init(struct xe_gsc *gsc); +int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc); +void xe_gsc_wait_for_worker_completion(struct xe_gsc *gsc); +void xe_gsc_stop_prepare(struct xe_gsc *gsc); +void xe_gsc_load_start(struct xe_gsc *gsc); +void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec); + +void xe_gsc_wa_14015076503(struct xe_gt *gt, bool prep); + +void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gsc_debugfs.c b/drivers/gpu/drm/xe/xe_gsc_debugfs.c new file mode 100644 index 000000000..d4977e666 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_debugfs.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gsc_debugfs.h" + +#include +#include +#include + +#include "xe_gt_types.h" +#include "xe_gsc.h" +#include "xe_pm.h" + +static struct xe_gt * +gsc_to_gt(struct xe_gsc *gsc) +{ + return container_of(gsc, struct xe_gt, uc.gsc); +} + +static struct xe_device * +gsc_to_xe(struct xe_gsc *gsc) +{ + return gt_to_xe(gsc_to_gt(gsc)); +} + +static struct xe_gsc *node_to_gsc(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static int gsc_info(struct seq_file *m, void *data) +{ + struct xe_gsc *gsc = node_to_gsc(m->private); + struct xe_device *xe = gsc_to_xe(gsc); + struct drm_printer p = drm_seq_file_printer(m); + + guard(xe_pm_runtime)(xe); + xe_gsc_print_info(gsc, &p); + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"gsc_info", gsc_info, 0}, +}; + +void xe_gsc_debugfs_register(struct xe_gsc *gsc, struct dentry *parent) +{ + struct drm_minor *minor = gsc_to_xe(gsc)->drm.primary; + struct drm_info_list *local; + int i; + +#define DEBUGFS_SIZE (ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list)) + local = drmm_kmalloc(&gsc_to_xe(gsc)->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) + return; + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = gsc; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + parent, minor); +} diff --git a/drivers/gpu/drm/xe/xe_gsc_debugfs.h b/drivers/gpu/drm/xe/xe_gsc_debugfs.h new file mode 100644 index 000000000..c2e2645dc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GSC_DEBUGFS_H_ +#define _XE_GSC_DEBUGFS_H_ + +struct dentry; +struct xe_gsc; + +void xe_gsc_debugfs_register(struct xe_gsc *gsc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c new file mode 100644 index 000000000..707db650a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gsc_proxy.h" + +#include +#include + +#include +#include +#include + +#include "abi/gsc_proxy_commands_abi.h" +#include "regs/xe_gsc_regs.h" +#include "xe_bo.h" +#include "xe_force_wake.h" +#include "xe_gsc.h" +#include "xe_gsc_submit.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_tile.h" + +/* + * GSC proxy: + * The GSC uC needs to communicate with the CSME to perform certain operations. + * Since the GSC can't perform this communication directly on platforms where it + * is integrated in GT, the graphics driver needs to transfer the messages from + * GSC to CSME and back. The proxy flow must be manually started after the GSC + * is loaded to signal to GSC that we're ready to handle its messages and allow + * it to query its init data from CSME; GSC will then trigger an HECI2 interrupt + * if it needs to send messages to CSME again. + * The proxy flow is as follow: + * 1 - Xe submits a request to GSC asking for the message to CSME + * 2 - GSC replies with the proxy header + payload for CSME + * 3 - Xe sends the reply from GSC as-is to CSME via the mei proxy component + * 4 - CSME replies with the proxy header + payload for GSC + * 5 - Xe submits a request to GSC with the reply from CSME + * 6 - GSC replies either with a new header + payload (same as step 2, so we + * restart from there) or with an end message. + */ + +/* + * The component should load quite quickly in most cases, but it could take + * a bit. Using a very big timeout just to cover the worst case scenario + */ +#define GSC_PROXY_INIT_TIMEOUT_MS 20000 + +/* shorthand define for code compactness */ +#define PROXY_HDR_SIZE (sizeof(struct xe_gsc_proxy_header)) + +/* the protocol supports up to 32K in each direction */ +#define GSC_PROXY_BUFFER_SIZE SZ_32K +#define GSC_PROXY_CHANNEL_SIZE (GSC_PROXY_BUFFER_SIZE * 2) + +static struct xe_gt * +gsc_to_gt(struct xe_gsc *gsc) +{ + return container_of(gsc, struct xe_gt, uc.gsc); +} + +bool xe_gsc_proxy_init_done(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + u32 fwsts1 = xe_mmio_read32(>->mmio, HECI_FWSTS1(MTL_GSC_HECI1_BASE)); + + return REG_FIELD_GET(HECI1_FWSTS1_CURRENT_STATE, fwsts1) == + HECI1_FWSTS1_PROXY_STATE_NORMAL; +} + +int xe_gsc_wait_for_proxy_init_done(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + + /* Proxy init can take up to 500ms, so wait double that for safety */ + return xe_mmio_wait32(>->mmio, HECI_FWSTS1(MTL_GSC_HECI1_BASE), + HECI1_FWSTS1_CURRENT_STATE, + HECI1_FWSTS1_PROXY_STATE_NORMAL, + USEC_PER_SEC, NULL, false); +} + +static void __gsc_proxy_irq_rmw(struct xe_gsc *gsc, u32 clr, u32 set) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + + /* make sure we never accidentally write the RST bit */ + clr |= HECI_H_CSR_RST; + + xe_mmio_rmw32(>->mmio, HECI_H_CSR(MTL_GSC_HECI2_BASE), clr, set); +} + +static void gsc_proxy_irq_clear(struct xe_gsc *gsc) +{ + /* The status bit is cleared by writing to it */ + __gsc_proxy_irq_rmw(gsc, 0, HECI_H_CSR_IS); +} + +static void gsc_proxy_irq_toggle(struct xe_gsc *gsc, bool enabled) +{ + u32 set = enabled ? HECI_H_CSR_IE : 0; + u32 clr = enabled ? 0 : HECI_H_CSR_IE; + + __gsc_proxy_irq_rmw(gsc, clr, set); +} + +static int proxy_send_to_csme(struct xe_gsc *gsc, u32 size) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct i915_gsc_proxy_component *comp = gsc->proxy.component; + int ret; + + ret = comp->ops->send(comp->mei_dev, gsc->proxy.to_csme, size); + if (ret < 0) { + xe_gt_err(gt, "Failed to send CSME proxy message\n"); + return ret; + } + + ret = comp->ops->recv(comp->mei_dev, gsc->proxy.from_csme, GSC_PROXY_BUFFER_SIZE); + if (ret < 0) { + xe_gt_err(gt, "Failed to receive CSME proxy message\n"); + return ret; + } + + return ret; +} + +static int proxy_send_to_gsc(struct xe_gsc *gsc, u32 size) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + u64 addr_in = xe_bo_ggtt_addr(gsc->proxy.bo); + u64 addr_out = addr_in + GSC_PROXY_BUFFER_SIZE; + int err; + + /* the message must contain at least the gsc and proxy headers */ + if (size > GSC_PROXY_BUFFER_SIZE) { + xe_gt_err(gt, "Invalid GSC proxy message size: %u\n", size); + return -EINVAL; + } + + err = xe_gsc_pkt_submit_kernel(gsc, addr_in, size, + addr_out, GSC_PROXY_BUFFER_SIZE); + if (err) { + xe_gt_err(gt, "Failed to submit gsc proxy rq (%pe)\n", ERR_PTR(err)); + return err; + } + + return 0; +} + +static int validate_proxy_header(struct xe_gt *gt, + struct xe_gsc_proxy_header *header, + u32 source, u32 dest, u32 max_size) +{ + u32 type = FIELD_GET(GSC_PROXY_TYPE, header->hdr); + u32 length = FIELD_GET(GSC_PROXY_PAYLOAD_LENGTH, header->hdr); + int ret = 0; + + if (header->destination != dest || header->source != source) { + ret = -ENOEXEC; + goto out; + } + + if (length + PROXY_HDR_SIZE > max_size) { + ret = -E2BIG; + goto out; + } + + /* We only care about the status if this is a message for the driver */ + if (dest == GSC_PROXY_ADDRESSING_KMD && header->status != 0) { + ret = -EIO; + goto out; + } + + switch (type) { + case GSC_PROXY_MSG_TYPE_PROXY_PAYLOAD: + if (length > 0) + break; + fallthrough; + case GSC_PROXY_MSG_TYPE_PROXY_INVALID: + ret = -EIO; + break; + default: + break; + } + +out: + if (ret) + xe_gt_err(gt, + "GSC proxy error: s=0x%x[0x%x], d=0x%x[0x%x], t=%u, l=0x%x, st=0x%x\n", + header->source, source, header->destination, dest, + type, length, header->status); + + return ret; +} + +#define proxy_header_wr(xe_, map_, offset_, field_, val_) \ + xe_map_wr_field(xe_, map_, offset_, struct xe_gsc_proxy_header, field_, val_) + +#define proxy_header_rd(xe_, map_, offset_, field_) \ + xe_map_rd_field(xe_, map_, offset_, struct xe_gsc_proxy_header, field_) + +static u32 emit_proxy_header(struct xe_device *xe, struct iosys_map *map, u32 offset) +{ + xe_map_memset(xe, map, offset, 0, PROXY_HDR_SIZE); + + proxy_header_wr(xe, map, offset, hdr, + FIELD_PREP(GSC_PROXY_TYPE, GSC_PROXY_MSG_TYPE_PROXY_QUERY) | + FIELD_PREP(GSC_PROXY_PAYLOAD_LENGTH, 0)); + + proxy_header_wr(xe, map, offset, source, GSC_PROXY_ADDRESSING_KMD); + proxy_header_wr(xe, map, offset, destination, GSC_PROXY_ADDRESSING_GSC); + proxy_header_wr(xe, map, offset, status, 0); + + return offset + PROXY_HDR_SIZE; +} + +static int proxy_query(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + struct xe_gsc_proxy_header *to_csme_hdr = gsc->proxy.to_csme; + void *to_csme_payload = gsc->proxy.to_csme + PROXY_HDR_SIZE; + u32 wr_offset; + u32 reply_offset; + u32 size; + int ret; + + wr_offset = xe_gsc_emit_header(xe, &gsc->proxy.to_gsc, 0, + HECI_MEADDRESS_PROXY, 0, PROXY_HDR_SIZE); + wr_offset = emit_proxy_header(xe, &gsc->proxy.to_gsc, wr_offset); + + size = wr_offset; + + while (1) { + /* + * Poison the GSC response header space to make sure we don't + * read a stale reply. + */ + xe_gsc_poison_header(xe, &gsc->proxy.from_gsc, 0); + + /* send proxy message to GSC */ + ret = proxy_send_to_gsc(gsc, size); + if (ret) + goto proxy_error; + + /* check the reply from GSC */ + ret = xe_gsc_read_out_header(xe, &gsc->proxy.from_gsc, 0, + PROXY_HDR_SIZE, &reply_offset); + if (ret) { + xe_gt_err(gt, "Invalid gsc header in proxy reply (%pe)\n", + ERR_PTR(ret)); + goto proxy_error; + } + + /* copy the proxy header reply from GSC */ + xe_map_memcpy_from(xe, to_csme_hdr, &gsc->proxy.from_gsc, + reply_offset, PROXY_HDR_SIZE); + + /* Check the status and stop if this was the last message */ + if (FIELD_GET(GSC_PROXY_TYPE, to_csme_hdr->hdr) == GSC_PROXY_MSG_TYPE_PROXY_END) { + ret = validate_proxy_header(gt, to_csme_hdr, + GSC_PROXY_ADDRESSING_GSC, + GSC_PROXY_ADDRESSING_KMD, + GSC_PROXY_BUFFER_SIZE - reply_offset); + break; + } + + /* make sure the GSC-to-CSME proxy header is sane */ + ret = validate_proxy_header(gt, to_csme_hdr, + GSC_PROXY_ADDRESSING_GSC, + GSC_PROXY_ADDRESSING_CSME, + GSC_PROXY_BUFFER_SIZE - reply_offset); + if (ret) { + xe_gt_err(gt, "invalid GSC to CSME proxy header! (%pe)\n", + ERR_PTR(ret)); + goto proxy_error; + } + + /* copy the rest of the message */ + size = FIELD_GET(GSC_PROXY_PAYLOAD_LENGTH, to_csme_hdr->hdr); + xe_map_memcpy_from(xe, to_csme_payload, &gsc->proxy.from_gsc, + reply_offset + PROXY_HDR_SIZE, size); + + /* send the GSC message to the CSME */ + ret = proxy_send_to_csme(gsc, size + PROXY_HDR_SIZE); + if (ret < 0) + goto proxy_error; + + /* reply size from CSME, including the proxy header */ + size = ret; + if (size < PROXY_HDR_SIZE) { + xe_gt_err(gt, "CSME to GSC proxy msg too small: 0x%x\n", size); + ret = -EPROTO; + goto proxy_error; + } + + /* make sure the CSME-to-GSC proxy header is sane */ + ret = validate_proxy_header(gt, gsc->proxy.from_csme, + GSC_PROXY_ADDRESSING_CSME, + GSC_PROXY_ADDRESSING_GSC, + GSC_PROXY_BUFFER_SIZE - reply_offset); + if (ret) { + xe_gt_err(gt, "invalid CSME to GSC proxy header! %d\n", ret); + goto proxy_error; + } + + /* Emit a new header for sending the reply to the GSC */ + wr_offset = xe_gsc_emit_header(xe, &gsc->proxy.to_gsc, 0, + HECI_MEADDRESS_PROXY, 0, size); + + /* copy the CSME reply and update the total msg size to include the GSC header */ + xe_map_memcpy_to(xe, &gsc->proxy.to_gsc, wr_offset, gsc->proxy.from_csme, size); + + size += wr_offset; + } + +proxy_error: + return ret < 0 ? ret : 0; +} + +int xe_gsc_proxy_request_handler(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + int slept; + int err; + + if (!gsc->proxy.component_added) + return -ENODEV; + + /* when GSC is loaded, we can queue this before the component is bound */ + for (slept = 0; slept < GSC_PROXY_INIT_TIMEOUT_MS; slept += 100) { + if (gsc->proxy.component) + break; + + msleep(100); + } + + mutex_lock(&gsc->proxy.mutex); + if (!gsc->proxy.component) { + xe_gt_err(gt, "GSC proxy component not bound!\n"); + err = -EIO; + } else { + /* + * clear the pending interrupt and allow new proxy requests to + * be generated while we handle the current one + */ + gsc_proxy_irq_clear(gsc); + err = proxy_query(gsc); + } + mutex_unlock(&gsc->proxy.mutex); + return err; +} + +void xe_gsc_proxy_irq_handler(struct xe_gsc *gsc, u32 iir) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + + if (unlikely(!iir)) + return; + + if (!gsc->proxy.component) { + xe_gt_err(gt, "GSC proxy irq received without the component being bound!\n"); + return; + } + + spin_lock(&gsc->lock); + gsc->work_actions |= GSC_ACTION_SW_PROXY; + spin_unlock(&gsc->lock); + + queue_work(gsc->wq, &gsc->work); +} + +static int xe_gsc_proxy_component_bind(struct device *xe_kdev, + struct device *mei_kdev, void *data) +{ + struct xe_device *xe = kdev_to_xe_device(xe_kdev); + struct xe_gt *gt = xe->tiles[0].media_gt; + struct xe_gsc *gsc = >->uc.gsc; + + mutex_lock(&gsc->proxy.mutex); + gsc->proxy.component = data; + gsc->proxy.component->mei_dev = mei_kdev; + mutex_unlock(&gsc->proxy.mutex); + + return 0; +} + +static void xe_gsc_proxy_component_unbind(struct device *xe_kdev, + struct device *mei_kdev, void *data) +{ + struct xe_device *xe = kdev_to_xe_device(xe_kdev); + struct xe_gt *gt = xe->tiles[0].media_gt; + struct xe_gsc *gsc = >->uc.gsc; + + xe_gsc_wait_for_worker_completion(gsc); + + mutex_lock(&gsc->proxy.mutex); + gsc->proxy.component = NULL; + mutex_unlock(&gsc->proxy.mutex); +} + +static const struct component_ops xe_gsc_proxy_component_ops = { + .bind = xe_gsc_proxy_component_bind, + .unbind = xe_gsc_proxy_component_unbind, +}; + +static int proxy_channel_alloc(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *bo; + void *csme; + + csme = drmm_kzalloc(&xe->drm, GSC_PROXY_CHANNEL_SIZE, GFP_KERNEL); + if (!csme) + return -ENOMEM; + + bo = xe_managed_bo_create_pin_map(xe, tile, GSC_PROXY_CHANNEL_SIZE, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + gsc->proxy.bo = bo; + gsc->proxy.to_gsc = IOSYS_MAP_INIT_OFFSET(&bo->vmap, 0); + gsc->proxy.from_gsc = IOSYS_MAP_INIT_OFFSET(&bo->vmap, GSC_PROXY_BUFFER_SIZE); + gsc->proxy.to_csme = csme; + gsc->proxy.from_csme = csme + GSC_PROXY_BUFFER_SIZE; + + return 0; +} + +static void xe_gsc_proxy_stop(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + + /* disable HECI2 IRQs */ + scoped_guard(xe_pm_runtime, xe) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + if (!fw_ref.domains) + xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n"); + + /* try do disable irq even if forcewake failed */ + gsc_proxy_irq_toggle(gsc, false); + } + + xe_gsc_wait_for_worker_completion(gsc); + gsc->proxy.started = false; +} + +static void xe_gsc_proxy_remove(void *arg) +{ + struct xe_gsc *gsc = arg; + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + + if (!gsc->proxy.component_added) + return; + + /* + * GSC proxy start is an async process that can be ongoing during + * Xe module load/unload. Using devm managed action to register + * xe_gsc_proxy_stop could cause issues if Xe module unload has + * already started when the action is registered, potentially leading + * to the cleanup being called at the wrong time. Therefore, instead + * of registering a separate devm action to undo what is done in + * proxy start, we call it from here, but only if the start has + * completed successfully (tracked with the 'started' flag). + */ + if (gsc->proxy.started) + xe_gsc_proxy_stop(gsc); + + component_del(xe->drm.dev, &xe_gsc_proxy_component_ops); + gsc->proxy.component_added = false; +} + +/** + * xe_gsc_proxy_init() - init objects and MEI component required by GSC proxy + * @gsc: the GSC uC + * + * Return: 0 if the initialization was successful, a negative errno otherwise. + */ +int xe_gsc_proxy_init(struct xe_gsc *gsc) +{ + int err; + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = tile_to_xe(tile); + + mutex_init(&gsc->proxy.mutex); + + if (!IS_ENABLED(CONFIG_INTEL_MEI_GSC_PROXY)) { + xe_gt_info(gt, "can't init GSC proxy due to missing mei component\n"); + return -ENODEV; + } + + /* no multi-tile devices with this feature yet */ + if (!xe_tile_is_root(tile)) { + xe_gt_err(gt, "unexpected GSC proxy init on tile %u\n", tile->id); + return -EINVAL; + } + + err = proxy_channel_alloc(gsc); + if (err) + return err; + + err = component_add_typed(xe->drm.dev, &xe_gsc_proxy_component_ops, + I915_COMPONENT_GSC_PROXY); + if (err < 0) { + xe_gt_err(gt, "Failed to add GSC_PROXY component (%pe)\n", ERR_PTR(err)); + return err; + } + + gsc->proxy.component_added = true; + + return devm_add_action_or_reset(xe->drm.dev, xe_gsc_proxy_remove, gsc); +} + +/** + * xe_gsc_proxy_start() - start the proxy by submitting the first request + * @gsc: the GSC uC + * + * Return: 0 if the proxy are now enabled, a negative errno otherwise. + */ +int xe_gsc_proxy_start(struct xe_gsc *gsc) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + int err; + + /* enable the proxy interrupt in the GSC shim layer */ + gsc_proxy_irq_toggle(gsc, true); + + /* + * The handling of the first proxy request must be manually triggered to + * notify the GSC that we're ready to support the proxy flow. + */ + err = xe_gsc_proxy_request_handler(gsc); + if (err) + goto err_irq_disable; + + if (!xe_gsc_proxy_init_done(gsc)) { + xe_gt_err(gt, "GSC FW reports proxy init not completed\n"); + err = -EIO; + goto err_irq_disable; + } + + gsc->proxy.started = true; + return 0; + +err_irq_disable: + gsc_proxy_irq_toggle(gsc, false); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.h b/drivers/gpu/drm/xe/xe_gsc_proxy.h new file mode 100644 index 000000000..765602221 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_proxy.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GSC_PROXY_H_ +#define _XE_GSC_PROXY_H_ + +#include + +struct xe_gsc; + +int xe_gsc_proxy_init(struct xe_gsc *gsc); +bool xe_gsc_proxy_init_done(struct xe_gsc *gsc); +int xe_gsc_wait_for_proxy_init_done(struct xe_gsc *gsc); +int xe_gsc_proxy_start(struct xe_gsc *gsc); + +int xe_gsc_proxy_request_handler(struct xe_gsc *gsc); +void xe_gsc_proxy_irq_handler(struct xe_gsc *gsc, u32 iir); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gsc_submit.c b/drivers/gpu/drm/xe/xe_gsc_submit.c new file mode 100644 index 000000000..08082b596 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_submit.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gsc_submit.h" + +#include + +#include "abi/gsc_command_header_abi.h" +#include "xe_assert.h" +#include "xe_bb.h" +#include "xe_exec_queue.h" +#include "xe_gt_types.h" +#include "xe_map.h" +#include "xe_sched_job.h" +#include "instructions/xe_gsc_commands.h" + +#define GSC_HDR_SIZE (sizeof(struct intel_gsc_mtl_header)) /* shorthand define */ + +#define mtl_gsc_header_wr(xe_, map_, offset_, field_, val_) \ + xe_map_wr_field(xe_, map_, offset_, struct intel_gsc_mtl_header, field_, val_) + +#define mtl_gsc_header_rd(xe_, map_, offset_, field_) \ + xe_map_rd_field(xe_, map_, offset_, struct intel_gsc_mtl_header, field_) + +/* + * GSC FW allows us to define the host_session_handle as we see fit, as long + * as we use unique identifier for each user, with handle 0 being reserved for + * kernel usage. + * To be able to differentiate which client subsystem owns the given session, we + * include the client id in the top 8 bits of the handle. + */ +#define HOST_SESSION_CLIENT_MASK GENMASK_ULL(63, 56) + +static struct xe_gt * +gsc_to_gt(struct xe_gsc *gsc) +{ + return container_of(gsc, struct xe_gt, uc.gsc); +} + +/** + * xe_gsc_create_host_session_id - Creates a random 64 bit host_session id with + * bits 56-63 masked. + * + * Returns: random host_session_id which can be used to send messages to gsc cs + */ +u64 xe_gsc_create_host_session_id(void) +{ + u64 host_session_id; + + get_random_bytes(&host_session_id, sizeof(u64)); + host_session_id &= ~HOST_SESSION_CLIENT_MASK; + return host_session_id; +} + +/** + * xe_gsc_emit_header - write the MTL GSC header in memory + * @xe: the Xe device + * @map: the iosys map to write to + * @offset: offset from the start of the map at which to write the header + * @heci_client_id: client id identifying the type of command (see abi for values) + * @host_session_id: host session ID of the caller + * @payload_size: size of the payload that follows the header + * + * Returns: offset memory location following the header + */ +u32 xe_gsc_emit_header(struct xe_device *xe, struct iosys_map *map, u32 offset, + u8 heci_client_id, u64 host_session_id, u32 payload_size) +{ + xe_assert(xe, !(host_session_id & HOST_SESSION_CLIENT_MASK)); + + if (host_session_id) + host_session_id |= FIELD_PREP(HOST_SESSION_CLIENT_MASK, heci_client_id); + + xe_map_memset(xe, map, offset, 0, GSC_HDR_SIZE); + + mtl_gsc_header_wr(xe, map, offset, validity_marker, GSC_HECI_VALIDITY_MARKER); + mtl_gsc_header_wr(xe, map, offset, heci_client_id, heci_client_id); + mtl_gsc_header_wr(xe, map, offset, host_session_handle, host_session_id); + mtl_gsc_header_wr(xe, map, offset, header_version, MTL_GSC_HEADER_VERSION); + mtl_gsc_header_wr(xe, map, offset, message_size, payload_size + GSC_HDR_SIZE); + + return offset + GSC_HDR_SIZE; +}; + +/** + * xe_gsc_poison_header - poison the MTL GSC header in memory + * @xe: the Xe device + * @map: the iosys map to write to + * @offset: offset from the start of the map at which the header resides + */ +void xe_gsc_poison_header(struct xe_device *xe, struct iosys_map *map, u32 offset) +{ + xe_map_memset(xe, map, offset, POISON_FREE, GSC_HDR_SIZE); +}; + +/** + * xe_gsc_check_and_update_pending - check the pending bit and update the input + * header with the retry handle from the output header + * @xe: the Xe device + * @in: the iosys map containing the input buffer + * @offset_in: offset within the iosys at which the input buffer is located + * @out: the iosys map containing the output buffer + * @offset_out: offset within the iosys at which the output buffer is located + * + * Returns: true if the pending bit was set, false otherwise + */ +bool xe_gsc_check_and_update_pending(struct xe_device *xe, + struct iosys_map *in, u32 offset_in, + struct iosys_map *out, u32 offset_out) +{ + if (mtl_gsc_header_rd(xe, out, offset_out, flags) & GSC_OUTFLAG_MSG_PENDING) { + u64 handle = mtl_gsc_header_rd(xe, out, offset_out, gsc_message_handle); + + mtl_gsc_header_wr(xe, in, offset_in, gsc_message_handle, handle); + + return true; + } + + return false; +} + +/** + * xe_gsc_read_out_header - reads and validates the output header and returns + * the offset of the reply following the header + * @xe: the Xe device + * @map: the iosys map containing the output buffer + * @offset: offset within the iosys at which the output buffer is located + * @min_payload_size: minimum size of the message excluding the gsc header + * @payload_offset: optional pointer to be set to the payload offset + * + * Returns: -errno value on failure, 0 otherwise + */ +int xe_gsc_read_out_header(struct xe_device *xe, + struct iosys_map *map, u32 offset, + u32 min_payload_size, + u32 *payload_offset) +{ + u32 marker = mtl_gsc_header_rd(xe, map, offset, validity_marker); + u32 size = mtl_gsc_header_rd(xe, map, offset, message_size); + u32 status = mtl_gsc_header_rd(xe, map, offset, status); + u32 payload_size = size - GSC_HDR_SIZE; + + if (marker != GSC_HECI_VALIDITY_MARKER) + return -EPROTO; + + if (status != 0) { + drm_err(&xe->drm, "GSC header readout indicates error: %d\n", + status); + return -EINVAL; + } + + if (size < GSC_HDR_SIZE || payload_size < min_payload_size) + return -ENODATA; + + if (payload_offset) + *payload_offset = offset + GSC_HDR_SIZE; + + return 0; +} + +/** + * xe_gsc_pkt_submit_kernel - submit a kernel heci pkt to the GSC + * @gsc: the GSC uC + * @addr_in: GGTT address of the message to send to the GSC + * @size_in: size of the message to send to the GSC + * @addr_out: GGTT address for the GSC to write the reply to + * @size_out: size of the memory reserved for the reply + */ +int xe_gsc_pkt_submit_kernel(struct xe_gsc *gsc, u64 addr_in, u32 size_in, + u64 addr_out, u32 size_out) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_bb *bb; + struct xe_sched_job *job; + struct dma_fence *fence; + long timeout; + + if (size_in < GSC_HDR_SIZE) + return -ENODATA; + + if (size_out < GSC_HDR_SIZE) + return -ENOMEM; + + bb = xe_bb_new(gt, 8, false); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + bb->cs[bb->len++] = GSC_HECI_CMD_PKT; + bb->cs[bb->len++] = lower_32_bits(addr_in); + bb->cs[bb->len++] = upper_32_bits(addr_in); + bb->cs[bb->len++] = size_in; + bb->cs[bb->len++] = lower_32_bits(addr_out); + bb->cs[bb->len++] = upper_32_bits(addr_out); + bb->cs[bb->len++] = size_out; + bb->cs[bb->len++] = 0; + + job = xe_bb_create_job(gsc->q, bb); + if (IS_ERR(job)) { + xe_bb_free(bb, NULL); + return PTR_ERR(job); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, HZ); + dma_fence_put(fence); + xe_bb_free(bb, NULL); + if (timeout < 0) + return timeout; + else if (!timeout) + return -ETIME; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gsc_submit.h b/drivers/gpu/drm/xe/xe_gsc_submit.h new file mode 100644 index 000000000..1416b5745 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_submit.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GSC_SUBMIT_H_ +#define _XE_GSC_SUBMIT_H_ + +#include + +struct iosys_map; +struct xe_device; +struct xe_gsc; + +u32 xe_gsc_emit_header(struct xe_device *xe, struct iosys_map *map, u32 offset, + u8 heci_client_id, u64 host_session_id, u32 payload_size); +void xe_gsc_poison_header(struct xe_device *xe, struct iosys_map *map, u32 offset); + +bool xe_gsc_check_and_update_pending(struct xe_device *xe, + struct iosys_map *in, u32 offset_in, + struct iosys_map *out, u32 offset_out); + +int xe_gsc_read_out_header(struct xe_device *xe, + struct iosys_map *map, u32 offset, + u32 min_payload_size, + u32 *payload_offset); + +int xe_gsc_pkt_submit_kernel(struct xe_gsc *gsc, u64 addr_in, u32 size_in, + u64 addr_out, u32 size_out); + +u64 xe_gsc_create_host_session_id(void); +#endif diff --git a/drivers/gpu/drm/xe/xe_gsc_types.h b/drivers/gpu/drm/xe/xe_gsc_types.h new file mode 100644 index 000000000..5aaa2a758 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_types.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GSC_TYPES_H_ +#define _XE_GSC_TYPES_H_ + +#include +#include +#include +#include +#include + +#include "xe_uc_fw_types.h" +#include "xe_device_types.h" + +struct xe_bo; +struct xe_exec_queue; +struct i915_gsc_proxy_component; + +/** + * struct xe_gsc - GSC + */ +struct xe_gsc { + /** @fw: Generic uC firmware management */ + struct xe_uc_fw fw; + + /** @security_version: SVN found in the fetched blob */ + u32 security_version; + + /** @private: Private data for use by the GSC FW */ + struct xe_bo *private; + + /** @q: Default queue used for submissions to GSC FW */ + struct xe_exec_queue *q; + + /** @wq: workqueue to handle jobs for delayed load and proxy handling */ + struct workqueue_struct *wq; + + /** @work: delayed load and proxy handling work */ + struct work_struct work; + + /** @lock: protects access to the work_actions mask */ + spinlock_t lock; + + /** @work_actions: mask of actions to be performed in the work */ + u32 work_actions; +#define GSC_ACTION_FW_LOAD BIT(0) +#define GSC_ACTION_SW_PROXY BIT(1) +#define GSC_ACTION_ER_COMPLETE BIT(2) + + /** @proxy: sub-structure containing the SW proxy-related variables */ + struct { + /** @proxy.component: struct for communication with mei component */ + struct i915_gsc_proxy_component *component; + /** @proxy.mutex: protects the component binding and usage */ + struct mutex mutex; + /** @proxy.component_added: whether the component has been added */ + bool component_added; + /** @proxy.started: whether the proxy has been started */ + bool started; + /** @proxy.bo: object to store message to and from the GSC */ + struct xe_bo *bo; + /** @proxy.to_gsc: map of the memory used to send messages to the GSC */ + struct iosys_map to_gsc; + /** @proxy.from_gsc: map of the memory used to recv messages from the GSC */ + struct iosys_map from_gsc; + /** @proxy.to_csme: pointer to the memory used to send messages to CSME */ + void *to_csme; + /** @proxy.from_csme: pointer to the memory used to recv messages from CSME */ + void *from_csme; + } proxy; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c new file mode 100644 index 000000000..dfdacc0f6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -0,0 +1,1193 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gt.h" + +#include + +#include + +#include +#include + +#include +#include + +#include "instructions/xe_alu_commands.h" +#include "instructions/xe_mi_commands.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "xe_assert.h" +#include "xe_bb.h" +#include "xe_debugfs.h" +#include "xe_device.h" +#include "xe_eu_stall.h" +#include "xe_exec_queue.h" +#include "xe_execlist.h" +#include "xe_force_wake.h" +#include "xe_ggtt.h" +#include "xe_gsc.h" +#include "xe_gt_ccs_mode.h" +#include "xe_gt_clock.h" +#include "xe_gt_freq.h" +#include "xe_gt_idle.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_pf.h" +#include "xe_gt_sriov_vf.h" +#include "xe_gt_stats.h" +#include "xe_gt_sysfs.h" +#include "xe_gt_topology.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_guc_pc.h" +#include "xe_guc_rc.h" +#include "xe_guc_submit.h" +#include "xe_hw_fence.h" +#include "xe_hw_engine_class_sysfs.h" +#include "xe_irq.h" +#include "xe_lmtt.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_migrate.h" +#include "xe_mmio.h" +#include "xe_pagefault.h" +#include "xe_pat.h" +#include "xe_pm.h" +#include "xe_mocs.h" +#include "xe_reg_sr.h" +#include "xe_ring_ops.h" +#include "xe_sa.h" +#include "xe_sched_job.h" +#include "xe_sriov.h" +#include "xe_tlb_inval.h" +#include "xe_tuning.h" +#include "xe_uc.h" +#include "xe_uc_fw.h" +#include "xe_vm.h" +#include "xe_wa.h" +#include "xe_wopcm.h" + +struct xe_gt *xe_gt_alloc(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct drm_device *drm = &xe->drm; + bool shared_wq = xe->info.needs_shared_vf_gt_wq && tile->primary_gt && + IS_SRIOV_VF(xe); + struct workqueue_struct *ordered_wq; + struct xe_gt *gt; + + gt = drmm_kzalloc(drm, sizeof(*gt), GFP_KERNEL); + if (!gt) + return ERR_PTR(-ENOMEM); + + gt->tile = tile; + if (shared_wq && tile->primary_gt->ordered_wq) + ordered_wq = tile->primary_gt->ordered_wq; + else + ordered_wq = drmm_alloc_ordered_workqueue(drm, "gt-ordered-wq", + WQ_MEM_RECLAIM); + if (IS_ERR(ordered_wq)) + return ERR_CAST(ordered_wq); + + gt->ordered_wq = ordered_wq; + + return gt; +} + +void xe_gt_sanitize(struct xe_gt *gt) +{ + /* + * FIXME: if xe_uc_sanitize is called here, on TGL driver will not + * reload + */ + xe_guc_submit_disable(>->uc.guc); +} + +static void xe_gt_enable_host_l2_vram(struct xe_gt *gt) +{ + u32 reg; + + if (!XE_GT_WA(gt, 16023588340)) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return; + + if (xe_gt_is_main_type(gt)) { + reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL); + reg |= CG_DIS_CNTLBUS; + xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg); + } + + xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF); +} + +static void xe_gt_disable_host_l2_vram(struct xe_gt *gt) +{ + u32 reg; + + if (!XE_GT_WA(gt, 16023588340)) + return; + + if (xe_gt_is_media_type(gt)) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return; + + reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL); + reg &= ~CG_DIS_CNTLBUS; + xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg); +} + +static void xe_gt_enable_comp_1wcoh(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 reg; + + if (IS_SRIOV_VF(xe)) + return; + + if (GRAPHICS_VER(xe) >= 30 && xe->info.has_flat_ccs) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return; + + reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL); + reg |= EN_CMP_1WCOH; + xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg); + + if (xe_gt_is_media_type(gt)) { + xe_mmio_rmw32(>->mmio, XE2_GAMWALK_CTRL_MEDIA, 0, EN_CMP_1WCOH_GW); + } else { + reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMWALK_CTRL_3D); + reg |= EN_CMP_1WCOH_GW; + xe_gt_mcr_multicast_write(gt, XE2_GAMWALK_CTRL_3D, reg); + } + } +} + +static void gt_reset_worker(struct work_struct *w); + +static int emit_job_sync(struct xe_exec_queue *q, struct xe_bb *bb, + long timeout_jiffies, bool force_reset) +{ + struct xe_sched_job *job; + struct dma_fence *fence; + long timeout; + + job = xe_bb_create_job(q, bb); + if (IS_ERR(job)) + return PTR_ERR(job); + + job->ring_ops_force_reset = force_reset; + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, timeout_jiffies); + dma_fence_put(fence); + if (timeout < 0) + return timeout; + else if (!timeout) + return -ETIME; + + return 0; +} + +static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) +{ + struct xe_bb *bb; + int ret; + + bb = xe_bb_new(gt, 4, false); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + ret = emit_job_sync(q, bb, HZ, false); + xe_bb_free(bb, NULL); + + return ret; +} + +/* Dwords required to emit a RMW of a register */ +#define EMIT_RMW_DW 20 + +static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) +{ + struct xe_hw_engine *hwe = q->hwe; + struct xe_reg_sr *sr = &hwe->reg_lrc; + struct xe_reg_sr_entry *entry; + int count_rmw = 0, count_rmw_mcr = 0, count = 0, ret; + unsigned long idx; + struct xe_bb *bb; + size_t bb_len = 0; + u32 *cs; + + /* count RMW registers as those will be handled separately */ + xa_for_each(&sr->xa, idx, entry) { + if (entry->reg.masked || entry->clr_bits == ~0) + ++count; + else if (entry->reg.mcr) + ++count_rmw_mcr; + else + ++count_rmw; + } + + if (count) + bb_len += count * 2 + 1; + + /* + * RMW of MCR registers is the same as a normal RMW, except an + * additional LRI (3 dwords) is required per register to steer the read + * to a nom-terminated instance. + * + * We could probably shorten the batch slightly by eliding the + * steering for consecutive MCR registers that have the same + * group/instance target, but it's not worth the extra complexity to do + * so. + */ + bb_len += count_rmw * EMIT_RMW_DW; + bb_len += count_rmw_mcr * (EMIT_RMW_DW + 3); + + /* + * After doing all RMW, we need 7 trailing dwords to clean up, + * plus an additional 3 dwords to reset steering if any of the + * registers were MCR. + */ + if (count_rmw || count_rmw_mcr) + bb_len += 7 + (count_rmw_mcr ? 3 : 0); + + if (hwe->class == XE_ENGINE_CLASS_RENDER) + /* + * Big enough to emit all of the context's 3DSTATE via + * xe_lrc_emit_hwe_state_instructions() + */ + bb_len += xe_gt_lrc_size(gt, hwe->class) / sizeof(u32); + + xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", hwe->name, bb_len); + + bb = xe_bb_new(gt, bb_len, false); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + cs = bb->cs; + + if (count) { + /* + * Emit single LRI with all non RMW regs: 1 leading dw + 2dw per + * reg + 1 + */ + + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count); + + xa_for_each(&sr->xa, idx, entry) { + struct xe_reg reg = entry->reg; + u32 val; + + if (reg.masked) + val = entry->clr_bits << 16; + else if (entry->clr_bits == ~0) + val = 0; + else + continue; + + val |= entry->set_bits; + + *cs++ = reg.addr; + *cs++ = val; + xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); + } + } + + if (count_rmw || count_rmw_mcr) { + xa_for_each(&sr->xa, idx, entry) { + if (entry->reg.masked || entry->clr_bits == ~0) + continue; + + if (entry->reg.mcr) { + struct xe_reg_mcr reg = { .__reg.raw = entry->reg.raw }; + u8 group, instance; + + xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance); + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(hwe->mmio_base).addr; + *cs++ = SELECTIVE_READ_ADDRESSING | + REG_FIELD_PREP(SELECTIVE_READ_GROUP, group) | + REG_FIELD_PREP(SELECTIVE_READ_INSTANCE, instance); + } + + *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_DST_CS_MMIO; + *cs++ = entry->reg.addr; + *cs++ = CS_GPR_REG(0, 0).addr; + + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) | + MI_LRI_LRM_CS_MMIO; + *cs++ = CS_GPR_REG(0, 1).addr; + *cs++ = entry->clr_bits; + *cs++ = CS_GPR_REG(0, 2).addr; + *cs++ = entry->set_bits; + + *cs++ = MI_MATH(8); + *cs++ = CS_ALU_INSTR_LOAD(SRCA, REG0); + *cs++ = CS_ALU_INSTR_LOADINV(SRCB, REG1); + *cs++ = CS_ALU_INSTR_AND; + *cs++ = CS_ALU_INSTR_STORE(REG0, ACCU); + *cs++ = CS_ALU_INSTR_LOAD(SRCA, REG0); + *cs++ = CS_ALU_INSTR_LOAD(SRCB, REG2); + *cs++ = CS_ALU_INSTR_OR; + *cs++ = CS_ALU_INSTR_STORE(REG0, ACCU); + + *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_SRC_CS_MMIO; + *cs++ = CS_GPR_REG(0, 0).addr; + *cs++ = entry->reg.addr; + + xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x%s\n", + entry->reg.addr, entry->clr_bits, entry->set_bits, + entry->reg.mcr ? " (MCR)" : ""); + } + + /* reset used GPR */ + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(3) | + MI_LRI_LRM_CS_MMIO; + *cs++ = CS_GPR_REG(0, 0).addr; + *cs++ = 0; + *cs++ = CS_GPR_REG(0, 1).addr; + *cs++ = 0; + *cs++ = CS_GPR_REG(0, 2).addr; + *cs++ = 0; + + /* reset steering */ + if (count_rmw_mcr) { + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(q->hwe->mmio_base).addr; + *cs++ = 0; + } + } + + cs = xe_lrc_emit_hwe_state_instructions(q, cs); + + bb->len = cs - bb->cs; + + /* only VFs need to trigger reset to get a clean NULL context */ + ret = emit_job_sync(q, bb, HZ, IS_SRIOV_VF(gt_to_xe(gt))); + + xe_bb_free(bb, NULL); + + return ret; +} + +int xe_gt_record_default_lrcs(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int err = 0; + + for_each_hw_engine(hwe, gt, id) { + struct xe_exec_queue *q, *nop_q; + void *default_lrc; + + if (gt->default_lrc[hwe->class]) + continue; + + xe_hw_engine_setup_reg_lrc(hwe); + + default_lrc = drmm_kzalloc(&xe->drm, + xe_gt_lrc_size(gt, hwe->class), + GFP_KERNEL); + if (!default_lrc) + return -ENOMEM; + + q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1, + hwe, EXEC_QUEUE_FLAG_KERNEL, 0); + if (IS_ERR(q)) { + err = PTR_ERR(q); + xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n", + hwe->name, q); + return err; + } + + /* Prime golden LRC with known good state */ + err = emit_wa_job(gt, q); + if (err) { + xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n", + hwe->name, ERR_PTR(err), q->guc->id); + goto put_exec_queue; + } + + nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), + 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0); + if (IS_ERR(nop_q)) { + err = PTR_ERR(nop_q); + xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n", + hwe->name, nop_q); + goto put_exec_queue; + } + + /* Switch to different LRC */ + err = emit_nop_job(gt, nop_q); + if (err) { + xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n", + hwe->name, ERR_PTR(err), nop_q->guc->id); + goto put_nop_q; + } + + xe_map_memcpy_from(xe, default_lrc, + &q->lrc[0]->bo->vmap, + xe_lrc_pphwsp_offset(q->lrc[0]), + xe_gt_lrc_size(gt, hwe->class)); + + gt->default_lrc[hwe->class] = default_lrc; +put_nop_q: + xe_exec_queue_put(nop_q); +put_exec_queue: + xe_exec_queue_put(q); + if (err) + break; + } + + return err; +} + +static void wa_14026539277(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 val; + + /* + * FIXME: We currently can't use FUNC(xe_rtp_match_not_sriov_vf) in the + * rules for Wa_14026539277 due to xe_wa_process_device_oob() being + * called before xe_sriov_probe_early(); and we can't move the call to + * the former to happen after the latter because MMIO read functions + * already depend on a device OOB workaround. This needs to be fixed by + * allowing workaround checks to happen at different stages of driver + * initialization. + */ + if (IS_SRIOV_VF(xe)) + return; + + if (!XE_DEVICE_WA(xe, 14026539277)) + return; + + if (!xe_gt_is_main_type(gt)) + return; + + val = xe_gt_mcr_unicast_read_any(gt, L2COMPUTESIDECTRL); + val &= ~CECTRL; + val |= CECTRL_CENODATA_ALWAYS; + xe_gt_mcr_multicast_write(gt, L2COMPUTESIDECTRL, val); +} + +int xe_gt_init_early(struct xe_gt *gt) +{ + int err; + + if (IS_SRIOV_PF(gt_to_xe(gt))) { + err = xe_gt_sriov_pf_init_early(gt); + if (err) + return err; + } + + if (IS_SRIOV_VF(gt_to_xe(gt))) { + err = xe_gt_sriov_vf_init_early(gt); + if (err) + return err; + } + + xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); + + err = xe_wa_gt_init(gt); + if (err) + return err; + + err = xe_tuning_init(gt); + if (err) + return err; + + xe_wa_process_gt_oob(gt); + + xe_force_wake_init_gt(gt, gt_to_fw(gt)); + spin_lock_init(>->global_invl_lock); + + err = xe_gt_tlb_inval_init_early(gt); + if (err) + return err; + + xe_mocs_init_early(gt); + + /* + * Only after this point can GT-specific MMIO operations + * (including things like communication with the GuC) + * be performed. + */ + xe_gt_mmio_init(gt); + + err = xe_uc_init_noalloc(>->uc); + if (err) + return err; + + err = xe_gt_stats_init(gt); + if (err) + return err; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + xe_gt_mcr_init_early(gt); + xe_pat_init(gt); + + return 0; +} + +static void dump_pat_on_error(struct xe_gt *gt) +{ + struct drm_printer p; + char prefix[32]; + + snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id); + p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix); + + xe_pat_dump(gt, &p); +} + +static int gt_init_with_gt_forcewake(struct xe_gt *gt) +{ + int err; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + err = xe_uc_init(>->uc); + if (err) + return err; + + xe_gt_topology_init(gt); + xe_gt_mcr_init(gt); + xe_gt_enable_host_l2_vram(gt); + xe_gt_enable_comp_1wcoh(gt); + + if (xe_gt_is_main_type(gt)) { + err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt); + if (err) + return err; + if (IS_SRIOV_PF(gt_to_xe(gt))) + xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt); + } + + /* Enable per hw engine IRQs */ + xe_irq_enable_hwe(gt); + + /* Rerun MCR init as we now have hw engine list */ + xe_gt_mcr_init(gt); + + err = xe_hw_engines_init_early(gt); + if (err) { + dump_pat_on_error(gt); + return err; + } + + err = xe_hw_engine_class_sysfs_init(gt); + if (err) + return err; + + /* Initialize CCS mode sysfs after early initialization of HW engines */ + err = xe_gt_ccs_mode_sysfs_init(gt); + if (err) + return err; + + /* + * Stash hardware-reported version. Since this register does not exist + * on pre-MTL platforms, reading it there will (correctly) return 0. + */ + gt->info.gmdid = xe_mmio_read32(>->mmio, GMD_ID); + + /* + * Wa_14026539277 can't be implemented as a regular GT workaround (i.e. + * as an entry in gt_was[]) for two reasons: it is actually a device + * workaround that happens to involve programming a GT register; and it + * needs to be applied early to avoid getting the hardware in a bad + * state before we have a chance to do the necessary programming. + */ + wa_14026539277(gt); + + return 0; +} + +static int gt_init_with_all_forcewake(struct xe_gt *gt) +{ + int err; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + return -ETIMEDOUT; + + xe_gt_mcr_set_implicit_defaults(gt); + xe_wa_process_gt(gt); + xe_tuning_process_gt(gt); + xe_reg_sr_apply_mmio(>->reg_sr, gt); + + err = xe_gt_clock_init(gt); + if (err) + return err; + + xe_mocs_init(gt); + err = xe_execlist_init(gt); + if (err) + return err; + + err = xe_hw_engines_init(gt); + if (err) + return err; + + err = xe_uc_init_post_hwconfig(>->uc); + if (err) + return err; + + if (xe_gt_is_main_type(gt)) { + /* + * USM has its only SA pool to non-block behind user operations + */ + if (gt_to_xe(gt)->info.has_usm) { + struct xe_device *xe = gt_to_xe(gt); + + gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt), + IS_DGFX(xe) ? SZ_1M : SZ_512K, 16); + if (IS_ERR(gt->usm.bb_pool)) + return PTR_ERR(gt->usm.bb_pool); + } + } + + if (xe_gt_is_main_type(gt)) { + struct xe_tile *tile = gt_to_tile(gt); + + err = xe_migrate_init(tile->migrate); + if (err) + return err; + } + + err = xe_uc_load_hw(>->uc); + if (err) + return err; + + /* Configure default CCS mode of 1 engine with all resources */ + if (xe_gt_ccs_mode_enabled(gt)) { + gt->ccs_mode = 1; + xe_gt_apply_ccs_mode(gt); + } + + if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_is_main_type(gt)) + xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); + + if (IS_SRIOV_PF(gt_to_xe(gt))) + xe_gt_sriov_pf_init_hw(gt); + + return 0; +} + +static void xe_gt_fini(void *arg) +{ + struct xe_gt *gt = arg; + int i; + + if (disable_work_sync(>->reset.worker)) + /* + * If gt_reset_worker was halted from executing, take care of + * releasing the rpm reference here. + */ + xe_pm_runtime_put(gt_to_xe(gt)); + + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) + xe_hw_fence_irq_finish(>->fence_irq[i]); + + xe_gt_disable_host_l2_vram(gt); +} + +int xe_gt_init(struct xe_gt *gt) +{ + int err; + int i; + + INIT_WORK(>->reset.worker, gt_reset_worker); + + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) { + gt->ring_ops[i] = xe_ring_ops_get(gt, i); + xe_hw_fence_irq_init(>->fence_irq[i]); + } + + err = devm_add_action_or_reset(gt_to_xe(gt)->drm.dev, xe_gt_fini, gt); + if (err) + return err; + + err = xe_gt_sysfs_init(gt); + if (err) + return err; + + err = gt_init_with_gt_forcewake(gt); + if (err) + return err; + + err = xe_gt_idle_init(>->gtidle); + if (err) + return err; + + err = xe_gt_freq_init(gt); + if (err) + return err; + + xe_force_wake_init_engines(gt, gt_to_fw(gt)); + + err = gt_init_with_all_forcewake(gt); + if (err) + return err; + + xe_gt_record_user_engines(gt); + + err = xe_eu_stall_init(gt); + if (err) + return err; + + if (IS_SRIOV_VF(gt_to_xe(gt))) { + err = xe_gt_sriov_vf_init(gt); + if (err) + return err; + } + + return 0; +} + +/** + * xe_gt_mmio_init() - Initialize GT's MMIO access + * @gt: the GT object + * + * Initialize GT's MMIO accessor, which will be used to access registers inside + * this GT. + */ +void xe_gt_mmio_init(struct xe_gt *gt) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = tile_to_xe(tile); + + xe_mmio_init(>->mmio, tile, tile->mmio.regs, tile->mmio.regs_size); + + if (gt->info.type == XE_GT_TYPE_MEDIA) { + gt->mmio.adj_offset = MEDIA_GT_GSI_OFFSET; + gt->mmio.adj_limit = MEDIA_GT_GSI_LENGTH; + } else { + gt->mmio.adj_offset = 0; + gt->mmio.adj_limit = 0; + } + + if (IS_SRIOV_VF(xe)) + gt->mmio.sriov_vf_gt = gt; +} +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mmio_init); + +void xe_gt_record_user_engines(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + gt->user_engines.mask = 0; + memset(gt->user_engines.instances_per_class, 0, + sizeof(gt->user_engines.instances_per_class)); + + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + + gt->user_engines.mask |= BIT_ULL(id); + gt->user_engines.instances_per_class[hwe->class]++; + } + + xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask) + == gt->info.engine_mask); +} + +static int do_gt_reset(struct xe_gt *gt) +{ + int err; + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return xe_gt_sriov_vf_reset(gt); + + xe_gsc_wa_14015076503(gt, true); + + xe_mmio_write32(>->mmio, GDRST, GRDOM_FULL); + err = xe_mmio_wait32(>->mmio, GDRST, GRDOM_FULL, 0, 5000, NULL, false); + if (err) + xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n", + ERR_PTR(err)); + + xe_gsc_wa_14015076503(gt, false); + + return err; +} + +static int vf_gt_restart(struct xe_gt *gt) +{ + int err; + + err = xe_uc_sanitize_reset(>->uc); + if (err) + return err; + + err = xe_uc_load_hw(>->uc); + if (err) + return err; + + err = xe_uc_start(>->uc); + if (err) + return err; + + return 0; +} + +static int do_gt_restart(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int err; + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return vf_gt_restart(gt); + + xe_pat_init(gt); + + xe_gt_enable_host_l2_vram(gt); + xe_gt_enable_comp_1wcoh(gt); + + xe_gt_mcr_set_implicit_defaults(gt); + xe_reg_sr_apply_mmio(>->reg_sr, gt); + + err = xe_wopcm_init(>->uc.wopcm); + if (err) + return err; + + for_each_hw_engine(hwe, gt, id) + xe_hw_engine_enable_ring(hwe); + + err = xe_uc_sanitize_reset(>->uc); + if (err) + return err; + + err = xe_uc_load_hw(>->uc); + if (err) + return err; + + if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_is_main_type(gt)) + xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); + + if (IS_SRIOV_PF(gt_to_xe(gt))) + xe_gt_sriov_pf_init_hw(gt); + + xe_mocs_init(gt); + + for_each_hw_engine(hwe, gt, id) + xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); + + /* Get CCS mode in sync between sw/hw */ + xe_gt_apply_ccs_mode(gt); + + err = xe_uc_start(>->uc); + if (err) + return err; + + /* Restore GT freq to expected values */ + xe_gt_sanitize_freq(gt); + + if (IS_SRIOV_PF(gt_to_xe(gt))) + xe_gt_sriov_pf_restart(gt); + + return 0; +} + +static void gt_reset_worker(struct work_struct *w) +{ + struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker); + unsigned int fw_ref; + int err; + + if (xe_device_wedged(gt_to_xe(gt))) + goto err_pm_put; + + if (xe_device_is_in_reset(gt_to_xe(gt))) + goto err_pm_put; + + /* We only support GT resets with GuC submission */ + if (!xe_device_uc_enabled(gt_to_xe(gt))) + goto err_pm_put; + + xe_gt_info(gt, "reset started\n"); + + if (xe_fault_gt_reset()) { + err = -ECANCELED; + goto err_fail; + } + + xe_gt_sanitize(gt); + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { + err = -ETIMEDOUT; + goto err_out; + } + + if (IS_SRIOV_PF(gt_to_xe(gt))) + xe_gt_sriov_pf_stop_prepare(gt); + + xe_guc_rc_disable(>->uc.guc); + xe_uc_stop_prepare(>->uc); + xe_pagefault_reset(gt_to_xe(gt), gt); + + xe_uc_stop(>->uc); + + xe_tlb_inval_reset(>->tlb_inval); + + err = do_gt_reset(gt); + if (err) + goto err_out; + + err = do_gt_restart(gt); + if (err) + goto err_out; + + xe_force_wake_put(gt_to_fw(gt), fw_ref); + + /* Pair with get while enqueueing the work in xe_gt_reset_async() */ + xe_pm_runtime_put(gt_to_xe(gt)); + + xe_gt_info(gt, "reset done\n"); + + return; + +err_out: + xe_force_wake_put(gt_to_fw(gt), fw_ref); + XE_WARN_ON(xe_uc_start(>->uc)); + +err_fail: + xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); + xe_device_declare_wedged(gt_to_xe(gt)); +err_pm_put: + xe_pm_runtime_put(gt_to_xe(gt)); +} + +void xe_gt_reset_async(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (xe_device_is_in_reset(xe)) + return; + + /* Don't do a reset while one is already in flight */ + if (!xe_fault_gt_reset() && xe_uc_reset_prepare(>->uc)) + return; + + xe_gt_info(gt, "reset queued from %ps\n", __builtin_return_address(0)); + + /* Pair with put in gt_reset_worker() if work is enqueued */ + xe_pm_runtime_get_noresume(xe); + if (!queue_work(gt->ordered_wq, >->reset.worker)) + xe_pm_runtime_put(xe); +} + +void xe_gt_suspend_prepare(struct xe_gt *gt) +{ + xe_uc_suspend_prepare(>->uc); +} + +int xe_gt_suspend(struct xe_gt *gt) +{ + int err; + + xe_gt_dbg(gt, "suspending\n"); + xe_gt_sanitize(gt); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } + + err = xe_uc_suspend(>->uc); + if (err) { + xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err)); + return err; + } + + xe_gt_idle_disable_pg(gt); + + xe_gt_disable_host_l2_vram(gt); + + xe_gt_dbg(gt, "suspended\n"); + + return 0; +} + +void xe_gt_shutdown(struct xe_gt *gt) +{ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + do_gt_reset(gt); +} + +/** + * xe_gt_sanitize_freq() - Restore saved frequencies if necessary. + * @gt: the GT object + * + * Called after driver init/GSC load completes to restore GT frequencies if we + * limited them for any WAs. + */ +int xe_gt_sanitize_freq(struct xe_gt *gt) +{ + int ret = 0; + + if ((!xe_uc_fw_is_available(>->uc.gsc.fw) || + xe_uc_fw_is_loaded(>->uc.gsc.fw) || + xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) && + XE_GT_WA(gt, 22019338487)) + ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc); + + return ret; +} + +int xe_gt_resume(struct xe_gt *gt) +{ + int err; + + xe_gt_dbg(gt, "resuming\n"); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } + + err = do_gt_restart(gt); + if (err) + return err; + + xe_gt_idle_enable_pg(gt); + + xe_gt_dbg(gt, "resumed\n"); + + return 0; +} + +/** + * xe_gt_runtime_suspend() - GT runtime suspend + * @gt: the GT object + * + * Return: 0 on success, negative error code otherwise. + */ +int xe_gt_runtime_suspend(struct xe_gt *gt) +{ + xe_gt_dbg(gt, "runtime suspending\n"); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "runtime suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } + + xe_uc_runtime_suspend(>->uc); + xe_gt_disable_host_l2_vram(gt); + + xe_gt_dbg(gt, "runtime suspended\n"); + + return 0; +} + +/** + * xe_gt_runtime_resume() - GT runtime resume + * @gt: the GT object + * + * Return: 0 on success, negative error code otherwise. + */ +int xe_gt_runtime_resume(struct xe_gt *gt) +{ + xe_gt_dbg(gt, "runtime resuming\n"); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "runtime resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } + + xe_gt_enable_host_l2_vram(gt); + xe_uc_runtime_resume(>->uc); + + xe_gt_dbg(gt, "runtime resumed\n"); + + return 0; +} + +struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, + enum xe_engine_class class, + u16 instance, bool logical) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) + if (hwe->class == class && + ((!logical && hwe->instance == instance) || + (logical && hwe->logical_instance == instance))) + return hwe; + + return NULL; +} + +struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, + enum xe_engine_class class) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) { + switch (class) { + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + if (hwe->class == XE_ENGINE_CLASS_RENDER || + hwe->class == XE_ENGINE_CLASS_COMPUTE) + return hwe; + break; + default: + if (hwe->class == class) + return hwe; + } + } + + return NULL; +} + +struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) + return hwe; + + return NULL; +} + +/** + * xe_gt_declare_wedged() - Declare GT wedged + * @gt: the GT object + * + * Wedge the GT which stops all submission, saves desired debug state, and + * cleans up anything which could timeout. + */ +void xe_gt_declare_wedged(struct xe_gt *gt) +{ + xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode); + + xe_uc_declare_wedged(>->uc); + xe_tlb_inval_reset(>->tlb_inval); +} diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h new file mode 100644 index 000000000..65a4655b0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_H_ +#define _XE_GT_H_ + +#include + +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_gt_sriov_vf.h" +#include "xe_hw_engine.h" + +#define for_each_hw_engine(hwe__, gt__, id__) \ + for ((id__) = 0; (id__) < ARRAY_SIZE((gt__)->hw_engines); (id__)++) \ + for_each_if(((hwe__) = (gt__)->hw_engines + (id__)) && \ + xe_hw_engine_is_valid((hwe__))) + +#define XE_ENGINE_INSTANCES_FROM_MASK(gt, NAME) \ + (((gt)->info.engine_mask & XE_HW_ENGINE_##NAME##_MASK) >> XE_HW_ENGINE_##NAME##0) + +#define RCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, RCS) +#define VCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, VCS) +#define VECS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, VECS) +#define CCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, CCS) +#define GSCCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, GSCCS) + +/* Our devices have up to 4 media slices */ +#define MAX_MEDIA_SLICES 4 + +#define GT_VER(gt) ({ \ + typeof(gt) gt_ = (gt); \ + struct xe_device *xe = gt_to_xe(gt_); \ + xe_gt_is_media_type(gt_) ? MEDIA_VER(xe) : GRAPHICS_VER(xe); \ +}) + +struct xe_gt *xe_gt_alloc(struct xe_tile *tile); +int xe_gt_init_early(struct xe_gt *gt); +int xe_gt_init(struct xe_gt *gt); +void xe_gt_mmio_init(struct xe_gt *gt); +void xe_gt_declare_wedged(struct xe_gt *gt); +int xe_gt_record_default_lrcs(struct xe_gt *gt); + +/** + * xe_gt_record_user_engines - save data related to engines available to + * userspace + * @gt: GT structure + * + * Walk the available HW engines from gt->info.engine_mask and calculate data + * related to those engines that may be used by userspace. To be used whenever + * available engines change in runtime (e.g. with ccs_mode) or during + * initialization + */ +void xe_gt_record_user_engines(struct xe_gt *gt); + +void xe_gt_suspend_prepare(struct xe_gt *gt); +int xe_gt_suspend(struct xe_gt *gt); +void xe_gt_shutdown(struct xe_gt *gt); +int xe_gt_resume(struct xe_gt *gt); +void xe_gt_reset_async(struct xe_gt *gt); +int xe_gt_runtime_resume(struct xe_gt *gt); +int xe_gt_runtime_suspend(struct xe_gt *gt); +void xe_gt_sanitize(struct xe_gt *gt); +int xe_gt_sanitize_freq(struct xe_gt *gt); + +/** + * xe_gt_wait_for_reset - wait for gt's async reset to finalize. + * @gt: GT structure + * Return: + * %true if it waited for the work to finish execution, + * %false if there was no scheduled reset or it was done. + */ +static inline bool xe_gt_wait_for_reset(struct xe_gt *gt) +{ + return flush_work(>->reset.worker); +} + +/** + * xe_gt_reset - perform synchronous reset + * @gt: GT structure + * Return: + * %true if it waited for the reset to finish, + * %false if there was no scheduled reset. + */ +static inline bool xe_gt_reset(struct xe_gt *gt) +{ + xe_gt_reset_async(gt); + return xe_gt_wait_for_reset(gt); +} + +/** + * xe_gt_any_hw_engine_by_reset_domain - scan the list of engines and return the + * first that matches the same reset domain as @class + * @gt: GT structure + * @class: hw engine class to lookup + */ +struct xe_hw_engine * +xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, enum xe_engine_class class); + +/** + * xe_gt_any_hw_engine - scan the list of engines and return the + * first available + * @gt: GT structure + */ +struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt); + +struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, + enum xe_engine_class class, + u16 instance, + bool logical); + +static inline bool xe_gt_has_indirect_ring_state(struct xe_gt *gt) +{ + return gt->info.has_indirect_ring_state && + xe_device_uc_enabled(gt_to_xe(gt)); +} + +static inline bool xe_gt_is_main_type(struct xe_gt *gt) +{ + return gt->info.type == XE_GT_TYPE_MAIN; +} + +static inline bool xe_gt_is_media_type(struct xe_gt *gt) +{ + return gt->info.type == XE_GT_TYPE_MEDIA; +} + +static inline bool xe_gt_is_usm_hwe(struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + return hwe->class == XE_ENGINE_CLASS_COPY && + (gt->usm.paging_logical_mask & BIT(hwe->logical_instance)); +} + +/** + * xe_gt_recovery_pending() - GT recovery pending + * @gt: the &xe_gt + * + * Return: True if GT recovery in pending, False otherwise + */ +static inline bool xe_gt_recovery_pending(struct xe_gt *gt) +{ + return IS_SRIOV_VF(gt_to_xe(gt)) && + xe_gt_sriov_vf_recovery_pending(gt); +} + +/** + * xe_gt_supports_multi_queue() - Check if gt supports multi queue for the + * specified engine class. + * + * @gt: the GT object + * @class: hwe class type + * + * Return: true if the hw engine class supports multi queue, else false + */ +static inline bool xe_gt_supports_multi_queue(const struct xe_gt *gt, + enum xe_engine_class class) +{ + return gt->info.multi_queue_engine_class_mask & BIT(class); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c new file mode 100644 index 000000000..baee1f4a6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include "regs/xe_gt_regs.h" +#include "xe_assert.h" +#include "xe_gt.h" +#include "xe_gt_ccs_mode.h" +#include "xe_gt_printk.h" +#include "xe_gt_sysfs.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_sriov.h" +#include "xe_sriov_pf.h" + +static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines) +{ + u32 mode = CCS_MODE_CSLICE_0_3_MASK; /* disable all by default */ + int num_slices = hweight32(CCS_INSTANCES(gt)); + struct xe_device *xe = gt_to_xe(gt); + int width, cslice = 0; + u32 config = 0; + + xe_assert(xe, xe_gt_ccs_mode_enabled(gt)); + + xe_assert(xe, num_engines && num_engines <= num_slices); + xe_assert(xe, !(num_slices % num_engines)); + + /* + * Loop over all available slices and assign each a user engine. + * For example, if there are four compute slices available, the + * assignment of compute slices to compute engines would be, + * + * With 1 engine (ccs0): + * slice 0, 1, 2, 3: ccs0 + * + * With 2 engines (ccs0, ccs1): + * slice 0, 2: ccs0 + * slice 1, 3: ccs1 + * + * With 4 engines (ccs0, ccs1, ccs2, ccs3): + * slice 0: ccs0 + * slice 1: ccs1 + * slice 2: ccs2 + * slice 3: ccs3 + */ + for (width = num_slices / num_engines; width; width--) { + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) { + if (hwe->class != XE_ENGINE_CLASS_COMPUTE) + continue; + + if (hwe->logical_instance >= num_engines) + break; + + config |= BIT(hwe->instance) << XE_HW_ENGINE_CCS0; + + /* If a slice is fused off, leave disabled */ + while ((CCS_INSTANCES(gt) & BIT(cslice)) == 0) + cslice++; + + mode &= ~CCS_MODE_CSLICE(cslice, CCS_MODE_CSLICE_MASK); + mode |= CCS_MODE_CSLICE(cslice, hwe->instance); + cslice++; + } + } + + /* + * Mask bits need to be set for the register. Though only Xe2+ + * platforms require setting of mask bits, it won't harm for older + * platforms as these bits are unused there. + */ + mode |= CCS_MODE_CSLICE_0_3_MASK << 16; + xe_mmio_write32(>->mmio, CCS_MODE, mode); + + xe_gt_dbg(gt, "CCS_MODE=%x config:%08x, num_engines:%d, num_slices:%d\n", + mode, config, num_engines, num_slices); +} + +void xe_gt_apply_ccs_mode(struct xe_gt *gt) +{ + if (!gt->ccs_mode || IS_SRIOV_VF(gt_to_xe(gt))) + return; + + __xe_gt_apply_ccs_mode(gt, gt->ccs_mode); +} + +static bool gt_ccs_mode_default(struct xe_gt *gt) +{ + return gt->ccs_mode == 1; +} + +static ssize_t +num_cslices_show(struct device *kdev, + struct device_attribute *attr, char *buf) +{ + struct xe_gt *gt = kobj_to_gt(&kdev->kobj); + + return sysfs_emit(buf, "%u\n", hweight32(CCS_INSTANCES(gt))); +} + +static DEVICE_ATTR_RO(num_cslices); + +static ssize_t +ccs_mode_show(struct device *kdev, + struct device_attribute *attr, char *buf) +{ + struct xe_gt *gt = kobj_to_gt(&kdev->kobj); + + return sysfs_emit(buf, "%u\n", gt->ccs_mode); +} + +static ssize_t +ccs_mode_store(struct device *kdev, struct device_attribute *attr, + const char *buff, size_t count) +{ + struct xe_gt *gt = kobj_to_gt(&kdev->kobj); + struct xe_device *xe = gt_to_xe(gt); + u32 num_engines, num_slices; + int ret; + + ret = kstrtou32(buff, 0, &num_engines); + if (ret) + return ret; + + /* + * Ensure number of engines specified is valid and there is an + * exact multiple of engines for slices. + */ + num_slices = hweight32(CCS_INSTANCES(gt)); + if (!num_engines || num_engines > num_slices || num_slices % num_engines) { + xe_gt_dbg(gt, "Invalid compute config, %d engines %d slices\n", + num_engines, num_slices); + return -EINVAL; + } + + /* CCS mode can only be updated when there are no drm clients */ + guard(mutex)(&xe->drm.filelist_mutex); + if (!list_empty(&xe->drm.filelist)) { + xe_gt_dbg(gt, "Rejecting compute mode change as there are active drm clients\n"); + return -EBUSY; + } + + if (gt->ccs_mode == num_engines) + return count; + + /* + * Changing default CCS mode is only allowed when there + * are no VFs. Try to lockdown PF to find out. + */ + if (gt_ccs_mode_default(gt) && IS_SRIOV_PF(xe)) { + ret = xe_sriov_pf_lockdown(xe); + if (ret) { + xe_gt_dbg(gt, "Can't change CCS Mode: VFs are enabled\n"); + return ret; + } + } + + xe_gt_info(gt, "Setting compute mode to %d\n", num_engines); + gt->ccs_mode = num_engines; + xe_gt_record_user_engines(gt); + guard(xe_pm_runtime)(xe); + xe_gt_reset(gt); + + /* We may end PF lockdown once CCS mode is default again */ + if (gt_ccs_mode_default(gt) && IS_SRIOV_PF(xe)) + xe_sriov_pf_end_lockdown(xe); + + return count; +} + +static DEVICE_ATTR_RW(ccs_mode); + +static const struct attribute *gt_ccs_mode_attrs[] = { + &dev_attr_ccs_mode.attr, + &dev_attr_num_cslices.attr, + NULL, +}; + +static void xe_gt_ccs_mode_sysfs_fini(void *arg) +{ + struct xe_gt *gt = arg; + + sysfs_remove_files(gt->sysfs, gt_ccs_mode_attrs); +} + +/** + * xe_gt_ccs_mode_sysfs_init - Initialize CCS mode sysfs interfaces + * @gt: GT structure + * + * Through a per-gt 'ccs_mode' sysfs interface, the user can enable a fixed + * number of compute hardware engines to which the available compute slices + * are to be allocated. This user configuration change triggers a gt reset + * and it is expected that there are no open drm clients while doing so. + * The number of available compute slices is exposed to user through a per-gt + * 'num_cslices' sysfs interface. + * + * Returns: Returns error value for failure and 0 for success. + */ +int xe_gt_ccs_mode_sysfs_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + if (!xe_gt_ccs_mode_enabled(gt) || IS_SRIOV_VF(xe)) + return 0; + + err = sysfs_create_files(gt->sysfs, gt_ccs_mode_attrs); + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, xe_gt_ccs_mode_sysfs_fini, gt); +} diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.h b/drivers/gpu/drm/xe/xe_gt_ccs_mode.h new file mode 100644 index 000000000..ef3b853f5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_CCS_MODE_H_ +#define _XE_GT_CCS_MODE_H_ + +#include "xe_device_types.h" +#include "xe_gt.h" +#include "xe_gt_types.h" +#include "xe_platform_types.h" + +void xe_gt_apply_ccs_mode(struct xe_gt *gt); +int xe_gt_ccs_mode_sysfs_init(struct xe_gt *gt); + +static inline bool xe_gt_ccs_mode_enabled(const struct xe_gt *gt) +{ + /* Check if there are more than one compute engines available */ + return hweight32(CCS_INSTANCES(gt)) > 1; +} + +#endif + diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c new file mode 100644 index 000000000..53b383519 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_clock.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_gt_clock.h" + +#include "regs/xe_gt_regs.h" +#include "xe_gt_types.h" +#include "xe_gt_printk.h" +#include "xe_mmio.h" + +#define f19_2_mhz 19200000 +#define f24_mhz 24000000 +#define f25_mhz 25000000 +#define f38_4_mhz 38400000 +#define ts_base_83 83333 +#define ts_base_52 52083 +#define ts_base_80 80000 + +static void read_crystal_clock(struct xe_gt *gt, u32 rpm_config_reg, u32 *freq, + u32 *timestamp_base) +{ + u32 crystal_clock = REG_FIELD_GET(RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK, + rpm_config_reg); + + switch (crystal_clock) { + case RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ: + *freq = f24_mhz; + *timestamp_base = ts_base_83; + return; + case RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ: + *freq = f19_2_mhz; + *timestamp_base = ts_base_52; + return; + case RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ: + *freq = f38_4_mhz; + *timestamp_base = ts_base_52; + return; + case RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ: + *freq = f25_mhz; + *timestamp_base = ts_base_80; + return; + default: + xe_gt_warn(gt, "Invalid crystal clock frequency: %u", crystal_clock); + *freq = 0; + *timestamp_base = 0; + return; + } +} + +int xe_gt_clock_init(struct xe_gt *gt) +{ + u32 freq; + u32 c0; + + c0 = xe_mmio_read32(>->mmio, RPM_CONFIG0); + read_crystal_clock(gt, c0, &freq, >->info.timestamp_base); + + /* + * Now figure out how the command stream's timestamp + * register increments from this frequency (it might + * increment only every few clock cycle). + */ + freq >>= 3 - REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, c0); + + gt->info.reference_clock = freq; + return 0; +} + +/** + * xe_gt_clock_interval_to_ms - Convert sampled GT clock ticks to msec + * + * @gt: the &xe_gt + * @count: count of GT clock ticks + * + * Returns: time in msec + */ +u64 xe_gt_clock_interval_to_ms(struct xe_gt *gt, u64 count) +{ + return mul_u64_u32_div(count, MSEC_PER_SEC, gt->info.reference_clock); +} diff --git a/drivers/gpu/drm/xe/xe_gt_clock.h b/drivers/gpu/drm/xe/xe_gt_clock.h new file mode 100644 index 000000000..3adeb7baa --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_clock.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_CLOCK_H_ +#define _XE_GT_CLOCK_H_ + +#include + +struct xe_gt; + +int xe_gt_clock_init(struct xe_gt *gt); +u64 xe_gt_clock_interval_to_ms(struct xe_gt *gt, u64 count); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c new file mode 100644 index 000000000..c38bcacb2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gt_debugfs.h" + +#include + +#include +#include + +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_idle.h" +#include "xe_gt_sriov_pf_debugfs.h" +#include "xe_gt_sriov_vf_debugfs.h" +#include "xe_gt_stats.h" +#include "xe_gt_topology.h" +#include "xe_guc_hwconfig.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_mocs.h" +#include "xe_pat.h" +#include "xe_pm.h" +#include "xe_reg_sr.h" +#include "xe_reg_whitelist.h" +#include "xe_sa.h" +#include "xe_sriov.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_tuning.h" +#include "xe_uc_debugfs.h" +#include "xe_wa.h" + +static struct xe_gt *node_to_gt(struct drm_info_node *node) +{ + return node->dent->d_parent->d_inode->i_private; +} + +/** + * xe_gt_debugfs_simple_show - A show callback for struct drm_info_list + * @m: the &seq_file + * @data: data used by the drm debugfs helpers + * + * This callback can be used in struct drm_info_list to describe debugfs + * files that are &xe_gt specific. + * + * It is assumed that those debugfs files will be created on directory entry + * which struct dentry d_inode->i_private points to &xe_gt. + * + * This function assumes that &m->private will be set to the &struct + * drm_info_node corresponding to the instance of the info on a given &struct + * drm_minor (see struct drm_info_list.show for details). + * + * This function also assumes that struct drm_info_list.data will point to the + * function code that will actually print a file content:: + * + * int (*print)(struct xe_gt *, struct drm_printer *) + * + * Example:: + * + * int foo(struct xe_gt *gt, struct drm_printer *p) + * { + * drm_printf(p, "GT%u\n", gt->info.id); + * return 0; + * } + * + * static const struct drm_info_list bar[] = { + * { name = "foo", .show = xe_gt_debugfs_simple_show, .data = foo }, + * }; + * + * dir = debugfs_create_dir("gt", parent); + * dir->d_inode->i_private = gt; + * drm_debugfs_create_files(bar, ARRAY_SIZE(bar), dir, minor); + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_debugfs_simple_show(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + struct drm_info_node *node = m->private; + struct xe_gt *gt = node_to_gt(node); + int (*print)(struct xe_gt *, struct drm_printer *) = node->info_ent->data; + + if (WARN_ON(!print)) + return -EINVAL; + + return print(gt, &p); +} + +/** + * xe_gt_debugfs_show_with_rpm - A show callback for struct drm_info_list + * @m: the &seq_file + * @data: data used by the drm debugfs helpers + * + * Similar to xe_gt_debugfs_simple_show() but implicitly takes a RPM ref. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_debugfs_show_with_rpm(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct xe_gt *gt = node_to_gt(node); + struct xe_device *xe = gt_to_xe(gt); + + guard(xe_pm_runtime)(xe); + return xe_gt_debugfs_simple_show(m, data); +} + +static int hw_engines(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + return -ETIMEDOUT; + + for_each_hw_engine(hwe, gt, id) + xe_hw_engine_print(hwe, p); + + return 0; +} + +static int steering(struct xe_gt *gt, struct drm_printer *p) +{ + xe_gt_mcr_steering_dump(gt, p); + return 0; +} + +static int register_save_restore(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + xe_reg_sr_dump(>->reg_sr, p); + drm_printf(p, "\n"); + + drm_printf(p, "Engine\n"); + for_each_hw_engine(hwe, gt, id) + xe_reg_sr_dump(&hwe->reg_sr, p); + drm_printf(p, "\n"); + + drm_printf(p, "LRC\n"); + for_each_hw_engine(hwe, gt, id) + xe_reg_sr_dump(&hwe->reg_lrc, p); + drm_printf(p, "\n"); + + drm_printf(p, "Whitelist\n"); + for_each_hw_engine(hwe, gt, id) { + xe_reg_whitelist_dump(&hwe->reg_whitelist, p); + xe_reg_whitelist_dump(&hwe->oa_whitelist, p); + } + + return 0; +} + +/* + * Check the registers referenced on a save-restore list and report any + * save-restore entries that did not get applied. + */ +static int register_save_restore_check(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + drm_printf(p, "ERROR: Could not acquire forcewake\n"); + return -ETIMEDOUT; + } + + xe_reg_sr_readback_check(>->reg_sr, gt, p); + for_each_hw_engine(hwe, gt, id) + xe_reg_sr_readback_check(&hwe->reg_sr, gt, p); + for_each_hw_engine(hwe, gt, id) + xe_reg_sr_lrc_check(&hwe->reg_lrc, gt, hwe, p); + + return 0; +} + +static int rcs_default_lrc(struct xe_gt *gt, struct drm_printer *p) +{ + xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_RENDER); + return 0; +} + +static int ccs_default_lrc(struct xe_gt *gt, struct drm_printer *p) +{ + xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_COMPUTE); + return 0; +} + +static int bcs_default_lrc(struct xe_gt *gt, struct drm_printer *p) +{ + xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_COPY); + return 0; +} + +static int vcs_default_lrc(struct xe_gt *gt, struct drm_printer *p) +{ + xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_VIDEO_DECODE); + return 0; +} + +static int vecs_default_lrc(struct xe_gt *gt, struct drm_printer *p) +{ + xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); + return 0; +} + +static int hwconfig(struct xe_gt *gt, struct drm_printer *p) +{ + xe_guc_hwconfig_dump(>->uc.guc, p); + return 0; +} + +/* + * only for GT debugfs files which can be safely used on the VF as well: + * - without access to the GT privileged registers + * - without access to the PF specific data + */ +static const struct drm_info_list vf_safe_debugfs_list[] = { + { "topology", .show = xe_gt_debugfs_show_with_rpm, .data = xe_gt_topology_dump }, + { "register-save-restore", + .show = xe_gt_debugfs_show_with_rpm, .data = register_save_restore }, + { "workarounds", .show = xe_gt_debugfs_show_with_rpm, .data = xe_wa_gt_dump }, + { "tunings", .show = xe_gt_debugfs_show_with_rpm, .data = xe_tuning_dump }, + { "default_lrc_rcs", .show = xe_gt_debugfs_show_with_rpm, .data = rcs_default_lrc }, + { "default_lrc_ccs", .show = xe_gt_debugfs_show_with_rpm, .data = ccs_default_lrc }, + { "default_lrc_bcs", .show = xe_gt_debugfs_show_with_rpm, .data = bcs_default_lrc }, + { "default_lrc_vcs", .show = xe_gt_debugfs_show_with_rpm, .data = vcs_default_lrc }, + { "default_lrc_vecs", .show = xe_gt_debugfs_show_with_rpm, .data = vecs_default_lrc }, + { "hwconfig", .show = xe_gt_debugfs_show_with_rpm, .data = hwconfig }, + { "pat_sw_config", .show = xe_gt_debugfs_simple_show, .data = xe_pat_dump_sw_config }, + { "register-save-restore-check", + .show = xe_gt_debugfs_show_with_rpm, .data = register_save_restore_check }, +}; + +/* everything else should be added here */ +static const struct drm_info_list pf_only_debugfs_list[] = { + { "hw_engines", .show = xe_gt_debugfs_show_with_rpm, .data = hw_engines }, + { "mocs", .show = xe_gt_debugfs_show_with_rpm, .data = xe_mocs_dump }, + { "pat", .show = xe_gt_debugfs_show_with_rpm, .data = xe_pat_dump }, + { "powergate_info", .show = xe_gt_debugfs_show_with_rpm, .data = xe_gt_idle_pg_print }, + { "steering", .show = xe_gt_debugfs_show_with_rpm, .data = steering }, +}; + +static ssize_t write_to_gt_call(const char __user *userbuf, size_t count, loff_t *ppos, + void (*call)(struct xe_gt *), struct xe_gt *gt) +{ + bool yes; + int ret; + + if (*ppos) + return -EINVAL; + ret = kstrtobool_from_user(userbuf, count, &yes); + if (ret < 0) + return ret; + if (yes) + call(gt); + return count; +} + +static ssize_t stats_write(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct xe_gt *gt = s->private; + + return write_to_gt_call(userbuf, count, ppos, xe_gt_stats_clear, gt); +} + +static int stats_show(struct seq_file *s, void *unused) +{ + struct drm_printer p = drm_seq_file_printer(s); + struct xe_gt *gt = s->private; + + return xe_gt_stats_print_info(gt, &p); +} +DEFINE_SHOW_STORE_ATTRIBUTE(stats); + +static void force_reset(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + guard(xe_pm_runtime)(xe); + xe_gt_reset_async(gt); +} + +static ssize_t force_reset_write(struct file *file, + const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct xe_gt *gt = s->private; + + return write_to_gt_call(userbuf, count, ppos, force_reset, gt); +} + +static int force_reset_show(struct seq_file *s, void *unused) +{ + struct xe_gt *gt = s->private; + + force_reset(gt); /* to be deprecated! */ + return 0; +} +DEFINE_SHOW_STORE_ATTRIBUTE(force_reset); + +static void force_reset_sync(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + guard(xe_pm_runtime)(xe); + xe_gt_reset(gt); +} + +static ssize_t force_reset_sync_write(struct file *file, + const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct xe_gt *gt = s->private; + + return write_to_gt_call(userbuf, count, ppos, force_reset_sync, gt); +} + +static int force_reset_sync_show(struct seq_file *s, void *unused) +{ + struct xe_gt *gt = s->private; + + force_reset_sync(gt); /* to be deprecated! */ + return 0; +} +DEFINE_SHOW_STORE_ATTRIBUTE(force_reset_sync); + +void xe_gt_debugfs_register(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct drm_minor *minor = gt_to_xe(gt)->drm.primary; + struct dentry *parent = gt->tile->debugfs; + struct dentry *root; + char symlink[16]; + char name[8]; + + xe_gt_assert(gt, minor->debugfs_root); + + if (IS_ERR(parent)) + return; + + snprintf(name, sizeof(name), "gt%d", gt->info.id); + root = debugfs_create_dir(name, parent); + if (IS_ERR(root)) { + drm_warn(&xe->drm, "Create GT directory failed"); + return; + } + + /* + * Store the xe_gt pointer as private data of the gt/ directory node + * so other GT specific attributes under that directory may refer to + * it by looking at its parent node private data. + */ + root->d_inode->i_private = gt; + + /* VF safe */ + debugfs_create_file("stats", 0600, root, gt, &stats_fops); + debugfs_create_file("force_reset", 0600, root, gt, &force_reset_fops); + debugfs_create_file("force_reset_sync", 0600, root, gt, &force_reset_sync_fops); + + drm_debugfs_create_files(vf_safe_debugfs_list, + ARRAY_SIZE(vf_safe_debugfs_list), + root, minor); + + if (!IS_SRIOV_VF(xe)) + drm_debugfs_create_files(pf_only_debugfs_list, + ARRAY_SIZE(pf_only_debugfs_list), + root, minor); + + xe_uc_debugfs_register(>->uc, root); + + if (IS_SRIOV_PF(xe)) + xe_gt_sriov_pf_debugfs_register(gt, root); + else if (IS_SRIOV_VF(xe)) + xe_gt_sriov_vf_debugfs_register(gt, root); + + /* + * Backwards compatibility only: create a link for the legacy clients + * who may expect gt/ directory at the root level, not the tile level. + */ + snprintf(symlink, sizeof(symlink), "tile%u/%s", gt->tile->id, name); + debugfs_create_symlink(name, minor->debugfs_root, symlink); +} diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.h b/drivers/gpu/drm/xe/xe_gt_debugfs.h new file mode 100644 index 000000000..32ee32640 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_DEBUGFS_H_ +#define _XE_GT_DEBUGFS_H_ + +struct seq_file; +struct xe_gt; + +void xe_gt_debugfs_register(struct xe_gt *gt); +int xe_gt_debugfs_simple_show(struct seq_file *m, void *data); +int xe_gt_debugfs_show_with_rpm(struct seq_file *m, void *data); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_freq.c b/drivers/gpu/drm/xe/xe_gt_freq.c new file mode 100644 index 000000000..a40dd0741 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_freq.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gt_freq.h" + +#include +#include + +#include +#include + +#include "xe_gt_sysfs.h" +#include "xe_gt_throttle.h" +#include "xe_gt_types.h" +#include "xe_guc_pc.h" +#include "xe_pm.h" + +/** + * DOC: Xe GT Frequency Management + * + * This component is responsible for the raw GT frequency management, including + * the sysfs API. + * + * Underneath, Xe enables GuC SLPC automated frequency management. GuC is then + * allowed to request PCODE any frequency between the Minimum and the Maximum + * selected by this component. Furthermore, it is important to highlight that + * PCODE is the ultimate decision maker of the actual running frequency, based + * on thermal and other running conditions. + * + * Xe's Freq provides a sysfs API for frequency management under + * ``/tile#/gt#/freq0/`` directory. + * + * **Read-only** attributes: + * + * - ``act_freq``: The actual resolved frequency decided by PCODE. + * - ``cur_freq``: The current one requested by GuC PC to the PCODE. + * - ``rpn_freq``: The Render Performance (RP) N level, which is the minimal one. + * - ``rpa_freq``: The Render Performance (RP) A level, which is the achievable one. + * Calculated by PCODE at runtime based on multiple running conditions + * - ``rpe_freq``: The Render Performance (RP) E level, which is the efficient one. + * Calculated by PCODE at runtime based on multiple running conditions + * - ``rp0_freq``: The Render Performance (RP) 0 level, which is the maximum one. + * + * **Read-write** attributes: + * + * - ``min_freq``: Min frequency request. + * - ``max_freq``: Max frequency request. + * If max <= min, then freq_min becomes a fixed frequency + * request. + */ + +static struct xe_guc_pc * +dev_to_pc(struct device *dev) +{ + return &kobj_to_gt(dev->kobj.parent)->uc.guc.pc; +} + +static struct xe_device * +dev_to_xe(struct device *dev) +{ + return gt_to_xe(kobj_to_gt(dev->kobj.parent)); +} + +static ssize_t act_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + freq = xe_guc_pc_get_act_freq(pc); + + return sysfs_emit(buf, "%d\n", freq); +} +static struct kobj_attribute attr_act_freq = __ATTR_RO(act_freq); + +static ssize_t cur_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + ret = xe_guc_pc_get_cur_freq(pc, &freq); + if (ret) + return ret; + + return sysfs_emit(buf, "%d\n", freq); +} +static struct kobj_attribute attr_cur_freq = __ATTR_RO(cur_freq); + +static ssize_t rp0_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + + return sysfs_emit(buf, "%d\n", xe_guc_pc_get_rp0_freq(pc)); +} +static struct kobj_attribute attr_rp0_freq = __ATTR_RO(rp0_freq); + +static ssize_t rpe_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + freq = xe_guc_pc_get_rpe_freq(pc); + + return sysfs_emit(buf, "%d\n", freq); +} +static struct kobj_attribute attr_rpe_freq = __ATTR_RO(rpe_freq); + +static ssize_t rpa_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + freq = xe_guc_pc_get_rpa_freq(pc); + + return sysfs_emit(buf, "%d\n", freq); +} +static struct kobj_attribute attr_rpa_freq = __ATTR_RO(rpa_freq); + +static ssize_t rpn_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + + return sysfs_emit(buf, "%d\n", xe_guc_pc_get_rpn_freq(pc)); +} +static struct kobj_attribute attr_rpn_freq = __ATTR_RO(rpn_freq); + +static ssize_t min_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + ret = xe_guc_pc_get_min_freq(pc, &freq); + if (ret) + return ret; + + return sysfs_emit(buf, "%d\n", freq); +} + +static ssize_t min_freq_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buff, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + ret = kstrtou32(buff, 0, &freq); + if (ret) + return ret; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + ret = xe_guc_pc_set_min_freq(pc, freq); + if (ret) + return ret; + + return count; +} +static struct kobj_attribute attr_min_freq = __ATTR_RW(min_freq); + +static ssize_t max_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + ret = xe_guc_pc_get_max_freq(pc, &freq); + if (ret) + return ret; + + return sysfs_emit(buf, "%d\n", freq); +} + +static ssize_t max_freq_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buff, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + ret = kstrtou32(buff, 0, &freq); + if (ret) + return ret; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + ret = xe_guc_pc_set_max_freq(pc, freq); + if (ret) + return ret; + + return count; +} +static struct kobj_attribute attr_max_freq = __ATTR_RW(max_freq); + +static ssize_t power_profile_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buff) +{ + struct device *dev = kobj_to_dev(kobj); + + xe_guc_pc_get_power_profile(dev_to_pc(dev), buff); + + return strlen(buff); +} + +static ssize_t power_profile_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_guc_pc *pc = dev_to_pc(dev); + int err; + + guard(xe_pm_runtime)(dev_to_xe(dev)); + err = xe_guc_pc_set_power_profile(pc, buff); + + return err ?: count; +} +static struct kobj_attribute attr_power_profile = __ATTR_RW(power_profile); + +static const struct attribute *freq_attrs[] = { + &attr_act_freq.attr, + &attr_cur_freq.attr, + &attr_rp0_freq.attr, + &attr_rpa_freq.attr, + &attr_rpe_freq.attr, + &attr_rpn_freq.attr, + &attr_min_freq.attr, + &attr_max_freq.attr, + &attr_power_profile.attr, + NULL +}; + +static void freq_fini(void *arg) +{ + struct kobject *kobj = arg; + + sysfs_remove_files(kobj, freq_attrs); + kobject_put(kobj); +} + +/** + * xe_gt_freq_init - Initialize Xe Freq component + * @gt: Xe GT object + * + * It needs to be initialized after GT Sysfs and GuC PC components are ready. + * + * Returns: Returns error value for failure and 0 for success. + */ +int xe_gt_freq_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + if (xe->info.skip_guc_pc) + return 0; + + gt->freq = kobject_create_and_add("freq0", gt->sysfs); + if (!gt->freq) + return -ENOMEM; + + err = sysfs_create_files(gt->freq, freq_attrs); + if (err) { + kobject_put(gt->freq); + return err; + } + + err = devm_add_action_or_reset(xe->drm.dev, freq_fini, gt->freq); + if (err) + return err; + + return xe_gt_throttle_init(gt); +} diff --git a/drivers/gpu/drm/xe/xe_gt_freq.h b/drivers/gpu/drm/xe/xe_gt_freq.h new file mode 100644 index 000000000..b7fddbe7b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_freq.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_FREQ_H_ +#define _XE_GT_FREQ_H_ + +struct xe_gt; + +int xe_gt_freq_init(struct xe_gt *gt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c new file mode 100644 index 000000000..7dc9873ae --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include + +#include +#include "xe_force_wake.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_idle.h" +#include "xe_gt_sysfs.h" +#include "xe_guc_pc.h" +#include "regs/xe_gt_regs.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_sriov.h" +#include "xe_wa.h" + +/** + * DOC: Xe GT Idle + * + * Contains functions that init GT idle features like C6 + * + * device/gt#/gtidle/name - name of the state + * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms + * device/gt#/gtidle/idle_status - Provides current idle state + */ + +static struct xe_gt_idle *dev_to_gtidle(struct device *dev) +{ + struct kobject *kobj = &dev->kobj; + + return &kobj_to_gt(kobj->parent)->gtidle; +} + +static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle) +{ + return container_of(gtidle, struct xe_gt, gtidle); +} + +static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle) +{ + return >idle_to_gt(gtidle)->uc.guc.pc; +} + +static struct xe_device * +pc_to_xe(struct xe_guc_pc *pc) +{ + struct xe_guc *guc = container_of(pc, struct xe_guc, pc); + struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc); + + return gt_to_xe(gt); +} + +static const char *gt_idle_state_to_string(enum xe_gt_idle_state state) +{ + switch (state) { + case GT_IDLE_C0: + return "gt-c0"; + case GT_IDLE_C6: + return "gt-c6"; + default: + return "unknown"; + } +} + +static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) +{ + u64 delta, overflow_residency, prev_residency; + + lockdep_assert_held(>idle->lock); + + overflow_residency = BIT_ULL(32); + + /* + * Counter wrap handling + * Store previous hw counter values for counter wrap-around handling + * Relying on sufficient frequency of queries otherwise counters can still wrap. + */ + prev_residency = gtidle->prev_residency; + gtidle->prev_residency = cur_residency; + + /* delta */ + if (cur_residency >= prev_residency) + delta = cur_residency - prev_residency; + else + delta = cur_residency + (overflow_residency - prev_residency); + + /* Add delta to extended raw driver copy of idle residency */ + cur_residency = gtidle->cur_residency + delta; + gtidle->cur_residency = cur_residency; + + /* residency multiplier in ns, convert to ms */ + cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, NSEC_PER_MSEC); + + return cur_residency; +} + +void xe_gt_idle_enable_pg(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_gt_idle *gtidle = >->gtidle; + struct xe_mmio *mmio = >->mmio; + u32 vcs_mask, vecs_mask; + int i, j; + + if (IS_SRIOV_VF(xe)) + return; + + /* Disable CPG for PVC */ + if (xe->info.platform == XE_PVC) + return; + + xe_device_assert_mem_access(gt_to_xe(gt)); + + vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); + vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); + + if (vcs_mask || vecs_mask) + gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE; + + if (xe_gt_is_main_type(gt)) + gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE; + + if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) + gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE; + + if (xe->info.platform != XE_DG1) { + for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { + if ((gt->info.engine_mask & BIT(i))) + gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) | + VDN_MFXVDENC_POWERGATE_ENABLE(j)); + } + } + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (xe->info.skip_guc_pc) { + /* + * GuC sets the hysteresis value when GuC PC is enabled + * else set it to 25 (25 * 1.28us) + */ + xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25); + xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); + } + + if (XE_GT_WA(gt, 14020316580)) + gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) | + VDN_MFXVDENC_POWERGATE_ENABLE(0) | + VDN_HCP_POWERGATE_ENABLE(2) | + VDN_MFXVDENC_POWERGATE_ENABLE(2)); + + xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); +} + +void xe_gt_idle_disable_pg(struct xe_gt *gt) +{ + struct xe_gt_idle *gtidle = >->gtidle; + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return; + + xe_device_assert_mem_access(gt_to_xe(gt)); + gtidle->powergate_enable = 0; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable); +} + +static void force_wake_domains_show(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_force_wake_domain *domain; + struct xe_force_wake *fw = gt_to_fw(gt); + unsigned int tmp; + unsigned long flags; + + spin_lock_irqsave(&fw->lock, flags); + for_each_fw_domain(domain, fw, tmp) { + drm_printf(p, "%s.ref_count=%u, %s.fwake=0x%x\n", + xe_force_wake_domain_to_str(domain->id), + READ_ONCE(domain->ref), + xe_force_wake_domain_to_str(domain->id), + xe_mmio_read32(>->mmio, domain->reg_ctl)); + } + spin_unlock_irqrestore(&fw->lock, flags); +} + +/** + * xe_gt_idle_pg_print - Xe powergating info + * @gt: GT object + * @p: drm_printer. + * + * This function prints the powergating information + * + * Return: 0 on success, negative error code otherwise + */ +int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_gt_idle *gtidle = >->gtidle; + struct xe_device *xe = gt_to_xe(gt); + enum xe_gt_idle_state state; + u32 pg_enabled, pg_status = 0; + u32 vcs_mask, vecs_mask; + int n; + /* + * Media Slices + * + * Slice 0: VCS0, VCS1, VECS0 + * Slice 1: VCS2, VCS3, VECS1 + * Slice 2: VCS4, VCS5, VECS2 + * Slice 3: VCS6, VCS7, VECS3 + */ + static const struct { + u64 engines; + u32 status_bit; + } media_slices[] = { + {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) | + BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS}, + + {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) | + BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS}, + + {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) | + BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS}, + + {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) | + BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS}, + }; + + if (xe->info.platform == XE_PVC) { + drm_printf(p, "Power Gating not supported\n"); + return 0; + } + + state = gtidle->idle_status(gtidle_to_pc(gtidle)); + pg_enabled = gtidle->powergate_enable; + + /* Do not wake the GT to read powergating status */ + if (state != GT_IDLE_C6) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE); + pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); + } + + if (gt->info.engine_mask & + (XE_HW_ENGINE_RCS_MASK | XE_HW_ENGINE_CCS_MASK)) { + drm_printf(p, "Render Power Gating Enabled: %s\n", + str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); + + drm_printf(p, "Render Power Gate Status: %s\n", + str_up_down(pg_status & RENDER_AWAKE_STATUS)); + } + + vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); + vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); + + /* Print media CPG status only if media is present */ + if (vcs_mask || vecs_mask) { + drm_printf(p, "Media Power Gating Enabled: %s\n", + str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE)); + + for (n = 0; n < ARRAY_SIZE(media_slices); n++) + if (gt->info.engine_mask & media_slices[n].engines) + drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n, + str_up_down(pg_status & media_slices[n].status_bit)); + } + + if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) + drm_printf(p, "Media Samplers Power Gating Enabled: %s\n", + str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE)); + + if (gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)) { + drm_printf(p, "GSC Power Gate Status: %s\n", + str_up_down(pg_status & GSC_AWAKE_STATUS)); + } + + force_wake_domains_show(gt, p); + + return 0; +} + +static ssize_t name_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_gt_idle *gtidle = dev_to_gtidle(dev); + struct xe_guc_pc *pc = gtidle_to_pc(gtidle); + + guard(xe_pm_runtime)(pc_to_xe(pc)); + return sysfs_emit(buff, "%s\n", gtidle->name); +} +static struct kobj_attribute name_attr = __ATTR_RO(name); + +static ssize_t idle_status_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_gt_idle *gtidle = dev_to_gtidle(dev); + struct xe_guc_pc *pc = gtidle_to_pc(gtidle); + enum xe_gt_idle_state state; + + scoped_guard(xe_pm_runtime, pc_to_xe(pc)) + state = gtidle->idle_status(pc); + + return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state)); +} +static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status); + +u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle) +{ + struct xe_guc_pc *pc = gtidle_to_pc(gtidle); + u64 residency; + unsigned long flags; + + raw_spin_lock_irqsave(>idle->lock, flags); + residency = get_residency_ms(gtidle, gtidle->idle_residency(pc)); + raw_spin_unlock_irqrestore(>idle->lock, flags); + + return residency; +} + + +static ssize_t idle_residency_ms_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_gt_idle *gtidle = dev_to_gtidle(dev); + struct xe_guc_pc *pc = gtidle_to_pc(gtidle); + u64 residency; + + scoped_guard(xe_pm_runtime, pc_to_xe(pc)) + residency = xe_gt_idle_residency_msec(gtidle); + + return sysfs_emit(buff, "%llu\n", residency); +} +static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms); + +static const struct attribute *gt_idle_attrs[] = { + &name_attr.attr, + &idle_status_attr.attr, + &idle_residency_attr.attr, + NULL, +}; + +static void gt_idle_fini(void *arg) +{ + struct kobject *kobj = arg; + struct xe_gt *gt = kobj_to_gt(kobj->parent); + + xe_gt_idle_disable_pg(gt); + + if (gt_to_xe(gt)->info.skip_guc_pc) + xe_gt_idle_disable_c6(gt); + + sysfs_remove_files(kobj, gt_idle_attrs); + kobject_put(kobj); +} + +int xe_gt_idle_init(struct xe_gt_idle *gtidle) +{ + struct xe_gt *gt = gtidle_to_gt(gtidle); + struct xe_device *xe = gt_to_xe(gt); + struct kobject *kobj; + int err; + + if (IS_SRIOV_VF(xe)) + return 0; + + kobj = kobject_create_and_add("gtidle", gt->sysfs); + if (!kobj) + return -ENOMEM; + + raw_spin_lock_init(>idle->lock); + + if (xe_gt_is_media_type(gt)) { + snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id); + gtidle->idle_residency = xe_guc_pc_mc6_residency; + } else { + snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id); + gtidle->idle_residency = xe_guc_pc_rc6_residency; + } + + /* Multiplier for Residency counter in units of 1.28us */ + gtidle->residency_multiplier = 1280; + gtidle->idle_status = xe_guc_pc_c_status; + + err = sysfs_create_files(kobj, gt_idle_attrs); + if (err) { + kobject_put(kobj); + return err; + } + + xe_gt_idle_enable_pg(gt); + + return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj); +} + +void xe_gt_idle_enable_c6(struct xe_gt *gt) +{ + xe_device_assert_mem_access(gt_to_xe(gt)); + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return; + + /* Units of 1280 ns for a total of 5s */ + xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA); + /* Enable RC6 */ + xe_mmio_write32(>->mmio, RC_CONTROL, + RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE); +} + +int xe_gt_idle_disable_c6(struct xe_gt *gt) +{ + xe_device_assert_mem_access(gt_to_xe(gt)); + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return 0; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + xe_mmio_write32(>->mmio, RC_CONTROL, 0); + xe_mmio_write32(>->mmio, RC_STATE, 0); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_idle.h b/drivers/gpu/drm/xe/xe_gt_idle.h new file mode 100644 index 000000000..9c34a155e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_idle.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_IDLE_H_ +#define _XE_GT_IDLE_H_ + +#include "xe_gt_idle_types.h" + +struct drm_printer; +struct xe_gt; + +int xe_gt_idle_init(struct xe_gt_idle *gtidle); +void xe_gt_idle_enable_c6(struct xe_gt *gt); +int xe_gt_idle_disable_c6(struct xe_gt *gt); +void xe_gt_idle_enable_pg(struct xe_gt *gt); +void xe_gt_idle_disable_pg(struct xe_gt *gt); +int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p); +u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle); + +#endif /* _XE_GT_IDLE_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_idle_types.h b/drivers/gpu/drm/xe/xe_gt_idle_types.h new file mode 100644 index 000000000..5b5d2e835 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_idle_types.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_IDLE_TYPES_H_ +#define _XE_GT_IDLE_TYPES_H_ + +#include +#include + +struct xe_guc_pc; + +/* States of GT Idle */ +enum xe_gt_idle_state { + GT_IDLE_C0, + GT_IDLE_C6, + GT_IDLE_UNKNOWN, +}; + +/** + * struct xe_gt_idle - A struct that contains idle properties based of gt + */ +struct xe_gt_idle { + /** @name: name */ + char name[16]; + /** @powergate_enable: copy of powergate enable bits */ + u32 powergate_enable; + /** @residency_multiplier: residency multiplier in ns */ + u32 residency_multiplier; + /** @cur_residency: raw driver copy of idle residency */ + u64 cur_residency; + /** @prev_residency: previous residency counter */ + u64 prev_residency; + /** @lock: Lock protecting idle residency counters */ + raw_spinlock_t lock; + /** @idle_status: get the current idle state */ + enum xe_gt_idle_state (*idle_status)(struct xe_guc_pc *pc); + /** @idle_residency: get idle residency counter */ + u64 (*idle_residency)(struct xe_guc_pc *pc); +}; + +#endif /* _XE_GT_IDLE_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c new file mode 100644 index 000000000..1949bf7d3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_mcr.c @@ -0,0 +1,911 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include "xe_gt_mcr.h" + +#include "regs/xe_gt_regs.h" +#include "xe_assert.h" +#include "xe_gt_printk.h" +#include "xe_gt_topology.h" +#include "xe_gt_types.h" +#include "xe_guc_hwconfig.h" +#include "xe_mmio.h" +#include "xe_sriov.h" + +/** + * DOC: GT Multicast/Replicated (MCR) Register Support + * + * Some GT registers are designed as "multicast" or "replicated" registers: + * multiple instances of the same register share a single MMIO offset. MCR + * registers are generally used when the hardware needs to potentially track + * independent values of a register per hardware unit (e.g., per-subslice, + * per-L3bank, etc.). The specific types of replication that exist vary + * per-platform. + * + * MMIO accesses to MCR registers are controlled according to the settings + * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR + * registers can be done in either multicast (a single write updates all + * instances of the register to the same value) or unicast (a write updates only + * one specific instance) form. Reads of MCR registers always operate in a + * unicast manner regardless of how the multicast/unicast bit is set in + * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is + * referred to as "steering." + * + * If MCR register operations are steered toward a hardware unit that is + * fused off or currently powered down due to power gating, the MMIO operation + * is "terminated" by the hardware. Terminated read operations will return a + * value of zero and terminated unicast write operations will be silently + * ignored. During device initialization, the goal of the various + * ``init_steering_*()`` functions is to apply the platform-specific rules for + * each MCR register type to identify a steering target that will select a + * non-terminated instance. + * + * MCR registers are not available on Virtual Function (VF). + */ + +static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) +{ + return reg_mcr.__reg; +} + +enum { + MCR_OP_READ, + MCR_OP_WRITE +}; + +static const struct xe_mmio_range xelp_l3bank_steering_table[] = { + { 0x00B100, 0x00B3FF }, + {}, +}; + +static const struct xe_mmio_range xehp_l3bank_steering_table[] = { + { 0x008C80, 0x008CFF }, + { 0x00B100, 0x00B3FF }, + {}, +}; + +/* + * Although the bspec lists more "MSLICE" ranges than shown here, some of those + * are of a "GAM" subclass that has special rules and doesn't need to be + * included here. + */ +static const struct xe_mmio_range xehp_mslice_steering_table[] = { + { 0x00DD00, 0x00DDFF }, + { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ + {}, +}; + +static const struct xe_mmio_range xehp_lncf_steering_table[] = { + { 0x00B000, 0x00B0FF }, + { 0x00D880, 0x00D8FF }, + {}, +}; + +/* + * We have several types of MCR registers where steering to (0,0) will always + * provide us with a non-terminated value. We'll stick them all in the same + * table for simplicity. + */ +static const struct xe_mmio_range xehpc_instance0_steering_table[] = { + { 0x004000, 0x004AFF }, /* HALF-BSLICE */ + { 0x008800, 0x00887F }, /* CC */ + { 0x008A80, 0x008AFF }, /* TILEPSMI */ + { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ + { 0x00B100, 0x00B3FF }, /* L3BANK */ + { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ + { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ + { 0x00DD00, 0x00DDFF }, /* BSLICE */ + { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ + { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ + { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ + { 0x024180, 0x0241FF }, /* HALF-BSLICE */ + {}, +}; + +static const struct xe_mmio_range xelpg_instance0_steering_table[] = { + { 0x000B00, 0x000BFF }, /* SQIDI */ + { 0x001000, 0x001FFF }, /* SQIDI */ + { 0x004000, 0x0048FF }, /* GAM */ + { 0x008700, 0x0087FF }, /* SQIDI */ + { 0x00B000, 0x00B0FF }, /* NODE */ + { 0x00C800, 0x00CFFF }, /* GAM */ + { 0x00D880, 0x00D8FF }, /* NODE */ + { 0x00DD00, 0x00DDFF }, /* OAAL2 */ + {}, +}; + +static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { + { 0x00B100, 0x00B3FF }, + {}, +}; + +static const struct xe_mmio_range xelp_dss_steering_table[] = { + { 0x008150, 0x00815F }, + { 0x009520, 0x00955F }, + { 0x00DE80, 0x00E8FF }, + { 0x024A00, 0x024A7F }, + {}, +}; + +/* DSS steering is used for GSLICE ranges as well */ +static const struct xe_mmio_range xehp_dss_steering_table[] = { + { 0x005200, 0x0052FF }, /* GSLICE */ + { 0x005400, 0x007FFF }, /* GSLICE */ + { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ + { 0x008D00, 0x008DFF }, /* DSS */ + { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00D800, 0x00D87F }, /* GSLICE */ + { 0x00DC00, 0x00DCFF }, /* GSLICE */ + { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ + { 0x017000, 0x017FFF }, /* GSLICE */ + { 0x024A00, 0x024A7F }, /* DSS */ + {}, +}; + +/* DSS steering is used for COMPUTE ranges as well */ +static const struct xe_mmio_range xehpc_dss_steering_table[] = { + { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ + { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00DC00, 0x00DCFF }, /* COMPUTE */ + { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ + {}, +}; + +/* DSS steering is used for SLICE ranges as well */ +static const struct xe_mmio_range xelpg_dss_steering_table[] = { + { 0x005200, 0x0052FF }, /* SLICE */ + { 0x005500, 0x007FFF }, /* SLICE */ + { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ + { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00D800, 0x00D87F }, /* SLICE */ + { 0x00DC00, 0x00DCFF }, /* SLICE */ + { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ + {}, +}; + +static const struct xe_mmio_range xe3p_xpc_xecore_steering_table[] = { + { 0x008140, 0x00817F }, /* SLICE, XeCore, SLICE */ + { 0x009480, 0x00955F }, /* SLICE, XeCore */ + { 0x00D800, 0x00D87F }, /* SLICE */ + { 0x00DC00, 0x00E9FF }, /* SLICE, rsvd, XeCore, rsvd, XeCore, rsvd, XeCore */ + { 0x013000, 0x0135FF }, /* XeCore, SLICE */ + {}, +}; + +static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { + { 0x393200, 0x39323F }, + { 0x393400, 0x3934FF }, + {}, +}; + +static const struct xe_mmio_range dg2_implicit_steering_table[] = { + { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */ + { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */ + { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */ + { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */ + { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */ + { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */ + {}, +}; + +static const struct xe_mmio_range xe2lpg_dss_steering_table[] = { + { 0x005200, 0x0052FF }, /* SLICE */ + { 0x005500, 0x007FFF }, /* SLICE */ + { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ + { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00D800, 0x00D87F }, /* SLICE */ + { 0x00DC00, 0x00DCFF }, /* SLICE */ + { 0x00DE00, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ + { 0x00E980, 0x00E9FF }, /* SLICE */ + { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */ + {}, +}; + +static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { + { 0x000B00, 0x000BFF }, + { 0x001000, 0x001FFF }, + {}, +}; + +static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { + { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ + { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ + { 0x00C800, 0x00CFFF }, /* GAM */ + { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ + { 0x00E900, 0x00E97F }, /* MEMPIPE */ + { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ + { 0x013400, 0x0135FF }, /* MEMPIPE */ + {}, +}; + +static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = { + { 0x388160, 0x38817F }, + { 0x389480, 0x3894CF }, + {}, +}; + +static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = { + { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ + { 0x384900, 0x384AFF }, /* GAM */ + { 0x389560, 0x3895FF }, /* MEDIAINF */ + { 0x38B600, 0x38B8FF }, /* L3BANK */ + { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ + { 0x38F000, 0x38F0FF }, /* GAM */ + { 0x393C00, 0x393C7F }, /* MEDIAINF */ + {}, +}; + +static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = { + { 0x384000, 0x3841FF }, /* GAM */ + { 0x384400, 0x3847DF }, /* GAM */ + { 0x384900, 0x384AFF }, /* GAM */ + { 0x389560, 0x3895FF }, /* MEDIAINF */ + { 0x38B600, 0x38B8FF }, /* L3BANK */ + { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ + { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, rsvd, GAM */ + { 0x393C00, 0x393C7F }, /* MEDIAINF */ + {}, +}; + +/* + * Different "GAM" ranges have different rules; GAMWKRS, STLB, and GAMREQSTRM + * range subtypes need to be steered to (1,0), while all other GAM subtypes + * are steered to (0,0) and are included in the "INSTANCE0" table farther + * down. + */ +static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = { + { 0x004000, 0x004AFF }, /* GAMREQSTRM, rsvd, STLB, GAMWKRS, GAMREQSTRM */ + { 0x00F100, 0x00FFFF }, /* GAMWKRS */ + {}, +}; + +static const struct xe_mmio_range xe2_node_steering_table[] = { + { 0x00B000, 0x00B0FF }, + { 0x00D880, 0x00D8FF }, + {}, +}; + +static const struct xe_mmio_range xe3p_xpc_instance0_steering_table[] = { + { 0x00B500, 0x00B6FF }, /* PSMI */ + { 0x00C800, 0x00CFFF }, /* GAMCTRL */ + { 0x00F000, 0x00F0FF }, /* GAMCTRL */ + {}, +}; + +static const struct xe_mmio_range xe3p_lpg_instance0_steering_table[] = { + { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ + { 0x008700, 0x00887F }, /* NODE */ + { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ + { 0x00B500, 0x00B6FF }, /* PSMI */ + { 0x00C800, 0x00CFFF }, /* GAM */ + { 0x00D880, 0x00D8FF }, /* NODE */ + { 0x00DD00, 0x00DD7F }, /* MEMPIPE */ + { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ + { 0x013400, 0x0135FF }, /* MEMPIPE */ + {}, +}; + +static void init_steering_l3bank(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_mmio *mmio = >->mmio; + + if (GRAPHICS_VER(xe) >= 20) { + unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask); + const int banks_per_node = 4; + unsigned int node = first_bank / banks_per_node; + + /* L3BANK ranges place node in grpID, bank in instanceid */ + gt->steering[L3BANK].group_target = node; + gt->steering[L3BANK].instance_target = first_bank % banks_per_node; + + /* NODE ranges split the node across grpid and instanceid */ + gt->steering[NODE].group_target = node >> 1; + gt->steering[NODE].instance_target = node & 1; + } else if (GRAPHICS_VERx100(xe) >= 1270) { + u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, + xe_mmio_read32(mmio, MIRROR_FUSE3)); + u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, + xe_mmio_read32(mmio, XEHP_FUSE4)); + + /* + * Group selects mslice, instance selects bank within mslice. + * Bank 0 is always valid _except_ when the bank mask is 010b. + */ + gt->steering[L3BANK].group_target = __ffs(mslice_mask); + gt->steering[L3BANK].instance_target = + bank_mask & BIT(0) ? 0 : 2; + } else if (xe->info.platform == XE_DG2) { + u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, + xe_mmio_read32(mmio, MIRROR_FUSE3)); + u32 bank = __ffs(mslice_mask) * 8; + + /* + * Like mslice registers, look for a valid mslice and steer to + * the first L3BANK of that quad. Access to the Nth L3 bank is + * split between the first bits of group and instance + */ + gt->steering[L3BANK].group_target = (bank >> 2) & 0x7; + gt->steering[L3BANK].instance_target = bank & 0x3; + } else { + u32 fuse = REG_FIELD_GET(L3BANK_MASK, + ~xe_mmio_read32(mmio, MIRROR_FUSE3)); + + gt->steering[L3BANK].group_target = 0; /* unused */ + gt->steering[L3BANK].instance_target = __ffs(fuse); + } +} + +static void init_steering_mslice(struct xe_gt *gt) +{ + u32 mask = REG_FIELD_GET(MEML3_EN_MASK, + xe_mmio_read32(>->mmio, MIRROR_FUSE3)); + + /* + * mslice registers are valid (not terminated) if either the meml3 + * associated with the mslice is present, or at least one DSS associated + * with the mslice is present. There will always be at least one meml3 + * so we can just use that to find a non-terminated mslice and ignore + * the DSS fusing. + */ + gt->steering[MSLICE].group_target = __ffs(mask); + gt->steering[MSLICE].instance_target = 0; /* unused */ + + /* + * LNCF termination is also based on mslice presence, so we'll set + * it up here. Either LNCF within a non-terminated mslice will work, + * so we just always pick LNCF 0 here. + */ + gt->steering[LNCF].group_target = __ffs(mask) << 1; + gt->steering[LNCF].instance_target = 0; /* unused */ +} + +static unsigned int dss_per_group(struct xe_gt *gt) +{ + struct xe_guc *guc = >->uc.guc; + u32 max_slices = 0, max_subslices = 0; + int ret; + + /* + * Try to query the GuC's hwconfig table for the maximum number of + * slices and subslices. These don't reflect the platform's actual + * slice/DSS counts, just the physical layout by which we should + * determine the steering targets. On older platforms with older GuC + * firmware releases it's possible that these attributes may not be + * included in the table, so we can always fall back to the old + * hardcoded layouts. + */ +#define HWCONFIG_ATTR_MAX_SLICES 1 +#define HWCONFIG_ATTR_MAX_SUBSLICES 70 + + ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES, + &max_slices); + if (ret < 0 || max_slices == 0) + goto fallback; + + ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES, + &max_subslices); + if (ret < 0 || max_subslices == 0) + goto fallback; + + return DIV_ROUND_UP(max_subslices, max_slices); + +fallback: + /* + * Some older platforms don't have tables or don't have complete tables. + * Newer platforms should always have the required info. + */ + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000) + xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n"); + + if (gt_to_xe(gt)->info.platform == XE_PVC) + return 8; + else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250) + return 4; + else + return 6; +} + +/** + * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS + * @gt: GT structure + * @dss: DSS ID to obtain steering for + * @group: pointer to storage for steering group ID + * @instance: pointer to storage for steering instance ID + */ +void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance) +{ + xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS); + + *group = dss / gt->steering_dss_per_grp; + *instance = dss % gt->steering_dss_per_grp; +} + +static void init_steering_dss(struct xe_gt *gt) +{ + gt->steering_dss_per_grp = dss_per_group(gt); + + xe_gt_mcr_get_dss_steering(gt, + min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), + xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)), + >->steering[DSS].group_target, + >->steering[DSS].instance_target); +} + +static void init_steering_oaddrm(struct xe_gt *gt) +{ + u64 hwe_mask = XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0; + + /* TODO: Add 'VD per SCMI' and 'VE per SCMI' values into 'struct xe_media_desc' */ + if (MEDIA_VERx100(gt_to_xe(gt)) >= 3500) + hwe_mask |= XE_HW_ENGINE_VCS1 | XE_HW_ENGINE_VECS1; + /* + * First instance is only terminated if the entire first media slice + * is absent (i.e., no engines in hwe_mask). + */ + if (gt->info.engine_mask & hwe_mask) + gt->steering[OADDRM].group_target = 0; + else + gt->steering[OADDRM].group_target = 1; + + gt->steering[OADDRM].instance_target = 0; /* unused */ +} + +static void init_steering_sqidi_psmi(struct xe_gt *gt) +{ + u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, + xe_mmio_read32(>->mmio, MIRROR_FUSE3)); + u32 select = __ffs(mask); + + gt->steering[SQIDI_PSMI].group_target = select >> 1; + gt->steering[SQIDI_PSMI].instance_target = select & 0x1; +} + +static void init_steering_gam1(struct xe_gt *gt) +{ + gt->steering[GAM1].group_target = 1; + gt->steering[GAM1].instance_target = 0; +} + +static const struct { + const char *name; + void (*init)(struct xe_gt *gt); +} xe_steering_types[] = { + [L3BANK] = { "L3BANK", init_steering_l3bank }, + [NODE] = { "NODE", NULL }, /* initialized by l3bank init */ + [MSLICE] = { "MSLICE", init_steering_mslice }, + [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */ + [DSS] = { "DSS / XeCore", init_steering_dss }, + [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm }, + [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi }, + [GAM1] = { "GAMWKRS / STLB / GAMREQSTRM", init_steering_gam1 }, + [INSTANCE0] = { "INSTANCE 0", NULL }, + [IMPLICIT_STEERING] = { "IMPLICIT", NULL }, +}; + +/** + * xe_gt_mcr_init_early - Early initialization of the MCR support + * @gt: GT structure + * + * Perform early software only initialization of the MCR lock to allow + * the synchronization on accessing the STEER_SEMAPHORE register and + * use the xe_gt_mcr_multicast_write() function, plus the minimum + * safe MCR registers required for VRAM/CCS probing. + */ +void xe_gt_mcr_init_early(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES); + BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); + + spin_lock_init(>->mcr_lock); + + if (gt->info.type == XE_GT_TYPE_MEDIA) { + xe_gt_WARN_ON(gt, MEDIA_VER(xe) < 13); + + if (MEDIA_VER(xe) >= 30) { + gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; + gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table; + } else if (MEDIA_VERx100(xe) >= 1301) { + gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; + gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table; + } else { + gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; + } + } else { + if (GRAPHICS_VERx100(xe) == 3511) { + gt->steering[DSS].ranges = xe3p_xpc_xecore_steering_table; + gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table; + gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table; + gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; + gt->steering[NODE].ranges = xe2_node_steering_table; + } else if (GRAPHICS_VERx100(xe) >= 3510) { + gt->steering[DSS].ranges = xe2lpg_dss_steering_table; + gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table; + } else if (GRAPHICS_VER(xe) >= 20) { + gt->steering[DSS].ranges = xe2lpg_dss_steering_table; + gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; + gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; + gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; + gt->steering[NODE].ranges = xe2_node_steering_table; + } else if (GRAPHICS_VERx100(xe) >= 1270) { + gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; + gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; + gt->steering[DSS].ranges = xelpg_dss_steering_table; + } else if (xe->info.platform == XE_PVC) { + gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; + gt->steering[DSS].ranges = xehpc_dss_steering_table; + } else if (xe->info.platform == XE_DG2) { + gt->steering[L3BANK].ranges = xehp_l3bank_steering_table; + gt->steering[MSLICE].ranges = xehp_mslice_steering_table; + gt->steering[LNCF].ranges = xehp_lncf_steering_table; + gt->steering[DSS].ranges = xehp_dss_steering_table; + gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table; + } else { + gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; + gt->steering[DSS].ranges = xelp_dss_steering_table; + } + } + + /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */ + gt->steering[INSTANCE0].initialized = true; +} +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_init_early); + +/** + * xe_gt_mcr_init - Normal initialization of the MCR support + * @gt: GT structure + * + * Perform normal initialization of the MCR for all usages. + */ +void xe_gt_mcr_init(struct xe_gt *gt) +{ + /* Select non-terminated steering target for each type */ + for (int i = 0; i < NUM_STEERING_TYPES; i++) { + gt->steering[i].initialized = true; + if (gt->steering[i].ranges && xe_steering_types[i].init) + xe_steering_types[i].init(gt); + } +} + +/** + * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers + * @gt: GT structure + * + * Some register ranges don't need to have their steering control registers + * changed on each access - it's sufficient to set them once on initialization. + * This function sets those registers for each platform * + */ +void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (IS_SRIOV_VF(xe)) + return; + + if (xe->info.platform == XE_DG2) { + u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | + REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); + + xe_mmio_write32(>->mmio, STEER_SEMAPHORE, steer_val); + xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val); + /* + * For GAM registers, all reads should be directed to instance 1 + * (unicast reads against other instances are not allowed), + * and instance 1 is already the hardware's default steering + * target, which we never change + */ + } +} + +static bool reg_in_steering_type_ranges(struct xe_gt *gt, + struct xe_reg reg, + int type) +{ + if (!gt->steering[type].ranges) + return false; + + for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) + if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) + return true; + + return false; +} + +/* + * xe_gt_mcr_check_reg - check if a register is recognized by this GT as MCR + * @gt: GT structure + * @reg: The register to check + * + * Returns true if the register offset falls within one of the MMIO ranges + * classified as MCR for the GT. + */ +bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_gt_mcr_check_reg, gt, reg); + + for (int type = 0; type <= IMPLICIT_STEERING; type++) + if (reg_in_steering_type_ranges(gt, reg, type)) + return true; + + return false; +} +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_check_reg); + +/* + * xe_gt_mcr_get_nonterminated_steering - find group/instance values that + * will steer a register to a non-terminated instance + * @gt: GT structure + * @reg: register for which the steering is required + * @group: return variable for group steering + * @instance: return variable for instance steering + * + * This function returns a group/instance pair that is guaranteed to work for + * read steering of the given register. Note that a value will be returned even + * if the register is not replicated and therefore does not actually require + * steering. + * + * Returns true if the caller should steer to the @group/@instance values + * returned. Returns false if the caller need not perform any steering + */ +bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, + struct xe_reg_mcr reg_mcr, + u8 *group, u8 *instance) +{ + const struct xe_reg reg = to_xe_reg(reg_mcr); + + for (int type = 0; type < IMPLICIT_STEERING; type++) { + if (reg_in_steering_type_ranges(gt, reg, type)) { + xe_gt_WARN(gt, !gt->steering[type].initialized, + "Uninitialized usage of MCR register %s/%#x\n", + xe_steering_types[type].name, reg.addr); + + *group = gt->steering[type].group_target; + *instance = gt->steering[type].instance_target; + return true; + } + } + + if (reg_in_steering_type_ranges(gt, reg, IMPLICIT_STEERING)) + return false; + + /* + * Not found in a steering table and not a register with implicit + * steering. Just steer to 0/0 as a guess and raise a warning. + */ + xe_gt_WARN(gt, true, + "Did not find MCR register %#x in any MCR steering table\n", + reg.addr); + *group = 0; + *instance = 0; + + return true; +} + +/* + * Obtain exclusive access to MCR steering. On MTL and beyond we also need + * to synchronize with external clients (e.g., firmware), so a semaphore + * register will also need to be taken. + */ +static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock) +{ + struct xe_device *xe = gt_to_xe(gt); + int ret = 0; + + spin_lock(>->mcr_lock); + + /* + * Starting with MTL we also need to grab a semaphore register + * to synchronize with external agents (e.g., firmware) that now + * shares the same steering control register. The semaphore is obtained + * when a read to the relevant register returns 1. + */ + if (GRAPHICS_VERx100(xe) >= 1270) + ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL, + true); + + xe_gt_WARN_ON_ONCE(gt, ret == -ETIMEDOUT); +} + +static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock) +{ + /* Release hardware semaphore - this is done by writing 1 to the register */ + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) + xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1); + + spin_unlock(>->mcr_lock); +} + +/* + * Access a register with specific MCR steering + * + * Caller needs to make sure the relevant forcewake wells are up. + */ +static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, + u8 rw_flag, int group, int instance, u32 value) +{ + const struct xe_reg reg = to_xe_reg(reg_mcr); + struct xe_mmio *mmio = >->mmio; + struct xe_reg steer_reg; + u32 steer_val, val = 0; + + lockdep_assert_held(>->mcr_lock); + + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { + steer_reg = MTL_MCR_SELECTOR; + steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | + REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); + } else { + steer_reg = MCR_SELECTOR; + steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) | + REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance); + } + + /* + * Always leave the hardware in multicast mode when doing reads and only + * change it to unicast mode when doing writes of a specific instance. + * + * The setting of the multicast/unicast bit usually wouldn't matter for + * read operations (which always return the value from a single register + * instance regardless of how that bit is set), but some platforms may + * have workarounds requiring us to remain in multicast mode for reads, + * e.g. Wa_22013088509 on PVC. There's no real downside to this, so + * we'll just go ahead and do so on all platforms; we'll only clear the + * multicast bit from the mask when explicitly doing a write operation. + * + * No need to save old steering reg value. + */ + if (rw_flag == MCR_OP_READ) + steer_val |= MCR_MULTICAST; + + xe_mmio_write32(mmio, steer_reg, steer_val); + + if (rw_flag == MCR_OP_READ) + val = xe_mmio_read32(mmio, reg); + else + xe_mmio_write32(mmio, reg, value); + + /* + * If we turned off the multicast bit (during a write) we're required + * to turn it back on before finishing. The group and instance values + * don't matter since they'll be re-programmed on the next MCR + * operation. + */ + if (rw_flag == MCR_OP_WRITE) + xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST); + + return val; +} + +/** + * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register + * @gt: GT structure + * @reg_mcr: register to read + * + * Reads a GT MCR register. The read will be steered to a non-terminated + * instance (i.e., one that isn't fused off or powered down by power gating). + * This function assumes the caller is already holding any necessary forcewake + * domains. + * + * Returns the value from a non-terminated instance of @reg. + */ +u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr) +{ + const struct xe_reg reg = to_xe_reg(reg_mcr); + u8 group, instance; + u32 val; + bool steer; + + xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); + + steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr, + &group, &instance); + + if (steer) { + mcr_lock(gt); + val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, + group, instance, 0); + mcr_unlock(gt); + } else { + val = xe_mmio_read32(>->mmio, reg); + } + + return val; +} + +/** + * xe_gt_mcr_unicast_read - read a specific instance of an MCR register + * @gt: GT structure + * @reg_mcr: the MCR register to read + * @group: the MCR group + * @instance: the MCR instance + * + * Returns the value read from an MCR register after steering toward a specific + * group/instance. + */ +u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, + struct xe_reg_mcr reg_mcr, + int group, int instance) +{ + u32 val; + + xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); + + mcr_lock(gt); + val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0); + mcr_unlock(gt); + + return val; +} + +/** + * xe_gt_mcr_unicast_write - write a specific instance of an MCR register + * @gt: GT structure + * @reg_mcr: the MCR register to write + * @value: value to write + * @group: the MCR group + * @instance: the MCR instance + * + * Write an MCR register in unicast mode after steering toward a specific + * group/instance. + */ +void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, + u32 value, int group, int instance) +{ + xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); + + mcr_lock(gt); + rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value); + mcr_unlock(gt); +} + +/** + * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register + * @gt: GT structure + * @reg_mcr: the MCR register to write + * @value: value to write + * + * Write an MCR register in multicast mode to update all instances. + */ +void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, + u32 value) +{ + struct xe_reg reg = to_xe_reg(reg_mcr); + + xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); + + /* + * Synchronize with any unicast operations. Once we have exclusive + * access, the MULTICAST bit should already be set, so there's no need + * to touch the steering register. + */ + mcr_lock(gt); + xe_mmio_write32(>->mmio, reg, value); + mcr_unlock(gt); +} + +void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) +{ + for (int i = 0; i < NUM_STEERING_TYPES; i++) { + if (gt->steering[i].ranges) { + drm_printf(p, "%s steering: group=%#x, instance=%#x\n", + xe_steering_types[i].name, + gt->steering[i].group_target, + gt->steering[i].instance_target); + for (int j = 0; gt->steering[i].ranges[j].end; j++) + drm_printf(p, "\t0x%06x - 0x%06x\n", + gt->steering[i].ranges[j].start, + gt->steering[i].ranges[j].end); + } + } +} diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.h b/drivers/gpu/drm/xe/xe_gt_mcr.h new file mode 100644 index 000000000..75374662f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_mcr.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_MCR_H_ +#define _XE_GT_MCR_H_ + +#include "regs/xe_reg_defs.h" +#include "xe_gt_topology.h" + +struct drm_printer; +struct xe_gt; + +void xe_gt_mcr_init_early(struct xe_gt *gt); +void xe_gt_mcr_init(struct xe_gt *gt); + +void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt); + +u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, + int group, int instance); +u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr mcr_reg); + +void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, + u32 value, int group, int instance); +void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, + u32 value); + +bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg); +bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, + struct xe_reg_mcr reg_mcr, + u8 *group, u8 *instance); + +void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p); +void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, + unsigned int dss, u16 *group, u16 *instance); + +/* + * Loop over each DSS and determine the group and instance IDs that + * should be used to steer MCR accesses toward this DSS. + * @dss: DSS ID to obtain steering for + * @gt: GT structure + * @group: steering group ID, data type: u16 + * @instance: steering instance ID, data type: u16 + */ +#define for_each_dss_steering(dss, gt, group, instance) \ + for_each_dss((dss), (gt)) \ + for_each_if((xe_gt_mcr_get_dss_steering((gt), (dss), &(group), &(instance)), true)) + +/* + * Loop over each DSS available for geometry and determine the group and + * instance IDs that should be used to steer MCR accesses toward this DSS. + * @dss: DSS ID to obtain steering for + * @gt: GT structure + * @group: steering group ID, data type: u16 + * @instance: steering instance ID, data type: u16 + */ +#define for_each_geometry_dss(dss, gt, group, instance) \ + for_each_dss_steering(dss, gt, group, instance) \ + if (xe_gt_has_geometry_dss(gt, dss)) + +/* + * Loop over each DSS available for compute and determine the group and + * instance IDs that should be used to steer MCR accesses toward this DSS. + * @dss: DSS ID to obtain steering for + * @gt: GT structure + * @group: steering group ID, data type: u16 + * @instance: steering instance ID, data type: u16 + */ +#define for_each_compute_dss(dss, gt, group, instance) \ + for_each_dss_steering(dss, gt, group, instance) \ + if (xe_gt_has_compute_dss(gt, dss)) + +#endif /* _XE_GT_MCR_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_printk.h b/drivers/gpu/drm/xe/xe_gt_printk.h new file mode 100644 index 000000000..1313d3286 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_printk.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_PRINTK_H_ +#define _XE_GT_PRINTK_H_ + +#include "xe_gt_types.h" +#include "xe_tile_printk.h" + +#define __XE_GT_PRINTK_FMT(_gt, _fmt, _args...) "GT%u: " _fmt, (_gt)->info.id, ##_args + +#define xe_gt_printk(_gt, _level, _fmt, ...) \ + xe_tile_printk((_gt)->tile, _level, __XE_GT_PRINTK_FMT((_gt), _fmt, ##__VA_ARGS__)) + +#define xe_gt_err(_gt, _fmt, ...) \ + xe_gt_printk((_gt), err, _fmt, ##__VA_ARGS__) + +#define xe_gt_err_once(_gt, _fmt, ...) \ + xe_gt_printk((_gt), err_once, _fmt, ##__VA_ARGS__) + +#define xe_gt_err_ratelimited(_gt, _fmt, ...) \ + xe_gt_printk((_gt), err_ratelimited, _fmt, ##__VA_ARGS__) + +#define xe_gt_warn(_gt, _fmt, ...) \ + xe_gt_printk((_gt), warn, _fmt, ##__VA_ARGS__) + +#define xe_gt_notice(_gt, _fmt, ...) \ + xe_gt_printk((_gt), notice, _fmt, ##__VA_ARGS__) + +#define xe_gt_info(_gt, _fmt, ...) \ + xe_gt_printk((_gt), info, _fmt, ##__VA_ARGS__) + +#define xe_gt_dbg(_gt, _fmt, ...) \ + xe_gt_printk((_gt), dbg, _fmt, ##__VA_ARGS__) + +#define xe_gt_WARN_type(_gt, _type, _condition, _fmt, ...) \ + xe_tile_WARN##_type((_gt)->tile, _condition, _fmt, ## __VA_ARGS__) + +#define xe_gt_WARN(_gt, _condition, _fmt, ...) \ + xe_gt_WARN_type((_gt),, _condition, __XE_GT_PRINTK_FMT((_gt), _fmt, ##__VA_ARGS__)) + +#define xe_gt_WARN_ONCE(_gt, _condition, _fmt, ...) \ + xe_gt_WARN_type((_gt), _ONCE, _condition, __XE_GT_PRINTK_FMT((_gt), _fmt, ##__VA_ARGS__)) + +#define xe_gt_WARN_ON(_gt, _condition) \ + xe_gt_WARN((_gt), _condition, "%s(%s)", "WARN_ON", __stringify(_condition)) + +#define xe_gt_WARN_ON_ONCE(_gt, _condition) \ + xe_gt_WARN_ONCE((_gt), _condition, "%s(%s)", "WARN_ON_ONCE", __stringify(_condition)) + +static inline void __xe_gt_printfn_err(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_gt *gt = p->arg; + + xe_gt_err(gt, "%pV", vaf); +} + +static inline void __xe_gt_printfn_info(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_gt *gt = p->arg; + + xe_gt_info(gt, "%pV", vaf); +} + +static inline void __xe_gt_printfn_dbg(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_gt *gt = p->arg; + struct drm_printer dbg; + + /* + * The original xe_gt_dbg() callsite annotations are useless here, + * redirect to the tweaked xe_tile_dbg_printer() instead. + */ + dbg = xe_tile_dbg_printer((gt)->tile); + dbg.origin = p->origin; + + drm_printf(&dbg, __XE_GT_PRINTK_FMT(gt, "%pV", vaf)); +} + +/** + * xe_gt_err_printer - Construct a &drm_printer that outputs to xe_gt_err() + * @gt: the &xe_gt pointer to use in xe_gt_err() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_gt_err_printer(struct xe_gt *gt) +{ + struct drm_printer p = { + .printfn = __xe_gt_printfn_err, + .arg = gt, + }; + return p; +} + +/** + * xe_gt_info_printer - Construct a &drm_printer that outputs to xe_gt_info() + * @gt: the &xe_gt pointer to use in xe_gt_info() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_gt_info_printer(struct xe_gt *gt) +{ + struct drm_printer p = { + .printfn = __xe_gt_printfn_info, + .arg = gt, + }; + return p; +} + +/** + * xe_gt_dbg_printer - Construct a &drm_printer that outputs like xe_gt_dbg() + * @gt: the &xe_gt pointer to use in xe_gt_dbg() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_gt_dbg_printer(struct xe_gt *gt) +{ + struct drm_printer p = { + .printfn = __xe_gt_printfn_dbg, + .arg = gt, + .origin = (const void *)_THIS_IP_, + }; + return p; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c new file mode 100644 index 000000000..786805684 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include + +#include "regs/xe_guc_regs.h" +#include "regs/xe_regs.h" + +#include "xe_gt.h" +#include "xe_gt_sriov_pf.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_gt_sriov_pf_policy.h" +#include "xe_gt_sriov_pf_service.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc_submit.h" +#include "xe_mmio.h" +#include "xe_pm.h" + +static void pf_worker_restart_func(struct work_struct *w); + +/* + * VF's metadata is maintained in the flexible array where: + * - entry [0] contains metadata for the PF (only if applicable), + * - entries [1..n] contain metadata for VF1..VFn:: + * + * <--------------------------- 1 + total_vfs -----------> + * +-------+-------+-------+-----------------------+-------+ + * | 0 | 1 | 2 | | n | + * +-------+-------+-------+-----------------------+-------+ + * | PF | VF1 | VF2 | ... ... | VFn | + * +-------+-------+-------+-----------------------+-------+ + */ +static int pf_alloc_metadata(struct xe_gt *gt) +{ + unsigned int num_vfs = xe_gt_sriov_pf_get_totalvfs(gt); + + gt->sriov.pf.vfs = drmm_kcalloc(>_to_xe(gt)->drm, 1 + num_vfs, + sizeof(*gt->sriov.pf.vfs), GFP_KERNEL); + if (!gt->sriov.pf.vfs) + return -ENOMEM; + + return 0; +} + +static void pf_init_workers(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + INIT_WORK(>->sriov.pf.workers.restart, pf_worker_restart_func); +} + +static void pf_fini_workers(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + if (disable_work_sync(>->sriov.pf.workers.restart)) { + xe_gt_sriov_dbg_verbose(gt, "pending restart disabled!\n"); + /* release an rpm reference taken on the worker's behalf */ + xe_pm_runtime_put(gt_to_xe(gt)); + } +} + +/** + * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF. + * @gt: the &xe_gt to initialize + * + * Early initialization of the PF data. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_init_early(struct xe_gt *gt) +{ + int err; + + err = pf_alloc_metadata(gt); + if (err) + return err; + + err = xe_gt_sriov_pf_service_init(gt); + if (err) + return err; + + err = xe_gt_sriov_pf_control_init(gt); + if (err) + return err; + + pf_init_workers(gt); + + return 0; +} + +static void pf_fini_action(void *arg) +{ + struct xe_gt *gt = arg; + + pf_fini_workers(gt); +} + +static int pf_init_late(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + return devm_add_action_or_reset(xe->drm.dev, pf_fini_action, gt); +} + +/** + * xe_gt_sriov_pf_init - Prepare SR-IOV PF data structures on PF. + * @gt: the &xe_gt to initialize + * + * Late one-time initialization of the PF data. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_init(struct xe_gt *gt) +{ + int err; + + err = xe_gt_sriov_pf_config_init(gt); + if (err) + return err; + + xe_gt_sriov_pf_policy_init(gt); + + err = xe_gt_sriov_pf_migration_init(gt); + if (err) + return err; + + err = pf_init_late(gt); + if (err) + return err; + + return 0; +} + +static bool pf_needs_enable_ggtt_guest_update(struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) == 1200; +} + +static void pf_enable_ggtt_guest_update(struct xe_gt *gt) +{ + xe_mmio_write32(>->mmio, VIRTUAL_CTRL_REG, GUEST_GTT_UPDATE_EN); +} + +/** + * xe_gt_sriov_pf_init_hw - Initialize SR-IOV hardware support. + * @gt: the &xe_gt to initialize + * + * On some platforms the PF must explicitly enable VF's access to the GGTT. + */ +void xe_gt_sriov_pf_init_hw(struct xe_gt *gt) +{ + if (pf_needs_enable_ggtt_guest_update(gt_to_xe(gt))) + pf_enable_ggtt_guest_update(gt); + + xe_gt_sriov_pf_service_update(gt); +} + +static void pf_clear_vf_scratch_regs(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_mmio mmio; + int n; + + xe_mmio_init_vf_view(&mmio, >->mmio, vfid); + + if (xe_gt_is_media_type(gt)) { + for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) + xe_mmio_write32(&mmio, MED_VF_SW_FLAG(n), 0); + } else { + for (n = 0; n < VF_SW_FLAG_COUNT; n++) + xe_mmio_write32(&mmio, VF_SW_FLAG(n), 0); + } +} + +/** + * xe_gt_sriov_pf_sanitize_hw() - Reset hardware state related to a VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + pf_clear_vf_scratch_regs(gt, vfid); +} + +static void pf_cancel_restart(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + if (cancel_work_sync(>->sriov.pf.workers.restart)) { + xe_gt_sriov_dbg_verbose(gt, "pending restart canceled!\n"); + /* release an rpm reference taken on the worker's behalf */ + xe_pm_runtime_put(gt_to_xe(gt)); + } +} + +/** + * xe_gt_sriov_pf_stop_prepare() - Prepare to stop SR-IOV support. + * @gt: the &xe_gt + * + * This function can only be called on the PF. + */ +void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt) +{ + pf_cancel_restart(gt); +} + +static void pf_restart(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, !xe_pm_runtime_suspended(xe)); + + xe_gt_sriov_pf_policy_restart(gt); + xe_gt_sriov_pf_config_restart(gt); + xe_gt_sriov_pf_control_restart(gt); + + /* release an rpm reference taken on our behalf */ + xe_pm_runtime_put(xe); + + xe_gt_sriov_dbg(gt, "restart completed\n"); +} + +static void pf_worker_restart_func(struct work_struct *w) +{ + struct xe_gt *gt = container_of(w, typeof(*gt), sriov.pf.workers.restart); + + pf_restart(gt); +} + +static void pf_queue_restart(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + /* take an rpm reference on behalf of the worker */ + xe_pm_runtime_get_noresume(xe); + + if (!queue_work(xe->sriov.wq, >->sriov.pf.workers.restart)) { + xe_gt_sriov_dbg(gt, "restart already in queue!\n"); + xe_pm_runtime_put(xe); + } +} + +/** + * xe_gt_sriov_pf_restart - Restart SR-IOV support after a GT reset. + * @gt: the &xe_gt + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_restart(struct xe_gt *gt) +{ + pf_queue_restart(gt); +} + +static void pf_flush_restart(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + flush_work(>->sriov.pf.workers.restart); +} + +/** + * xe_gt_sriov_pf_wait_ready() - Wait until per-GT PF SR-IOV support is ready. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt) +{ + /* don't wait if there is another ongoing reset */ + if (xe_guc_read_stopped(>->uc.guc)) + return -EBUSY; + + pf_flush_restart(gt); + return 0; +} + +/** + * xe_gt_sriov_pf_sched_groups_enabled - Check if multiple scheduler groups are + * enabled + * @gt: the &xe_gt + * + * This function is for PF use only. + * + * Return: true if shed groups were enabled, false otherwise. + */ +bool xe_gt_sriov_pf_sched_groups_enabled(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + return xe_gt_sriov_pf_policy_sched_groups_enabled(gt); +} + diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h new file mode 100644 index 000000000..1ccfc7137 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_H_ +#define _XE_GT_SRIOV_PF_H_ + +#include + +struct xe_gt; + +#ifdef CONFIG_PCI_IOV +int xe_gt_sriov_pf_init_early(struct xe_gt *gt); +int xe_gt_sriov_pf_init(struct xe_gt *gt); +int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt); +void xe_gt_sriov_pf_init_hw(struct xe_gt *gt); +void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid); +void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt); +void xe_gt_sriov_pf_restart(struct xe_gt *gt); +bool xe_gt_sriov_pf_sched_groups_enabled(struct xe_gt *gt); +#else +static inline int xe_gt_sriov_pf_init_early(struct xe_gt *gt) +{ + return 0; +} + +static inline int xe_gt_sriov_pf_init(struct xe_gt *gt) +{ + return 0; +} + +static inline void xe_gt_sriov_pf_init_hw(struct xe_gt *gt) +{ +} + +static inline void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt) +{ +} + +static inline void xe_gt_sriov_pf_restart(struct xe_gt *gt) +{ +} + +static inline bool xe_gt_sriov_pf_sched_groups_enabled(struct xe_gt *gt) +{ + return false; +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c new file mode 100644 index 000000000..2c9b85b84 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -0,0 +1,3521 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include + +#include "abi/guc_actions_sriov_abi.h" +#include "abi/guc_klvs_abi.h" + +#include "regs/xe_gtt_defs.h" +#include "regs/xe_guc_regs.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_policy.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc.h" +#include "xe_guc_buf.h" +#include "xe_guc_ct.h" +#include "xe_guc_db_mgr.h" +#include "xe_guc_id_mgr.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_klv_thresholds_set.h" +#include "xe_guc_submit.h" +#include "xe_lmtt.h" +#include "xe_migrate.h" +#include "xe_sriov.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vram_types.h" +#include "xe_wopcm.h" + +#define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) + +/* + * Return: number of KLVs that were successfully parsed and saved, + * negative error code on failure. + */ +static int guc_action_update_vf_cfg(struct xe_guc *guc, u32 vfid, + u64 addr, u32 size) +{ + u32 request[] = { + GUC_ACTION_PF2GUC_UPDATE_VF_CFG, + vfid, + lower_32_bits(addr), + upper_32_bits(addr), + size, + }; + + return xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request)); +} + +/* + * Return: 0 on success, negative error code on failure. + */ +static int pf_send_vf_cfg_reset(struct xe_gt *gt, u32 vfid) +{ + struct xe_guc *guc = >->uc.guc; + int ret; + + ret = guc_action_update_vf_cfg(guc, vfid, 0, 0); + + return ret <= 0 ? ret : -EPROTO; +} + +/* + * Return: number of KLVs that were successfully parsed and saved, + * negative error code on failure. + */ +static int pf_send_vf_buf_klvs(struct xe_gt *gt, u32 vfid, struct xe_guc_buf buf, u32 num_dwords) +{ + struct xe_guc *guc = >->uc.guc; + + return guc_action_update_vf_cfg(guc, vfid, xe_guc_buf_flush(buf), num_dwords); +} + +/* + * Return: 0 on success, -ENOKEY if some KLVs were not updated, -EPROTO if reply was malformed, + * negative error code on failure. + */ +static int pf_push_vf_buf_klvs(struct xe_gt *gt, unsigned int vfid, u32 num_klvs, + struct xe_guc_buf buf, u32 num_dwords) +{ + int ret; + + ret = pf_send_vf_buf_klvs(gt, vfid, buf, num_dwords); + + if (ret != num_klvs) { + int err = ret < 0 ? ret : ret < num_klvs ? -ENOKEY : -EPROTO; + void *klvs = xe_guc_buf_cpu_ptr(buf); + struct drm_printer p = xe_gt_info_printer(gt); + char name[8]; + + xe_gt_sriov_notice(gt, "Failed to push %s %u config KLV%s (%pe)\n", + xe_sriov_function_name(vfid, name, sizeof(name)), + num_klvs, str_plural(num_klvs), ERR_PTR(err)); + xe_guc_klv_print(klvs, num_dwords, &p); + return err; + } + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + void *klvs = xe_guc_buf_cpu_ptr(buf); + char name[8]; + + xe_gt_sriov_dbg(gt, "pushed %s config with %u KLV%s:\n", + xe_sriov_function_name(vfid, name, sizeof(name)), + num_klvs, str_plural(num_klvs)); + xe_guc_klv_print(klvs, num_dwords, &p); + } + + return 0; +} + +/* + * Return: 0 on success, -ENOBUFS if no free buffer for the indirect data, + * negative error code on failure. + */ +static int pf_push_vf_cfg_klvs(struct xe_gt *gt, unsigned int vfid, u32 num_klvs, + const u32 *klvs, u32 num_dwords) +{ + CLASS(xe_guc_buf_from_data, buf)(>->uc.guc.buf, klvs, num_dwords * sizeof(u32)); + + xe_gt_assert(gt, num_klvs == xe_guc_klv_count(klvs, num_dwords)); + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + return pf_push_vf_buf_klvs(gt, vfid, num_klvs, buf, num_dwords); +} + +static int pf_push_vf_cfg_u32(struct xe_gt *gt, unsigned int vfid, u16 key, u32 value) +{ + u32 klv[] = { + FIELD_PREP(GUC_KLV_0_KEY, key) | FIELD_PREP(GUC_KLV_0_LEN, 1), + value, + }; + + return pf_push_vf_cfg_klvs(gt, vfid, 1, klv, ARRAY_SIZE(klv)); +} + +static int pf_push_vf_cfg_u64(struct xe_gt *gt, unsigned int vfid, u16 key, u64 value) +{ + u32 klv[] = { + FIELD_PREP(GUC_KLV_0_KEY, key) | FIELD_PREP(GUC_KLV_0_LEN, 2), + lower_32_bits(value), + upper_32_bits(value), + }; + + return pf_push_vf_cfg_klvs(gt, vfid, 1, klv, ARRAY_SIZE(klv)); +} + +static int pf_push_vf_cfg_ggtt(struct xe_gt *gt, unsigned int vfid, u64 start, u64 size) +{ + u32 klvs[] = { + PREP_GUC_KLV_TAG(VF_CFG_GGTT_START), + lower_32_bits(start), + upper_32_bits(start), + PREP_GUC_KLV_TAG(VF_CFG_GGTT_SIZE), + lower_32_bits(size), + upper_32_bits(size), + }; + + return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs)); +} + +static int pf_push_vf_cfg_ctxs(struct xe_gt *gt, unsigned int vfid, u32 begin, u32 num) +{ + u32 klvs[] = { + PREP_GUC_KLV_TAG(VF_CFG_BEGIN_CONTEXT_ID), + begin, + PREP_GUC_KLV_TAG(VF_CFG_NUM_CONTEXTS), + num, + }; + + return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs)); +} + +static int pf_push_vf_cfg_dbs(struct xe_gt *gt, unsigned int vfid, u32 begin, u32 num) +{ + u32 klvs[] = { + PREP_GUC_KLV_TAG(VF_CFG_BEGIN_DOORBELL_ID), + begin, + PREP_GUC_KLV_TAG(VF_CFG_NUM_DOORBELLS), + num, + }; + + return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs)); +} + +static int pf_push_vf_grp_cfg_u32(struct xe_gt *gt, unsigned int vfid, + u16 key, const u32 *values, u32 count) +{ + CLASS(xe_guc_buf, buf)(>->uc.guc.buf, GUC_KLV_LEN_MIN + GUC_MAX_SCHED_GROUPS); + u32 *klv; + + xe_gt_assert(gt, count && count <= GUC_MAX_SCHED_GROUPS); + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + klv = xe_guc_buf_cpu_ptr(buf); + + klv[0] = FIELD_PREP(GUC_KLV_0_KEY, key) | FIELD_PREP(GUC_KLV_0_LEN, count); + memcpy(&klv[1], values, count * sizeof(u32)); + + return pf_push_vf_buf_klvs(gt, vfid, 1, buf, GUC_KLV_LEN_MIN + count); +} + +static int pf_push_vf_cfg_exec_quantum(struct xe_gt *gt, unsigned int vfid, u32 *exec_quantum) +{ + /* GuC will silently clamp values exceeding max */ + *exec_quantum = min_t(u32, *exec_quantum, GUC_KLV_VF_CFG_EXEC_QUANTUM_MAX_VALUE); + + return pf_push_vf_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY, *exec_quantum); +} + +static int pf_push_vf_cfg_preempt_timeout(struct xe_gt *gt, unsigned int vfid, u32 *preempt_timeout) +{ + /* GuC will silently clamp values exceeding max */ + *preempt_timeout = min_t(u32, *preempt_timeout, GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_MAX_VALUE); + + return pf_push_vf_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY, *preempt_timeout); +} + +static int pf_push_vf_cfg_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority) +{ + return pf_push_vf_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_SCHED_PRIORITY_KEY, priority); +} + +static int pf_push_vf_cfg_lmem(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + return pf_push_vf_cfg_u64(gt, vfid, GUC_KLV_VF_CFG_LMEM_SIZE_KEY, size); +} + +static int pf_push_vf_cfg_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index, u32 value) +{ + u32 key = xe_guc_klv_threshold_index_to_key(index); + + xe_gt_assert(gt, key); + return pf_push_vf_cfg_u32(gt, vfid, key, value); +} + +static struct xe_gt_sriov_config *pf_pick_vf_config(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return >->sriov.pf.vfs[vfid].config; +} + +/* Return: number of configuration dwords written */ +static u32 encode_ggtt(u32 *cfg, u64 start, u64 size, bool details) +{ + u32 n = 0; + + if (details) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_GGTT_START); + cfg[n++] = lower_32_bits(start); + cfg[n++] = upper_32_bits(start); + } + + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_GGTT_SIZE); + cfg[n++] = lower_32_bits(size); + cfg[n++] = upper_32_bits(size); + + return n; +} + +/* Return: number of configuration dwords written */ +static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config, bool details) +{ + struct xe_ggtt_node *node = config->ggtt_region; + + if (!node) + return 0; + + return encode_ggtt(cfg, xe_ggtt_node_addr(node), xe_ggtt_node_size(node), details); +} + +static bool custom_sched_priority(struct xe_gt *gt, u32 priority) +{ + return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt) ? + priority != GUC_SCHED_PRIORITY_NORMAL : + priority != GUC_SCHED_PRIORITY_LOW; +} + +static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n, + const struct xe_gt_sriov_config *config) +{ + int i; + + if (xe_sriov_gt_pf_policy_has_multi_group_modes(gt)) { + BUILD_BUG_ON(ARRAY_SIZE(config->exec_quantum) > + GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MAX_LEN); + BUILD_BUG_ON(ARRAY_SIZE(config->preempt_timeout) > + GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN); + + cfg[n++] = PREP_GUC_KLV_CONST(GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY, + ARRAY_SIZE(config->exec_quantum)); + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) + cfg[n++] = config->exec_quantum[i]; + + cfg[n++] = PREP_GUC_KLV_CONST(GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY, + ARRAY_SIZE(config->preempt_timeout)); + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) + cfg[n++] = config->preempt_timeout[i]; + } else { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_EXEC_QUANTUM); + cfg[n++] = config->exec_quantum[0]; + + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT); + cfg[n++] = config->preempt_timeout[0]; + } + + if (custom_sched_priority(gt, config->sched_priority)) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_SCHED_PRIORITY); + cfg[n++] = config->sched_priority; + } + + return n; +} + +/* Return: number of configuration dwords written */ +static u32 encode_config(struct xe_gt *gt, u32 *cfg, const struct xe_gt_sriov_config *config, + bool details) +{ + u32 n = 0; + + n += encode_config_ggtt(cfg, config, details); + + if (details && config->num_ctxs) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_BEGIN_CONTEXT_ID); + cfg[n++] = config->begin_ctx; + } + + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_NUM_CONTEXTS); + cfg[n++] = config->num_ctxs; + + if (details && config->num_dbs) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_BEGIN_DOORBELL_ID); + cfg[n++] = config->begin_db; + } + + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_NUM_DOORBELLS); + cfg[n++] = config->num_dbs; + + if (config->lmem_obj) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_LMEM_SIZE); + cfg[n++] = lower_32_bits(xe_bo_size(config->lmem_obj)); + cfg[n++] = upper_32_bits(xe_bo_size(config->lmem_obj)); + } + + n = encode_config_sched(gt, cfg, n, config); + +#define encode_threshold_config(TAG, NAME, VER...) ({ \ + if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, VER), true, VER)) { \ + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG); \ + cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)]; \ + } \ +}); + + MAKE_XE_GUC_KLV_THRESHOLDS_SET(encode_threshold_config); +#undef encode_threshold_config + + return n; +} + +static int pf_push_full_vf_config(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + u32 max_cfg_dwords = xe_guc_buf_cache_dwords(>->uc.guc.buf); + CLASS(xe_guc_buf, buf)(>->uc.guc.buf, max_cfg_dwords); + u32 num_dwords; + int num_klvs; + u32 *cfg; + int err; + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + cfg = xe_guc_buf_cpu_ptr(buf); + num_dwords = encode_config(gt, cfg, config, true); + xe_gt_assert(gt, num_dwords <= max_cfg_dwords); + + if (xe_gt_is_media_type(gt)) { + struct xe_gt *primary = gt->tile->primary_gt; + struct xe_gt_sriov_config *other = pf_pick_vf_config(primary, vfid); + + /* media-GT will never include a GGTT config */ + xe_gt_assert(gt, !encode_config_ggtt(cfg + num_dwords, config, true)); + + /* the GGTT config must be taken from the primary-GT instead */ + num_dwords += encode_config_ggtt(cfg + num_dwords, other, true); + } + xe_gt_assert(gt, num_dwords <= max_cfg_dwords); + + if (vfid == PFID) { + u64 ggtt_start = xe_ggtt_start(gt_to_tile(gt)->mem.ggtt); + u64 ggtt_size = xe_ggtt_size(gt_to_tile(gt)->mem.ggtt); + + /* plain PF config data will never include a real GGTT region */ + xe_gt_assert(gt, !encode_config_ggtt(cfg + num_dwords, config, true)); + + /* fake PF GGTT config covers full GGTT range except reserved WOPCM */ + num_dwords += encode_ggtt(cfg + num_dwords, ggtt_start, ggtt_size, true); + } + + num_klvs = xe_guc_klv_count(cfg, num_dwords); + err = pf_push_vf_buf_klvs(gt, vfid, num_klvs, buf, num_dwords); + + return err; +} + +static int pf_push_vf_cfg(struct xe_gt *gt, unsigned int vfid, bool reset) +{ + int err = 0; + + xe_gt_assert(gt, vfid); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (reset) + err = pf_send_vf_cfg_reset(gt, vfid); + if (!err) + err = pf_push_full_vf_config(gt, vfid); + + return err; +} + +static int pf_refresh_vf_cfg(struct xe_gt *gt, unsigned int vfid) +{ + return pf_push_vf_cfg(gt, vfid, true); +} + +static u64 pf_get_ggtt_alignment(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + return IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; +} + +static u64 pf_get_min_spare_ggtt(struct xe_gt *gt) +{ + /* XXX: preliminary */ + return IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ? + pf_get_ggtt_alignment(gt) : SZ_64M; +} + +static u64 pf_get_spare_ggtt(struct xe_gt *gt) +{ + u64 spare; + + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + spare = gt->sriov.pf.spare.ggtt_size; + spare = max_t(u64, spare, pf_get_min_spare_ggtt(gt)); + + return spare; +} + +static int pf_set_spare_ggtt(struct xe_gt *gt, u64 size) +{ + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (size && size < pf_get_min_spare_ggtt(gt)) + return -EINVAL; + + size = round_up(size, pf_get_ggtt_alignment(gt)); + gt->sriov.pf.spare.ggtt_size = size; + + return 0; +} + +static int pf_distribute_config_ggtt(struct xe_tile *tile, unsigned int vfid, u64 start, u64 size) +{ + int err, err2 = 0; + + err = pf_push_vf_cfg_ggtt(tile->primary_gt, vfid, start, size); + + if (tile->media_gt && !err) + err2 = pf_push_vf_cfg_ggtt(tile->media_gt, vfid, start, size); + + return err ?: err2; +} + +static void pf_release_vf_config_ggtt(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + xe_ggtt_node_remove(config->ggtt_region, false); + config->ggtt_region = NULL; +} + +static int pf_provision_vf_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_ggtt_node *node; + struct xe_tile *tile = gt_to_tile(gt); + struct xe_ggtt *ggtt = tile->mem.ggtt; + u64 alignment = pf_get_ggtt_alignment(gt); + int err; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + size = round_up(size, alignment); + + if (config->ggtt_region) { + err = pf_distribute_config_ggtt(tile, vfid, 0, 0); + if (unlikely(err)) + return err; + + pf_release_vf_config_ggtt(gt, config); + + err = pf_refresh_vf_cfg(gt, vfid); + if (unlikely(err)) + return err; + } + xe_gt_assert(gt, !config->ggtt_region); + + if (!size) + return 0; + + node = xe_ggtt_insert_node(ggtt, size, alignment); + if (IS_ERR(node)) + return PTR_ERR(node); + + xe_ggtt_assign(node, vfid); + xe_gt_sriov_dbg_verbose(gt, "VF%u assigned GGTT %llx-%llx\n", + vfid, xe_ggtt_node_addr(node), xe_ggtt_node_addr(node) + size - 1); + + err = pf_distribute_config_ggtt(gt->tile, vfid, xe_ggtt_node_addr(node), size); + if (unlikely(err)) + goto err; + + config->ggtt_region = node; + return 0; +err: + xe_ggtt_node_remove(node, false); + return err; +} + +static u64 pf_get_vf_config_ggtt(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_ggtt_node *node = config->ggtt_region; + + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + return node ? xe_ggtt_node_size(node) : 0; +} + +/** + * xe_gt_sriov_pf_config_get_ggtt - Query size of GGTT address space of the VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function can only be called on PF. + * + * Return: size of the VF's assigned (or PF's spare) GGTT address space. + */ +u64 xe_gt_sriov_pf_config_get_ggtt(struct xe_gt *gt, unsigned int vfid) +{ + u64 size; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + size = pf_get_vf_config_ggtt(gt_to_tile(gt)->primary_gt, vfid); + else + size = pf_get_spare_ggtt(gt_to_tile(gt)->primary_gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return size; +} + +static int pf_config_set_u64_done(struct xe_gt *gt, unsigned int vfid, u64 value, + u64 actual, const char *what, int err) +{ + char size[10]; + char name[8]; + + xe_sriov_function_name(vfid, name, sizeof(name)); + + if (unlikely(err)) { + string_get_size(value, 1, STRING_UNITS_2, size, sizeof(size)); + xe_gt_sriov_notice(gt, "Failed to provision %s with %llu (%s) %s (%pe)\n", + name, value, size, what, ERR_PTR(err)); + string_get_size(actual, 1, STRING_UNITS_2, size, sizeof(size)); + xe_gt_sriov_info(gt, "%s provisioning remains at %llu (%s) %s\n", + name, actual, size, what); + return err; + } + + /* the actual value may have changed during provisioning */ + string_get_size(actual, 1, STRING_UNITS_2, size, sizeof(size)); + xe_gt_sriov_info(gt, "%s provisioned with %llu (%s) %s\n", + name, actual, size, what); + return 0; +} + +/** + * xe_gt_sriov_pf_config_set_ggtt - Provision VF with GGTT space. + * @gt: the &xe_gt (can't be media) + * @vfid: the VF identifier + * @size: requested GGTT size + * + * If &vfid represents PF, then function will change PF's spare GGTT config. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + int err; + + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + err = pf_provision_vf_ggtt(gt, vfid, size); + else + err = pf_set_spare_ggtt(gt, size); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_set_u64_done(gt, vfid, size, + xe_gt_sriov_pf_config_get_ggtt(gt, vfid), + vfid ? "GGTT" : "spare GGTT", err); +} + +static int pf_config_bulk_set_u64_done(struct xe_gt *gt, unsigned int first, unsigned int num_vfs, + u64 value, u64 (*get)(struct xe_gt*, unsigned int), + const char *what, unsigned int last, int err) +{ + char size[10]; + + xe_gt_assert(gt, first); + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, first <= last); + + if (num_vfs == 1) + return pf_config_set_u64_done(gt, first, value, get(gt, first), what, err); + + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "Failed to bulk provision VF%u..VF%u with %s\n", + first, first + num_vfs - 1, what); + if (last > first) + pf_config_bulk_set_u64_done(gt, first, last - first, value, + get, what, last, 0); + return pf_config_set_u64_done(gt, last, value, get(gt, last), what, err); + } + + /* pick actual value from first VF - bulk provisioning shall be equal across all VFs */ + value = get(gt, first); + string_get_size(value, 1, STRING_UNITS_2, size, sizeof(size)); + xe_gt_sriov_info(gt, "VF%u..VF%u provisioned with %llu (%s) %s\n", + first, first + num_vfs - 1, value, size, what); + return 0; +} + +/** + * xe_gt_sriov_pf_config_bulk_set_ggtt - Provision many VFs with GGTT. + * @gt: the &xe_gt (can't be media) + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision + * @size: requested GGTT size + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs, u64 size) +{ + unsigned int n; + int err = 0; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + + if (!num_vfs) + return 0; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + for (n = vfid; n < vfid + num_vfs; n++) { + err = pf_provision_vf_ggtt(gt, n, size); + if (err) + break; + } + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_bulk_set_u64_done(gt, vfid, num_vfs, size, + xe_gt_sriov_pf_config_get_ggtt, + "GGTT", n, err); +} + +/* Return: size of the largest continuous GGTT region */ +static u64 pf_get_max_ggtt(struct xe_gt *gt) +{ + struct xe_ggtt *ggtt = gt_to_tile(gt)->mem.ggtt; + u64 alignment = pf_get_ggtt_alignment(gt); + u64 spare = pf_get_spare_ggtt(gt); + u64 max_hole; + + max_hole = xe_ggtt_largest_hole(ggtt, alignment, &spare); + + xe_gt_sriov_dbg_verbose(gt, "HOLE max %lluK reserved %lluK\n", + max_hole / SZ_1K, spare / SZ_1K); + return max_hole > spare ? max_hole - spare : 0; +} + +static u64 pf_estimate_fair_ggtt(struct xe_gt *gt, unsigned int num_vfs) +{ + u64 available = pf_get_max_ggtt(gt); + u64 alignment = pf_get_ggtt_alignment(gt); + u64 fair; + + /* + * To simplify the logic we only look at single largest GGTT region + * as that will be always the best fit for 1 VF case, and most likely + * will also nicely cover other cases where VFs are provisioned on the + * fresh and idle PF driver, without any stale GGTT allocations spread + * in the middle of the full GGTT range. + */ + + fair = div_u64(available, num_vfs); + fair = ALIGN_DOWN(fair, alignment); + xe_gt_sriov_dbg_verbose(gt, "GGTT available(%lluK) fair(%u x %lluK)\n", + available / SZ_1K, num_vfs, fair / SZ_1K); + return fair; +} + +static u64 pf_profile_fair_ggtt(struct xe_gt *gt, unsigned int num_vfs) +{ + bool admin_only_pf = xe_sriov_pf_admin_only(gt_to_xe(gt)); + u64 shareable = ALIGN_DOWN(GUC_GGTT_TOP, SZ_512M); + u64 alignment = pf_get_ggtt_alignment(gt); + + if (admin_only_pf && num_vfs == 1) + return ALIGN_DOWN(shareable, alignment); + + /* need to hardcode due to ~512M of GGTT being reserved */ + if (num_vfs > 56) + return SZ_64M - SZ_8M; + + return rounddown_pow_of_two(div_u64(shareable, num_vfs)); +} + +/** + * xe_gt_sriov_pf_config_set_fair_ggtt - Provision many VFs with fair GGTT. + * @gt: the &xe_gt (can't be media) + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs) +{ + u64 profile = pf_profile_fair_ggtt(gt, num_vfs); + u64 fair; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + fair = pf_estimate_fair_ggtt(gt, num_vfs); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (!fair) + return -ENOSPC; + + fair = min(fair, profile); + if (fair < profile) + xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %llu vs %llu)\n", + "GGTT", fair, profile); + + return xe_gt_sriov_pf_config_bulk_set_ggtt(gt, vfid, num_vfs, fair); +} + +/** + * xe_gt_sriov_pf_config_ggtt_save() - Save a VF provisioned GGTT data into a buffer. + * @gt: the &xe_gt + * @vfid: VF identifier (can't be 0) + * @buf: the GGTT data destination buffer (or NULL to query the buf size) + * @size: the size of the buffer (or 0 to query the buf size) + * + * This function can only be called on PF. + * + * Return: size of the buffer needed to save GGTT data if querying, + * 0 on successful save or a negative error code on failure. + */ +ssize_t xe_gt_sriov_pf_config_ggtt_save(struct xe_gt *gt, unsigned int vfid, + void *buf, size_t size) +{ + struct xe_ggtt_node *node; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, !(!buf ^ !size)); + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + node = pf_pick_vf_config(gt, vfid)->ggtt_region; + + if (!buf) + return xe_ggtt_node_pt_size(node); + + return xe_ggtt_node_save(node, buf, size, vfid); +} + +/** + * xe_gt_sriov_pf_config_ggtt_restore() - Restore a VF provisioned GGTT data from a buffer. + * @gt: the &xe_gt + * @vfid: VF identifier (can't be 0) + * @buf: the GGTT data source buffer + * @size: the size of the buffer + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_ggtt_restore(struct xe_gt *gt, unsigned int vfid, + const void *buf, size_t size) +{ + struct xe_ggtt_node *node; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid); + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + node = pf_pick_vf_config(gt, vfid)->ggtt_region; + + return xe_ggtt_node_load(node, buf, size, vfid); +} + +static u32 pf_get_min_spare_ctxs(struct xe_gt *gt) +{ + /* XXX: preliminary */ + return IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ? + hweight64(gt->info.engine_mask) : SZ_256; +} + +static u32 pf_get_spare_ctxs(struct xe_gt *gt) +{ + u32 spare; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + spare = gt->sriov.pf.spare.num_ctxs; + spare = max_t(u32, spare, pf_get_min_spare_ctxs(gt)); + + return spare; +} + +static int pf_set_spare_ctxs(struct xe_gt *gt, u32 spare) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (spare > GUC_ID_MAX) + return -EINVAL; + + if (spare && spare < pf_get_min_spare_ctxs(gt)) + return -EINVAL; + + gt->sriov.pf.spare.num_ctxs = spare; + + return 0; +} + +/* Return: start ID or negative error code on failure */ +static int pf_reserve_ctxs(struct xe_gt *gt, u32 num) +{ + struct xe_guc_id_mgr *idm = >->uc.guc.submission_state.idm; + unsigned int spare = pf_get_spare_ctxs(gt); + + return xe_guc_id_mgr_reserve(idm, num, spare); +} + +static void pf_release_ctxs(struct xe_gt *gt, u32 start, u32 num) +{ + struct xe_guc_id_mgr *idm = >->uc.guc.submission_state.idm; + + if (num) + xe_guc_id_mgr_release(idm, start, num); +} + +static void pf_release_config_ctxs(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + pf_release_ctxs(gt, config->begin_ctx, config->num_ctxs); + config->begin_ctx = 0; + config->num_ctxs = 0; +} + +static int pf_provision_vf_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int ret; + + xe_gt_assert(gt, vfid); + + if (num_ctxs > GUC_ID_MAX) + return -EINVAL; + + if (config->num_ctxs) { + ret = pf_push_vf_cfg_ctxs(gt, vfid, 0, 0); + if (unlikely(ret)) + return ret; + + pf_release_config_ctxs(gt, config); + + ret = pf_refresh_vf_cfg(gt, vfid); + if (unlikely(ret)) + return ret; + } + + if (!num_ctxs) + return 0; + + ret = pf_reserve_ctxs(gt, num_ctxs); + if (unlikely(ret < 0)) + return ret; + + config->begin_ctx = ret; + config->num_ctxs = num_ctxs; + + ret = pf_push_vf_cfg_ctxs(gt, vfid, config->begin_ctx, config->num_ctxs); + if (unlikely(ret)) { + pf_release_config_ctxs(gt, config); + return ret; + } + + xe_gt_sriov_dbg_verbose(gt, "VF%u contexts %u-%u\n", + vfid, config->begin_ctx, config->begin_ctx + config->num_ctxs - 1); + return 0; +} + +static u32 pf_get_vf_config_ctxs(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->num_ctxs; +} + +/** + * xe_gt_sriov_pf_config_get_ctxs - Get VF's GuC contexts IDs quota. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function can only be called on PF. + * If &vfid represents a PF then number of PF's spare GuC context IDs is returned. + * + * Return: VF's quota (or PF's spare). + */ +u32 xe_gt_sriov_pf_config_get_ctxs(struct xe_gt *gt, unsigned int vfid) +{ + u32 num_ctxs; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + num_ctxs = pf_get_vf_config_ctxs(gt, vfid); + else + num_ctxs = pf_get_spare_ctxs(gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return num_ctxs; +} + +static const char *no_unit(u32 unused) +{ + return ""; +} + +static const char *spare_unit(u32 unused) +{ + return " spare"; +} + +static int pf_config_set_u32_done(struct xe_gt *gt, unsigned int vfid, u32 value, u32 actual, + const char *what, const char *(*unit)(u32), int err) +{ + char name[8]; + + xe_sriov_function_name(vfid, name, sizeof(name)); + + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "Failed to provision %s with %u%s %s (%pe)\n", + name, value, unit(value), what, ERR_PTR(err)); + xe_gt_sriov_info(gt, "%s provisioning remains at %u%s %s\n", + name, actual, unit(actual), what); + return err; + } + + /* the actual value may have changed during provisioning */ + xe_gt_sriov_info(gt, "%s provisioned with %u%s %s\n", + name, actual, unit(actual), what); + return 0; +} + +static char *to_group_name(const char *what, u8 group, char *buf, size_t size) +{ + snprintf(buf, size, "group%u%s%s", group, what ? " " : "", what ?: ""); + return buf; +} + +static int +pf_groups_cfg_set_u32_done(struct xe_gt *gt, unsigned int vfid, u32 *values, u32 count, + void (*get_actual)(struct xe_gt *, unsigned int, u32 *, u32), + const char *what, const char *(*unit)(u32), int err) +{ + u32 actual[GUC_MAX_SCHED_GROUPS]; + char group_name[32]; + u8 g; + + xe_gt_assert(gt, count <= ARRAY_SIZE(actual)); + + get_actual(gt, vfid, actual, count); + + for (g = 0; g < count; g++) + pf_config_set_u32_done(gt, vfid, values[g], actual[g], + to_group_name(what, g, group_name, sizeof(group_name)), + unit, err); + + return err; +} + +/** + * xe_gt_sriov_pf_config_set_ctxs - Configure GuC contexts IDs quota for the VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @num_ctxs: requested number of GuC contexts IDs (0 to release) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs) +{ + int err; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + err = pf_provision_vf_ctxs(gt, vfid, num_ctxs); + else + err = pf_set_spare_ctxs(gt, num_ctxs); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_set_u32_done(gt, vfid, num_ctxs, + xe_gt_sriov_pf_config_get_ctxs(gt, vfid), + "GuC context IDs", vfid ? no_unit : spare_unit, err); +} + +static int pf_config_bulk_set_u32_done(struct xe_gt *gt, unsigned int first, unsigned int num_vfs, + u32 value, u32 (*get)(struct xe_gt*, unsigned int), + const char *what, const char *(*unit)(u32), + unsigned int last, int err) +{ + char name[8]; + + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, first <= last); + + if (num_vfs == 1) + return pf_config_set_u32_done(gt, first, value, get(gt, first), what, unit, err); + + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "Failed to bulk provision %s..VF%u with %s\n", + xe_sriov_function_name(first, name, sizeof(name)), + first + num_vfs - 1, what); + if (last > first) + pf_config_bulk_set_u32_done(gt, first, last - first, value, + get, what, unit, last, 0); + return pf_config_set_u32_done(gt, last, value, get(gt, last), what, unit, err); + } + + /* pick actual value from first VF - bulk provisioning shall be equal across all VFs */ + value = get(gt, first); + xe_gt_sriov_info(gt, "%s..VF%u provisioned with %u%s %s\n", + xe_sriov_function_name(first, name, sizeof(name)), + first + num_vfs - 1, value, unit(value), what); + return 0; +} + +/** + * xe_gt_sriov_pf_config_bulk_set_ctxs - Provision many VFs with GuC context IDs. + * @gt: the &xe_gt + * @vfid: starting VF identifier + * @num_vfs: number of VFs to provision + * @num_ctxs: requested number of GuC contexts IDs (0 to release) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs, u32 num_ctxs) +{ + unsigned int n; + int err = 0; + + xe_gt_assert(gt, vfid); + + if (!num_vfs) + return 0; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + for (n = vfid; n < vfid + num_vfs; n++) { + err = pf_provision_vf_ctxs(gt, n, num_ctxs); + if (err) + break; + } + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_bulk_set_u32_done(gt, vfid, num_vfs, num_ctxs, + xe_gt_sriov_pf_config_get_ctxs, + "GuC context IDs", no_unit, n, err); +} + +static u32 pf_profile_fair_ctxs(struct xe_gt *gt, unsigned int num_vfs) +{ + bool admin_only_pf = xe_sriov_pf_admin_only(gt_to_xe(gt)); + + if (admin_only_pf && num_vfs == 1) + return ALIGN_DOWN(GUC_ID_MAX, SZ_1K); + + return rounddown_pow_of_two(GUC_ID_MAX / num_vfs); +} + +static u32 pf_estimate_fair_ctxs(struct xe_gt *gt, unsigned int num_vfs) +{ + struct xe_guc_id_mgr *idm = >->uc.guc.submission_state.idm; + u32 spare = pf_get_spare_ctxs(gt); + u32 fair = (idm->total - spare) / num_vfs; + int ret; + + for (; fair; --fair) { + ret = xe_guc_id_mgr_reserve(idm, fair * num_vfs, spare); + if (ret < 0) + continue; + xe_guc_id_mgr_release(idm, ret, fair * num_vfs); + break; + } + + xe_gt_sriov_dbg_verbose(gt, "contexts fair(%u x %u)\n", num_vfs, fair); + return fair; +} + +/** + * xe_gt_sriov_pf_config_set_fair_ctxs - Provision many VFs with fair GuC context IDs. + * @gt: the &xe_gt + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs) +{ + u32 profile = pf_profile_fair_ctxs(gt, num_vfs); + u32 fair; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, num_vfs); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + fair = pf_estimate_fair_ctxs(gt, num_vfs); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (!fair) + return -ENOSPC; + + fair = min(fair, profile); + if (fair < profile) + xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %u vs %u)\n", + "GuC context IDs", fair, profile); + + return xe_gt_sriov_pf_config_bulk_set_ctxs(gt, vfid, num_vfs, fair); +} + +static u32 pf_get_min_spare_dbs(struct xe_gt *gt) +{ + /* XXX: preliminary, we don't use doorbells yet! */ + return IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ? 1 : 0; +} + +static u32 pf_get_spare_dbs(struct xe_gt *gt) +{ + u32 spare; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + spare = gt->sriov.pf.spare.num_dbs; + spare = max_t(u32, spare, pf_get_min_spare_dbs(gt)); + + return spare; +} + +static int pf_set_spare_dbs(struct xe_gt *gt, u32 spare) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (spare > GUC_NUM_DOORBELLS) + return -EINVAL; + + if (spare && spare < pf_get_min_spare_dbs(gt)) + return -EINVAL; + + gt->sriov.pf.spare.num_dbs = spare; + return 0; +} + +/* Return: start ID or negative error code on failure */ +static int pf_reserve_dbs(struct xe_gt *gt, u32 num) +{ + struct xe_guc_db_mgr *dbm = >->uc.guc.dbm; + unsigned int spare = pf_get_spare_dbs(gt); + + return xe_guc_db_mgr_reserve_range(dbm, num, spare); +} + +static void pf_release_dbs(struct xe_gt *gt, u32 start, u32 num) +{ + struct xe_guc_db_mgr *dbm = >->uc.guc.dbm; + + if (num) + xe_guc_db_mgr_release_range(dbm, start, num); +} + +static void pf_release_config_dbs(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + pf_release_dbs(gt, config->begin_db, config->num_dbs); + config->begin_db = 0; + config->num_dbs = 0; +} + +static int pf_provision_vf_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int ret; + + xe_gt_assert(gt, vfid); + + if (num_dbs > GUC_NUM_DOORBELLS) + return -EINVAL; + + if (config->num_dbs) { + ret = pf_push_vf_cfg_dbs(gt, vfid, 0, 0); + if (unlikely(ret)) + return ret; + + pf_release_config_dbs(gt, config); + + ret = pf_refresh_vf_cfg(gt, vfid); + if (unlikely(ret)) + return ret; + } + + if (!num_dbs) + return 0; + + ret = pf_reserve_dbs(gt, num_dbs); + if (unlikely(ret < 0)) + return ret; + + config->begin_db = ret; + config->num_dbs = num_dbs; + + ret = pf_push_vf_cfg_dbs(gt, vfid, config->begin_db, config->num_dbs); + if (unlikely(ret)) { + pf_release_config_dbs(gt, config); + return ret; + } + + xe_gt_sriov_dbg_verbose(gt, "VF%u doorbells %u-%u\n", + vfid, config->begin_db, config->begin_db + config->num_dbs - 1); + return 0; +} + +static u32 pf_get_vf_config_dbs(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->num_dbs; +} + +/** + * xe_gt_sriov_pf_config_get_dbs - Get VF's GuC doorbells IDs quota. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function can only be called on PF. + * If &vfid represents a PF then number of PF's spare GuC doorbells IDs is returned. + * + * Return: VF's quota (or PF's spare). + */ +u32 xe_gt_sriov_pf_config_get_dbs(struct xe_gt *gt, unsigned int vfid) +{ + u32 num_dbs; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + num_dbs = pf_get_vf_config_dbs(gt, vfid); + else + num_dbs = pf_get_spare_dbs(gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return num_dbs; +} + +/** + * xe_gt_sriov_pf_config_set_dbs - Configure GuC doorbells IDs quota for the VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @num_dbs: requested number of GuC doorbells IDs (0 to release) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs) +{ + int err; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + err = pf_provision_vf_dbs(gt, vfid, num_dbs); + else + err = pf_set_spare_dbs(gt, num_dbs); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_set_u32_done(gt, vfid, num_dbs, + xe_gt_sriov_pf_config_get_dbs(gt, vfid), + "GuC doorbell IDs", vfid ? no_unit : spare_unit, err); +} + +/** + * xe_gt_sriov_pf_config_bulk_set_dbs - Provision many VFs with GuC context IDs. + * @gt: the &xe_gt + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision + * @num_dbs: requested number of GuC doorbell IDs (0 to release) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs, u32 num_dbs) +{ + unsigned int n; + int err = 0; + + xe_gt_assert(gt, vfid); + + if (!num_vfs) + return 0; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + for (n = vfid; n < vfid + num_vfs; n++) { + err = pf_provision_vf_dbs(gt, n, num_dbs); + if (err) + break; + } + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_bulk_set_u32_done(gt, vfid, num_vfs, num_dbs, + xe_gt_sriov_pf_config_get_dbs, + "GuC doorbell IDs", no_unit, n, err); +} + +static u32 pf_profile_fair_dbs(struct xe_gt *gt, unsigned int num_vfs) +{ + bool admin_only_pf = xe_sriov_pf_admin_only(gt_to_xe(gt)); + + /* XXX: preliminary */ + if (admin_only_pf && num_vfs == 1) + return GUC_NUM_DOORBELLS - SZ_16; + + return rounddown_pow_of_two(GUC_NUM_DOORBELLS / (num_vfs + 1)); +} + +static u32 pf_estimate_fair_dbs(struct xe_gt *gt, unsigned int num_vfs) +{ + struct xe_guc_db_mgr *dbm = >->uc.guc.dbm; + u32 spare = pf_get_spare_dbs(gt); + u32 fair = (GUC_NUM_DOORBELLS - spare) / num_vfs; + int ret; + + for (; fair; --fair) { + ret = xe_guc_db_mgr_reserve_range(dbm, fair * num_vfs, spare); + if (ret < 0) + continue; + xe_guc_db_mgr_release_range(dbm, ret, fair * num_vfs); + break; + } + + xe_gt_sriov_dbg_verbose(gt, "doorbells fair(%u x %u)\n", num_vfs, fair); + return fair; +} + +/** + * xe_gt_sriov_pf_config_set_fair_dbs - Provision many VFs with fair GuC doorbell IDs. + * @gt: the &xe_gt + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs) +{ + u32 profile = pf_profile_fair_dbs(gt, num_vfs); + u32 fair; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, num_vfs); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + fair = pf_estimate_fair_dbs(gt, num_vfs); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (!fair) + return -ENOSPC; + + fair = min(fair, profile); + if (fair < profile) + xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %u vs %u)\n", + "GuC doorbell IDs", fair, profile); + + return xe_gt_sriov_pf_config_bulk_set_dbs(gt, vfid, num_vfs, fair); +} + +static u64 pf_get_lmem_alignment(struct xe_gt *gt) +{ + return xe_device_has_lmtt(gt_to_xe(gt)) ? + xe_lmtt_page_size(>_to_tile(gt)->sriov.pf.lmtt) : XE_PAGE_SIZE; +} + +static u64 pf_get_min_spare_lmem(struct xe_gt *gt) +{ + /* this might be platform dependent */ + return SZ_128M; /* XXX: preliminary */ +} + +static u64 pf_get_spare_lmem(struct xe_gt *gt) +{ + u64 spare; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + spare = gt->sriov.pf.spare.lmem_size; + spare = max_t(u64, spare, pf_get_min_spare_lmem(gt)); + + return spare; +} + +static int pf_set_spare_lmem(struct xe_gt *gt, u64 size) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (size && size < pf_get_min_spare_lmem(gt)) + return -EINVAL; + + gt->sriov.pf.spare.lmem_size = size; + return 0; +} + +static u64 pf_get_vf_config_lmem(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_bo *bo; + + bo = config->lmem_obj; + return bo ? xe_bo_size(bo) : 0; +} + +static int pf_distribute_config_lmem(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_tile *tile; + unsigned int tid; + int err; + + for_each_tile(tile, xe, tid) { + if (tile->primary_gt == gt) { + err = pf_push_vf_cfg_lmem(gt, vfid, size); + } else { + u64 lmem = pf_get_vf_config_lmem(tile->primary_gt, vfid); + + if (!lmem) + continue; + err = pf_push_vf_cfg_lmem(gt, vfid, lmem); + } + if (unlikely(err)) + return err; + } + return 0; +} + +static void pf_force_lmtt_invalidate(struct xe_device *xe) +{ + struct xe_lmtt *lmtt; + struct xe_tile *tile; + unsigned int tid; + + xe_assert(xe, xe_device_has_lmtt(xe)); + xe_assert(xe, IS_SRIOV_PF(xe)); + + for_each_tile(tile, xe, tid) { + lmtt = &tile->sriov.pf.lmtt; + xe_lmtt_invalidate_hw(lmtt); + } +} + +static void pf_reset_vf_lmtt(struct xe_device *xe, unsigned int vfid) +{ + struct xe_lmtt *lmtt; + struct xe_tile *tile; + unsigned int tid; + + xe_assert(xe, xe_device_has_lmtt(xe)); + xe_assert(xe, IS_SRIOV_PF(xe)); + + for_each_tile(tile, xe, tid) { + lmtt = &tile->sriov.pf.lmtt; + xe_lmtt_drop_pages(lmtt, vfid); + } +} + +static int pf_update_vf_lmtt(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt_sriov_config *config; + struct xe_tile *tile; + struct xe_lmtt *lmtt; + struct xe_bo *bo; + struct xe_gt *gt; + u64 total, offset; + unsigned int gtid; + unsigned int tid; + int err; + + xe_assert(xe, xe_device_has_lmtt(xe)); + xe_assert(xe, IS_SRIOV_PF(xe)); + + total = 0; + for_each_tile(tile, xe, tid) + total += pf_get_vf_config_lmem(tile->primary_gt, vfid); + + for_each_tile(tile, xe, tid) { + lmtt = &tile->sriov.pf.lmtt; + + xe_lmtt_drop_pages(lmtt, vfid); + if (!total) + continue; + + err = xe_lmtt_prepare_pages(lmtt, vfid, total); + if (err) + goto fail; + + offset = 0; + for_each_gt_with_type(gt, xe, gtid, BIT(XE_GT_TYPE_MAIN)) { + config = pf_pick_vf_config(gt, vfid); + bo = config->lmem_obj; + if (!bo) + continue; + + err = xe_lmtt_populate_pages(lmtt, vfid, bo, offset); + if (err) + goto fail; + offset += xe_bo_size(bo); + } + } + + pf_force_lmtt_invalidate(xe); + return 0; + +fail: + for_each_tile(tile, xe, tid) { + lmtt = &tile->sriov.pf.lmtt; + xe_lmtt_drop_pages(lmtt, vfid); + } + return err; +} + +/* Return: %true if there was an LMEM provisioned, %false otherwise */ +static bool pf_release_vf_config_lmem(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + xe_gt_assert(gt, IS_DGFX(gt_to_xe(gt))); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (config->lmem_obj) { + xe_bo_unpin_map_no_vm(config->lmem_obj); + config->lmem_obj = NULL; + return true; + } + return false; +} + +static int pf_provision_vf_lmem(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_device *xe = gt_to_xe(gt); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *bo; + u64 alignment; + int err; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, IS_DGFX(xe)); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + + alignment = pf_get_lmem_alignment(gt); + size = round_up(size, alignment); + + if (config->lmem_obj) { + err = pf_distribute_config_lmem(gt, vfid, 0); + if (unlikely(err)) + return err; + + if (xe_device_has_lmtt(xe)) + pf_reset_vf_lmtt(xe, vfid); + pf_release_vf_config_lmem(gt, config); + } + xe_gt_assert(gt, !config->lmem_obj); + + if (!size) + return 0; + + xe_gt_assert(gt, alignment == XE_PAGE_SIZE || alignment == SZ_2M); + bo = xe_bo_create_pin_range_novm(xe, tile, + ALIGN(size, PAGE_SIZE), 0, ~0ull, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM(tile->mem.vram) | + (alignment == SZ_2M ? XE_BO_FLAG_NEEDS_2M : 0) | + XE_BO_FLAG_PINNED | + XE_BO_FLAG_PINNED_LATE_RESTORE | + XE_BO_FLAG_FORCE_USER_VRAM); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + config->lmem_obj = bo; + + if (xe_device_has_lmtt(xe)) { + err = pf_update_vf_lmtt(xe, vfid); + if (unlikely(err)) + goto release; + } + + err = pf_push_vf_cfg_lmem(gt, vfid, xe_bo_size(bo)); + if (unlikely(err)) + goto reset_lmtt; + + xe_gt_sriov_dbg_verbose(gt, "VF%u LMEM %zu (%zuM)\n", + vfid, xe_bo_size(bo), xe_bo_size(bo) / SZ_1M); + return 0; + +reset_lmtt: + if (xe_device_has_lmtt(xe)) + pf_reset_vf_lmtt(xe, vfid); +release: + pf_release_vf_config_lmem(gt, config); + return err; +} + +/** + * xe_gt_sriov_pf_config_get_lmem - Get VF's LMEM quota. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function can only be called on PF. + * + * Return: VF's (or PF's spare) LMEM quota. + */ +u64 xe_gt_sriov_pf_config_get_lmem(struct xe_gt *gt, unsigned int vfid) +{ + u64 size; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + size = pf_get_vf_config_lmem(gt, vfid); + else + size = pf_get_spare_lmem(gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return size; +} + +/** + * xe_gt_sriov_pf_config_set_lmem - Provision VF with LMEM. + * @gt: the &xe_gt (can't be media) + * @vfid: the VF identifier + * @size: requested LMEM size + * + * This function can only be called on PF. + */ +int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + int err; + + if (!xe_device_has_lmtt(gt_to_xe(gt))) + return -EPERM; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + if (vfid) + err = pf_provision_vf_lmem(gt, vfid, size); + else + err = pf_set_spare_lmem(gt, size); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_set_u64_done(gt, vfid, size, + xe_gt_sriov_pf_config_get_lmem(gt, vfid), + vfid ? "LMEM" : "spare LMEM", err); +} + +/** + * xe_gt_sriov_pf_config_bulk_set_lmem_locked() - Provision many VFs with LMEM. + * @gt: the &xe_gt (can't be media) + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision + * @size: requested LMEM size + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_lmem_locked(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs, u64 size) +{ + unsigned int n; + int err = 0; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + xe_gt_assert(gt, xe_device_has_lmtt(gt_to_xe(gt))); + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + xe_gt_assert(gt, vfid); + + if (!num_vfs) + return 0; + + for (n = vfid; n < vfid + num_vfs; n++) { + err = pf_provision_vf_lmem(gt, n, size); + if (err) + break; + } + + return pf_config_bulk_set_u64_done(gt, vfid, num_vfs, size, + pf_get_vf_config_lmem, + "LMEM", n, err); +} + +/** + * xe_gt_sriov_pf_config_bulk_set_lmem() - Provision many VFs with LMEM. + * @gt: the &xe_gt (can't be media) + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision + * @size: requested LMEM size + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs, u64 size) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_config_bulk_set_lmem_locked(gt, vfid, num_vfs, size); +} + +/** + * xe_gt_sriov_pf_config_get_lmem_locked() - Get VF's LMEM quota. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0 == PFID) + * + * This function can only be called on PF. + * + * Return: VF's LMEM quota. + */ +u64 xe_gt_sriov_pf_config_get_lmem_locked(struct xe_gt *gt, unsigned int vfid) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid); + + return pf_get_vf_config_lmem(gt, vfid); +} + +/** + * xe_gt_sriov_pf_config_set_lmem_locked() - Provision VF with LMEM. + * @gt: the &xe_gt (can't be media) + * @vfid: the VF identifier (can't be 0 == PFID) + * @size: requested LMEM size + * + * This function can only be called on PF. + */ +int xe_gt_sriov_pf_config_set_lmem_locked(struct xe_gt *gt, unsigned int vfid, u64 size) +{ + int err; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + xe_gt_assert(gt, xe_device_has_lmtt(gt_to_xe(gt))); + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + xe_gt_assert(gt, vfid); + + err = pf_provision_vf_lmem(gt, vfid, size); + + return pf_config_set_u64_done(gt, vfid, size, + pf_get_vf_config_lmem(gt, vfid), + "LMEM", err); +} + +static struct xe_bo *pf_get_vf_config_lmem_obj(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->lmem_obj; +} + +/** + * xe_gt_sriov_pf_config_get_lmem_obj() - Take a reference to the struct &xe_bo backing VF LMEM. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * + * This function can only be called on PF. + * The caller is responsible for calling xe_bo_put() on the returned object. + * + * Return: pointer to struct &xe_bo backing VF LMEM (if any). + */ +struct xe_bo *xe_gt_sriov_pf_config_get_lmem_obj(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, vfid); + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_bo_get(pf_get_vf_config_lmem_obj(gt, vfid)); +} + +static u64 pf_query_free_lmem(struct xe_gt *gt) +{ + struct xe_tile *tile = gt->tile; + + return xe_ttm_vram_get_avail(&tile->mem.vram->ttm.manager); +} + +static u64 pf_query_max_lmem(struct xe_gt *gt) +{ + u64 alignment = pf_get_lmem_alignment(gt); + u64 spare = pf_get_spare_lmem(gt); + u64 free = pf_query_free_lmem(gt); + u64 avail; + + /* XXX: need to account for 2MB blocks only */ + avail = free > spare ? free - spare : 0; + avail = round_down(avail, alignment); + + return avail; +} + +#ifdef CONFIG_DRM_XE_DEBUG_SRIOV +#define MAX_FAIR_LMEM SZ_128M /* XXX: make it small for the driver bringup */ +#endif + +static u64 pf_estimate_fair_lmem(struct xe_gt *gt, unsigned int num_vfs) +{ + u64 available = pf_query_max_lmem(gt); + u64 alignment = pf_get_lmem_alignment(gt); + u64 fair; + + fair = div_u64(available, num_vfs); + fair = ALIGN_DOWN(fair, alignment); +#ifdef MAX_FAIR_LMEM + fair = min_t(u64, MAX_FAIR_LMEM, fair); +#endif + xe_gt_sriov_dbg_verbose(gt, "LMEM available(%lluM) fair(%u x %lluM)\n", + available / SZ_1M, num_vfs, fair / SZ_1M); + return fair; +} + +static u64 pf_profile_fair_lmem(struct xe_gt *gt, unsigned int num_vfs) +{ + struct xe_tile *tile = gt_to_tile(gt); + bool admin_only_pf = xe_sriov_pf_admin_only(tile->xe); + u64 usable = xe_vram_region_usable_size(tile->mem.vram); + u64 spare = pf_get_min_spare_lmem(gt); + u64 available = usable > spare ? usable - spare : 0; + u64 shareable = ALIGN_DOWN(available, SZ_1G); + u64 alignment = pf_get_lmem_alignment(gt); + u64 fair; + + if (admin_only_pf) + fair = div_u64(shareable, num_vfs); + else + fair = div_u64(shareable, 1 + num_vfs); + + if (!admin_only_pf && fair) + fair = rounddown_pow_of_two(fair); + + return ALIGN_DOWN(fair, alignment); +} + +static void __pf_show_provisioned(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned, + u32 (*get32)(struct xe_gt *, unsigned int), + u64 (*get64)(struct xe_gt *, unsigned int), + const char *what) +{ + unsigned int allvfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); /* PF plus VFs */ + unsigned long *bitmap __free(bitmap) = bitmap_zalloc(allvfs, GFP_KERNEL); + unsigned int weight; + unsigned int n; + bool pf; + + xe_gt_assert(gt, get32 || get64); + + if (!bitmap) + return; + + for (n = first_vf; n < first_vf + num_vfs; n++) { + if ((get32 && (!!get32(gt, VFID(n)) == provisioned)) || + (get64 && (!!get64(gt, VFID(n)) == provisioned))) + bitmap_set(bitmap, n, 1); + } + + pf = test_and_clear_bit(0, bitmap); + weight = bitmap_weight(bitmap, allvfs); + if (!pf && !weight) + return; + + xe_gt_sriov_info(gt, "%s%s%s%s%*pbl %s provisioned with %s\n", + pf ? "PF" : "", pf && weight ? " and " : "", + weight ? "VF" : "", weight > 1 ? "s " : "", + allvfs, bitmap, provisioned ? "already" : "not", what); +} + +static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned) +{ + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + NULL, pf_get_vf_config_lmem, "VRAM"); +} + +static void pf_show_all_provisioned_lmem(struct xe_gt *gt) +{ + __pf_show_provisioning_lmem(gt, VFID(1), xe_gt_sriov_pf_get_totalvfs(gt), true); +} + +static void pf_show_unprovisioned_lmem(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs) +{ + __pf_show_provisioning_lmem(gt, first_vf, num_vfs, false); +} + +static bool pf_needs_provision_lmem(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs) +{ + unsigned int vfid; + + for (vfid = first_vf; vfid < first_vf + num_vfs; vfid++) { + if (pf_get_vf_config_lmem(gt, vfid)) { + pf_show_all_provisioned_lmem(gt); + pf_show_unprovisioned_lmem(gt, first_vf, num_vfs); + return false; + } + } + + pf_show_all_provisioned_lmem(gt); + return true; +} + +/** + * xe_gt_sriov_pf_config_set_fair_lmem - Provision many VFs with fair LMEM. + * @gt: the &xe_gt (can't be media) + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs) +{ + u64 profile; + u64 fair; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, xe_gt_is_main_type(gt)); + + if (!xe_device_has_lmtt(gt_to_xe(gt))) + return 0; + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + if (!pf_needs_provision_lmem(gt, vfid, num_vfs)) + return 0; + + fair = pf_estimate_fair_lmem(gt, num_vfs); + if (!fair) + return -ENOSPC; + + profile = pf_profile_fair_lmem(gt, num_vfs); + fair = min(fair, profile); + if (fair < profile) + xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %llu vs %llu)\n", + "VRAM", fair, profile); + + return xe_gt_sriov_pf_config_bulk_set_lmem_locked(gt, vfid, num_vfs, fair); +} + +/** + * xe_gt_sriov_pf_config_set_fair - Provision many VFs with fair resources. + * @gt: the &xe_gt + * @vfid: starting VF identifier (can't be 0) + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs) +{ + int result = 0; + int err; + + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, num_vfs); + + if (xe_gt_is_main_type(gt)) { + err = xe_gt_sriov_pf_config_set_fair_ggtt(gt, vfid, num_vfs); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_fair_lmem(gt, vfid, num_vfs); + result = result ?: err; + } + err = xe_gt_sriov_pf_config_set_fair_ctxs(gt, vfid, num_vfs); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_fair_dbs(gt, vfid, num_vfs); + result = result ?: err; + + return result; +} + +static const char *exec_quantum_unit(u32 exec_quantum) +{ + return exec_quantum ? "ms" : "(infinity)"; +} + +static int pf_provision_exec_quantum(struct xe_gt *gt, unsigned int vfid, + u32 exec_quantum) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int err; + int i; + + err = pf_push_vf_cfg_exec_quantum(gt, vfid, &exec_quantum); + if (unlikely(err)) + return err; + + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) + config->exec_quantum[i] = exec_quantum; + + return 0; +} + +static u32 pf_get_exec_quantum(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->exec_quantum[0]; +} + +static bool pf_non_default_exec_quantum(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int i; + + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) + if (config->exec_quantum[i]) + return true; + return false; +} + +/** + * xe_gt_sriov_pf_config_set_exec_quantum_locked() - Configure PF/VF execution quantum. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * It will log the provisioned value or an error in case of the failure. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_exec_quantum_locked(struct xe_gt *gt, unsigned int vfid, + u32 exec_quantum) +{ + int err; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_provision_exec_quantum(gt, vfid, exec_quantum); + + return pf_config_set_u32_done(gt, vfid, exec_quantum, + pf_get_exec_quantum(gt, vfid), + "execution quantum", exec_quantum_unit, err); +} + +/** + * xe_gt_sriov_pf_config_set_exec_quantum() - Configure PF/VF execution quantum. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) + * + * This function can only be called on PF. + * It will log the provisioned value or an error in case of the failure. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_exec_quantum(struct xe_gt *gt, unsigned int vfid, + u32 exec_quantum) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_config_set_exec_quantum_locked(gt, vfid, exec_quantum); +} + +/** + * xe_gt_sriov_pf_config_get_exec_quantum_locked() - Get PF/VF execution quantum. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * + * This function can only be called on PF with the master mutex hold. + * + * Return: execution quantum in milliseconds (or 0 if infinity). + */ +u32 xe_gt_sriov_pf_config_get_exec_quantum_locked(struct xe_gt *gt, unsigned int vfid) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_get_exec_quantum(gt, vfid); +} + +/** + * xe_gt_sriov_pf_config_get_exec_quantum() - Get PF/VF execution quantum. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * + * This function can only be called on PF. + * + * Return: execution quantum in milliseconds (or 0 if infinity). + */ +u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_get_exec_quantum(gt, vfid); +} + +static int pf_bulk_set_exec_quantum(struct xe_gt *gt, u32 exec_quantum, + unsigned int first_vf, unsigned int num_vfs) +{ + unsigned int n; + int err = 0; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = first_vf; n < first_vf + num_vfs; n++) { + err = pf_provision_exec_quantum(gt, VFID(n), exec_quantum); + if (err) + break; + } + + return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, exec_quantum, + pf_get_exec_quantum, "execution quantum", + exec_quantum_unit, n, err); +} + +/** + * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs. + * @gt: the &xe_gt to configure + * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + + return pf_bulk_set_exec_quantum(gt, exec_quantum, PFID, 1 + totalvfs); +} + +static int pf_provision_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, + const u32 *exec_quantums, u32 count) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int err; + int i; + + err = pf_push_vf_grp_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY, + exec_quantums, count); + if (unlikely(err)) + return err; + + /* + * GuC silently clamps values exceeding the max and zeroes out the + * quantum for groups not in the klv payload + */ + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) { + if (i < count) + config->exec_quantum[i] = min_t(u32, exec_quantums[i], + GUC_KLV_VF_CFG_EXEC_QUANTUM_MAX_VALUE); + else + config->exec_quantum[i] = 0; + } + + return 0; +} + +static void pf_get_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, + u32 *exec_quantums, u32 max_count) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + u32 count = min_t(u32, max_count, ARRAY_SIZE(config->exec_quantum)); + + memcpy(exec_quantums, config->exec_quantum, sizeof(u32) * count); +} + +/** + * xe_gt_sriov_pf_config_set_groups_exec_quantums() - Configure PF/VF EQs for sched groups. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @exec_quantums: array of requested EQs in milliseconds (0 is infinity) + * @count: number of entries in the array + * + * This function can only be called on PF. + * It will log the provisioned value or an error in case of the failure. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, + u32 *exec_quantums, u32 count) +{ + int err; + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_provision_groups_exec_quantums(gt, vfid, exec_quantums, count); + + return pf_groups_cfg_set_u32_done(gt, vfid, exec_quantums, count, + pf_get_groups_exec_quantums, + "execution quantum", + exec_quantum_unit, err); +} + +/** + * xe_gt_sriov_pf_config_get_groups_exec_quantums() - Get PF/VF sched groups EQs + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @exec_quantums: array in which to store the execution quantums values + * @count: maximum number of entries to store + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_config_get_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, + u32 *exec_quantums, u32 count) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + xe_gt_assert(gt, count <= GUC_MAX_SCHED_GROUPS); + + pf_get_groups_exec_quantums(gt, vfid, exec_quantums, count); +} + +static const char *preempt_timeout_unit(u32 preempt_timeout) +{ + return preempt_timeout ? "us" : "(infinity)"; +} + +static int pf_provision_preempt_timeout(struct xe_gt *gt, unsigned int vfid, + u32 preempt_timeout) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int err; + int i; + + err = pf_push_vf_cfg_preempt_timeout(gt, vfid, &preempt_timeout); + if (unlikely(err)) + return err; + + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) + config->preempt_timeout[i] = preempt_timeout; + + return 0; +} + +static u32 pf_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->preempt_timeout[0]; +} + +static bool pf_non_default_preempt_timeout(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int i; + + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) + if (config->preempt_timeout[i]) + return true; + return false; +} + +/** + * xe_gt_sriov_pf_config_set_preempt_timeout_locked() - Configure PF/VF preemption timeout. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * It will log the provisioned value or an error in case of the failure. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_preempt_timeout_locked(struct xe_gt *gt, unsigned int vfid, + u32 preempt_timeout) +{ + int err; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_provision_preempt_timeout(gt, vfid, preempt_timeout); + + return pf_config_set_u32_done(gt, vfid, preempt_timeout, + pf_get_preempt_timeout(gt, vfid), + "preemption timeout", preempt_timeout_unit, err); +} + +/** + * xe_gt_sriov_pf_config_set_preempt_timeout() - Configure PF/VF preemption timeout. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_preempt_timeout(struct xe_gt *gt, unsigned int vfid, + u32 preempt_timeout) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_config_set_preempt_timeout_locked(gt, vfid, preempt_timeout); +} + +/** + * xe_gt_sriov_pf_config_get_preempt_timeout_locked() - Get PF/VF preemption timeout. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * + * This function can only be called on PF with the master mutex hold. + * + * Return: preemption timeout in microseconds (or 0 if infinity). + */ +u32 xe_gt_sriov_pf_config_get_preempt_timeout_locked(struct xe_gt *gt, unsigned int vfid) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_get_preempt_timeout(gt, vfid); +} + +/** + * xe_gt_sriov_pf_config_get_preempt_timeout() - Get PF/VF preemption timeout. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * + * This function can only be called on PF. + * + * Return: preemption timeout in microseconds (or 0 if infinity). + */ +u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_get_preempt_timeout(gt, vfid); +} + +static int pf_bulk_set_preempt_timeout(struct xe_gt *gt, u32 preempt_timeout, + unsigned int first_vf, unsigned int num_vfs) +{ + unsigned int n; + int err = 0; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = first_vf; n < first_vf + num_vfs; n++) { + err = pf_provision_preempt_timeout(gt, VFID(n), preempt_timeout); + if (err) + break; + } + + return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, preempt_timeout, + pf_get_preempt_timeout, "preemption timeout", + preempt_timeout_unit, n, err); +} + +/** + * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs. + * @gt: the &xe_gt to configure + * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + + return pf_bulk_set_preempt_timeout(gt, preempt_timeout, PFID, 1 + totalvfs); +} + +static int pf_provision_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, + const u32 *preempt_timeouts, u32 count) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int err; + int i; + + err = pf_push_vf_grp_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY, + preempt_timeouts, count); + if (unlikely(err)) + return err; + + /* + * GuC silently clamps values exceeding the max and zeroes out the + * quantum for groups not in the klv payload + */ + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) { + if (i < count) + config->preempt_timeout[i] = + min_t(u32, preempt_timeouts[i], + GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_MAX_VALUE); + else + config->preempt_timeout[i] = 0; + } + + return 0; +} + +static void pf_get_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, + u32 *preempt_timeouts, u32 max_count) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + u32 count = min_t(u32, max_count, ARRAY_SIZE(config->preempt_timeout)); + + memcpy(preempt_timeouts, config->preempt_timeout, sizeof(u32) * count); +} + +/** + * xe_gt_sriov_pf_config_set_groups_preempt_timeouts() - Configure PF/VF PTs for sched groups. + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @preempt_timeouts: array of requested PTs in microseconds (0 is infinity) + * @count: number of entries in the array + * + * This function can only be called on PF. + * It will log the provisioned value or an error in case of the failure. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, + u32 *preempt_timeouts, u32 count) +{ + int err; + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_provision_groups_preempt_timeouts(gt, vfid, preempt_timeouts, count); + + return pf_groups_cfg_set_u32_done(gt, vfid, preempt_timeouts, count, + pf_get_groups_preempt_timeouts, + "preempt_timeout", + preempt_timeout_unit, err); +} + +/** + * xe_gt_sriov_pf_config_get_groups_preempt_timeouts() - Get PF/VF sched groups PTs + * @gt: the &xe_gt + * @vfid: the PF or VF identifier + * @preempt_timeouts: array in which to store the preemption timeouts values + * @count: maximum number of entries to store + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_config_get_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, + u32 *preempt_timeouts, u32 count) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + xe_gt_assert(gt, count <= GUC_MAX_SCHED_GROUPS); + + pf_get_groups_preempt_timeouts(gt, vfid, preempt_timeouts, count); +} + +static const char *sched_priority_unit(u32 priority) +{ + return priority == GUC_SCHED_PRIORITY_LOW ? "(low)" : + priority == GUC_SCHED_PRIORITY_NORMAL ? "(normal)" : + priority == GUC_SCHED_PRIORITY_HIGH ? "(high)" : + "(?)"; +} + +static int pf_provision_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int err; + + err = pf_push_vf_cfg_sched_priority(gt, vfid, priority); + if (unlikely(err)) + return err; + + config->sched_priority = priority; + return 0; +} + +static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->sched_priority; +} + +/** + * xe_gt_sriov_pf_config_set_sched_priority() - Configure scheduling priority. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @priority: requested scheduling priority + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority) +{ + int err; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_provision_sched_priority(gt, vfid, priority); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_set_u32_done(gt, vfid, priority, + xe_gt_sriov_pf_config_get_sched_priority(gt, vfid), + "scheduling priority", sched_priority_unit, err); +} + +/** + * xe_gt_sriov_pf_config_get_sched_priority - Get VF's scheduling priority. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function can only be called on PF. + * + * Return: VF's (or PF's) scheduling priority. + */ +u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid) +{ + u32 priority; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + priority = pf_get_sched_priority(gt, vfid); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return priority; +} + +/** + * xe_gt_sriov_pf_config_force_sched_priority_locked() - Force update scheduling priority. + * @gt: the &xe_gt + * @priority: new scheduling priority to set + * + * This function allows to update cached values of the scheduling priorities of all + * VFs (and PF) as result of applying the `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ policy. + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority) +{ + unsigned int total_vfs = xe_gt_sriov_pf_get_totalvfs(gt); + struct xe_gt_sriov_config *config; + unsigned int n; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 0; n <= total_vfs; n++) { + config = pf_pick_vf_config(gt, VFID(n)); + config->sched_priority = priority; + } + + pf_config_bulk_set_u32_done(gt, PFID, 1 + total_vfs, priority, + pf_get_sched_priority, "scheduling priority", + sched_priority_unit, n, 0); +} + +static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + int i; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) { + config->exec_quantum[i] = 0; + config->preempt_timeout[i] = 0; + } +} + +static bool pf_non_default_sched(struct xe_gt *gt, unsigned int vfid) +{ + return pf_non_default_exec_quantum(gt, vfid) || + pf_non_default_preempt_timeout(gt, vfid) || + custom_sched_priority(gt, pf_get_sched_priority(gt, vfid)); +} + +static void __pf_show_provisioned_sched(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned) +{ + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + pf_get_exec_quantum, NULL, "EQ"); + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + pf_get_preempt_timeout, NULL, "PT"); + + /* we only care about non-default priorities */ + if (provisioned) + __pf_show_provisioned(gt, first_vf, num_vfs, true, + pf_get_sched_priority, NULL, "PRIORITY"); +} + +static void pf_show_all_provisioned_sched(struct xe_gt *gt) +{ + __pf_show_provisioned_sched(gt, PFID, 1 + xe_gt_sriov_pf_get_totalvfs(gt), true); +} + +static void pf_show_unprovisioned_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + __pf_show_provisioned_sched(gt, PFID, 1 + num_vfs, false); +} + +static bool pf_needs_provision_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + unsigned int vfid; + + for (vfid = PFID; vfid <= PFID + num_vfs; vfid++) { + if (pf_non_default_sched(gt, vfid)) { + pf_show_all_provisioned_sched(gt); + pf_show_unprovisioned_sched(gt, num_vfs); + return false; + } + } + + if (xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)) { + pf_show_all_provisioned_sched(gt); + pf_show_unprovisioned_sched(gt, num_vfs); + return false; + } + + pf_show_all_provisioned_sched(gt); + return true; +} + +/* With 16ms EQ/PT GuC should be able to handle up to 63 VFs within 2s */ +#define XE_FAIR_EXEC_QUANTUM_MS 16 +#define XE_FAIR_PREEMPT_TIMEOUT_US 16000 +#define XE_FAIR_SCHED_PRIORITY GUC_SCHED_PRIORITY_LOW +#define XE_ADMIN_PF_SCHED_PRIORITY GUC_SCHED_PRIORITY_HIGH + +/** + * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling. + * @gt: the &xe_gt + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + int result = 0; + int err; + + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, XE_FAIR_EXEC_QUANTUM_MS); + xe_gt_assert(gt, XE_FAIR_PREEMPT_TIMEOUT_US); + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + if (!pf_needs_provision_sched(gt, num_vfs)) + return 0; + + err = pf_bulk_set_exec_quantum(gt, XE_FAIR_EXEC_QUANTUM_MS, PFID, 1 + num_vfs); + result = result ?: err; + err = pf_bulk_set_preempt_timeout(gt, XE_FAIR_PREEMPT_TIMEOUT_US, PFID, 1 + num_vfs); + result = result ?: err; + + xe_gt_assert(gt, XE_FAIR_SCHED_PRIORITY == GUC_SCHED_PRIORITY_LOW); + xe_gt_assert(gt, !xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)); + + if (xe_sriov_pf_admin_only(gt_to_xe(gt))) { + err = pf_provision_sched_priority(gt, PFID, XE_ADMIN_PF_SCHED_PRIORITY); + result = result ?: err; + } + + return result; +} + +static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index, u32 value) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int err; + + err = pf_push_vf_cfg_threshold(gt, vfid, index, value); + if (unlikely(err)) + return err; + + config->thresholds[index] = value; + + return 0; +} + +static int pf_get_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + + return config->thresholds[index]; +} + +static const char *threshold_unit(u32 threshold) +{ + return threshold ? "" : "(disabled)"; +} + +/** + * xe_gt_sriov_pf_config_set_threshold - Configure threshold for the VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @index: the threshold index + * @value: requested value (0 means disabled) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index, u32 value) +{ + u32 key = xe_guc_klv_threshold_index_to_key(index); + const char *name = xe_guc_klv_key_to_string(key); + int err; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_provision_threshold(gt, vfid, index, value); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_config_set_u32_done(gt, vfid, value, + xe_gt_sriov_pf_config_get_threshold(gt, vfid, index), + name, threshold_unit, err); +} + +/** + * xe_gt_sriov_pf_config_get_threshold - Get VF's threshold. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @index: the threshold index + * + * This function can only be called on PF. + * + * Return: value of VF's (or PF's) threshold. + */ +u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index) +{ + u32 value; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + value = pf_get_threshold(gt, vfid, index); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return value; +} + +static void pf_reset_config_thresholds(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + +#define reset_threshold_config(TAG, ...) ({ \ + config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)] = 0; \ +}); + + MAKE_XE_GUC_KLV_THRESHOLDS_SET(reset_threshold_config); +#undef reset_threshold_config +} + +static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_device *xe = gt_to_xe(gt); + bool released; + + if (xe_gt_is_main_type(gt)) { + pf_release_vf_config_ggtt(gt, config); + if (IS_DGFX(xe)) { + released = pf_release_vf_config_lmem(gt, config); + if (released && xe_device_has_lmtt(xe)) + pf_update_vf_lmtt(xe, vfid); + } + } + pf_release_config_ctxs(gt, config); + pf_release_config_dbs(gt, config); + pf_reset_config_sched(gt, config); + pf_reset_config_thresholds(gt, config); +} + +/** + * xe_gt_sriov_pf_config_release - Release and reset VF configuration. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @force: force configuration release + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force) +{ + int err; + + xe_gt_assert(gt, vfid); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_send_vf_cfg_reset(gt, vfid); + if (!err || force) + pf_release_vf_config(gt, vfid); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "VF%u unprovisioning failed with error (%pe)%s\n", + vfid, ERR_PTR(err), + force ? " but all resources were released anyway!" : ""); + } + + return force ? 0 : err; +} + +static void pf_sanitize_ggtt(struct xe_ggtt_node *ggtt_region, unsigned int vfid) +{ + if (ggtt_region) + xe_ggtt_assign(ggtt_region, vfid); +} + +static int pf_sanitize_lmem(struct xe_tile *tile, struct xe_bo *bo, long timeout) +{ + struct xe_migrate *m = tile->migrate; + struct dma_fence *fence; + int err; + + if (!bo) + return 0; + + xe_bo_lock(bo, false); + fence = xe_migrate_clear(m, bo, bo->ttm.resource, XE_MIGRATE_CLEAR_FLAG_FULL); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + } else if (!fence) { + err = -ENOMEM; + } else { + long ret = dma_fence_wait_timeout(fence, false, timeout); + + err = ret > 0 ? 0 : ret < 0 ? ret : -ETIMEDOUT; + dma_fence_put(fence); + if (!err) + xe_gt_sriov_dbg_verbose(tile->primary_gt, "LMEM cleared in %dms\n", + jiffies_to_msecs(timeout - ret)); + } + xe_bo_unlock(bo); + + return err; +} + +static int pf_sanitize_vf_resources(struct xe_gt *gt, u32 vfid, long timeout) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + int err = 0; + + /* + * Only GGTT and LMEM requires to be cleared by the PF. + * GuC doorbell IDs and context IDs do not need any clearing. + */ + if (xe_gt_is_main_type(gt)) { + pf_sanitize_ggtt(config->ggtt_region, vfid); + if (IS_DGFX(xe)) + err = pf_sanitize_lmem(tile, config->lmem_obj, timeout); + } + + return err; +} + +/** + * xe_gt_sriov_pf_config_sanitize() - Sanitize VF's resources. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @timeout: maximum timeout to wait for completion in jiffies + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout) +{ + int err; + + xe_gt_assert(gt, vfid != PFID); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_sanitize_vf_resources(gt, vfid, timeout); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (unlikely(err)) + xe_gt_sriov_notice(gt, "VF%u resource sanitizing failed (%pe)\n", + vfid, ERR_PTR(err)); + return err; +} + +/** + * xe_gt_sriov_pf_config_push - Reprovision VF's configuration. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @refresh: explicit refresh + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh) +{ + int err = 0; + + xe_gt_assert(gt, vfid); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_push_vf_cfg(gt, vfid, refresh); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "Failed to %s VF%u configuration (%pe)\n", + refresh ? "refresh" : "push", vfid, ERR_PTR(err)); + } + + return err; +} + +static int pf_validate_vf_config(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt *primary_gt = gt_to_tile(gt)->primary_gt; + struct xe_device *xe = gt_to_xe(gt); + bool is_primary = xe_gt_is_main_type(gt); + bool valid_ggtt, valid_ctxs, valid_dbs; + bool valid_any, valid_all; + + valid_ggtt = pf_get_vf_config_ggtt(primary_gt, vfid); + valid_ctxs = pf_get_vf_config_ctxs(gt, vfid); + valid_dbs = pf_get_vf_config_dbs(gt, vfid); + + /* note that GuC doorbells are optional */ + valid_any = valid_ctxs || valid_dbs; + valid_all = valid_ctxs; + + /* and GGTT/LMEM is configured on primary GT only */ + valid_all = valid_all && valid_ggtt; + valid_any = valid_any || (valid_ggtt && is_primary); + + if (xe_device_has_lmtt(xe)) { + bool valid_lmem = pf_get_vf_config_lmem(primary_gt, vfid); + + valid_any = valid_any || (valid_lmem && is_primary); + valid_all = valid_all && valid_lmem; + } + + /* also check optional EQ/PT/PRIO */ + valid_any = valid_any || pf_non_default_sched(gt, vfid); + + return valid_all ? 0 : valid_any ? -ENOKEY : -ENODATA; +} + +/** + * xe_gt_sriov_pf_config_is_empty - Check VF's configuration. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * + * This function can only be called on PF. + * + * Return: true if VF mandatory configuration (GGTT, LMEM, ...) is empty. + */ +bool xe_gt_sriov_pf_config_is_empty(struct xe_gt *gt, unsigned int vfid) +{ + bool empty; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + empty = pf_validate_vf_config(gt, vfid) == -ENODATA; + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return empty; +} + +/** + * xe_gt_sriov_pf_config_save - Save a VF provisioning config as binary blob. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @buf: the buffer to save a config to (or NULL if query the buf size) + * @size: the size of the buffer (or 0 if query the buf size) + * + * This function can only be called on PF. + * + * Return: minimum size of the buffer or the number of bytes saved, + * or a negative error code on failure. + */ +ssize_t xe_gt_sriov_pf_config_save(struct xe_gt *gt, unsigned int vfid, void *buf, size_t size) +{ + struct xe_gt_sriov_config *config; + ssize_t ret; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid); + xe_gt_assert(gt, !(!buf ^ !size)); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + ret = pf_validate_vf_config(gt, vfid); + if (!size) { + ret = ret ? 0 : SZ_4K; + } else if (!ret) { + if (size < SZ_4K) { + ret = -ENOBUFS; + } else { + config = pf_pick_vf_config(gt, vfid); + ret = encode_config(gt, buf, config, false) * sizeof(u32); + } + } + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return ret; +} + +static int pf_restore_vf_config_klv(struct xe_gt *gt, unsigned int vfid, + u32 key, u32 len, const u32 *value) +{ + switch (key) { + case GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY: + if (len != GUC_KLV_VF_CFG_NUM_CONTEXTS_LEN) + return -EBADMSG; + return pf_provision_vf_ctxs(gt, vfid, value[0]); + + case GUC_KLV_VF_CFG_NUM_DOORBELLS_KEY: + if (len != GUC_KLV_VF_CFG_NUM_DOORBELLS_LEN) + return -EBADMSG; + return pf_provision_vf_dbs(gt, vfid, value[0]); + + case GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY: + if (len != GUC_KLV_VF_CFG_EXEC_QUANTUM_LEN) + return -EBADMSG; + return pf_provision_exec_quantum(gt, vfid, value[0]); + + case GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY: + if (len > GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MAX_LEN) + return -EBADMSG; + return pf_provision_groups_exec_quantums(gt, vfid, value, len); + + case GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY: + if (len > GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN) + return -EBADMSG; + return pf_provision_groups_preempt_timeouts(gt, vfid, value, len); + + case GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY: + if (len != GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_LEN) + return -EBADMSG; + return pf_provision_preempt_timeout(gt, vfid, value[0]); + + /* auto-generate case statements */ +#define define_threshold_key_to_provision_case(TAG, NAME, VER...) \ + case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG): \ + BUILD_BUG_ON(MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG) != 1u); \ + if (len != MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG)) \ + return -EBADMSG; \ + if (IF_ARGS(!GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, VER), false, VER)) \ + return -EKEYREJECTED; \ + return pf_provision_threshold(gt, vfid, \ + MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG), \ + value[0]); + + MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_threshold_key_to_provision_case) +#undef define_threshold_key_to_provision_case + } + + if (xe_gt_is_media_type(gt)) + return -EKEYREJECTED; + + switch (key) { + case GUC_KLV_VF_CFG_GGTT_SIZE_KEY: + if (len != GUC_KLV_VF_CFG_GGTT_SIZE_LEN) + return -EBADMSG; + return pf_provision_vf_ggtt(gt, vfid, make_u64_from_u32(value[1], value[0])); + + case GUC_KLV_VF_CFG_LMEM_SIZE_KEY: + if (!IS_DGFX(gt_to_xe(gt))) + return -EKEYREJECTED; + if (len != GUC_KLV_VF_CFG_LMEM_SIZE_LEN) + return -EBADMSG; + return pf_provision_vf_lmem(gt, vfid, make_u64_from_u32(value[1], value[0])); + } + + return -EKEYREJECTED; +} + +static int pf_restore_vf_config(struct xe_gt *gt, unsigned int vfid, + const u32 *klvs, size_t num_dwords) +{ + int err; + + while (num_dwords >= GUC_KLV_LEN_MIN) { + u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]); + u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + + klvs += GUC_KLV_LEN_MIN; + num_dwords -= GUC_KLV_LEN_MIN; + + if (num_dwords < len) + err = -EBADMSG; + else + err = pf_restore_vf_config_klv(gt, vfid, key, len, klvs); + + if (err) { + xe_gt_sriov_dbg(gt, "restore failed on key %#x (%pe)\n", key, ERR_PTR(err)); + return err; + } + + klvs += len; + num_dwords -= len; + } + + return pf_validate_vf_config(gt, vfid); +} + +/** + * xe_gt_sriov_pf_config_restore - Restore a VF provisioning config from binary blob. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @buf: the buffer with config data + * @size: the size of the config data + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_restore(struct xe_gt *gt, unsigned int vfid, + const void *buf, size_t size) +{ + int err; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid); + + if (!size) + return -ENODATA; + + if (size % sizeof(u32)) + return -EINVAL; + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + + drm_printf(&p, "restoring VF%u config:\n", vfid); + xe_guc_klv_print(buf, size / sizeof(u32), &p); + } + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_send_vf_cfg_reset(gt, vfid); + if (!err) { + pf_release_vf_config(gt, vfid); + err = pf_restore_vf_config(gt, vfid, buf, size / sizeof(u32)); + } + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return err; +} + +static void pf_prepare_self_config(struct xe_gt *gt) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, PFID); + + /* + * We want PF to be allowed to use all of context ID, doorbells IDs + * and whole usable GGTT area. While we can store ctxs/dbs numbers + * directly in the config structure, can't do the same with the GGTT + * configuration, so let it be prepared on demand while pushing KLVs. + */ + config->num_ctxs = GUC_ID_MAX; + config->num_dbs = GUC_NUM_DOORBELLS; +} + +static int pf_push_self_config(struct xe_gt *gt) +{ + int err; + + err = pf_push_full_vf_config(gt, PFID); + if (err) { + xe_gt_sriov_err(gt, "Failed to push self configuration (%pe)\n", + ERR_PTR(err)); + return err; + } + + xe_gt_sriov_dbg_verbose(gt, "self configuration completed\n"); + return 0; +} + +static void fini_config(void *arg) +{ + struct xe_gt *gt = arg; + struct xe_device *xe = gt_to_xe(gt); + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(xe); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + for (n = 1; n <= total_vfs; n++) + pf_release_vf_config(gt, n); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); +} + +/** + * xe_gt_sriov_pf_config_init - Initialize SR-IOV configuration data. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + pf_prepare_self_config(gt); + err = pf_push_self_config(gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, fini_config, gt); +} + +/** + * xe_gt_sriov_pf_config_restart - Restart SR-IOV configurations after a GT reset. + * @gt: the &xe_gt + * + * Any prior configurations pushed to GuC are lost when the GT is reset. + * Push again all non-empty VF configurations to the GuC. + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_config_restart(struct xe_gt *gt) +{ + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt)); + unsigned int fail = 0, skip = 0; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + pf_push_self_config(gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 1; n <= total_vfs; n++) { + if (xe_gt_sriov_pf_config_is_empty(gt, n)) + skip++; + else if (xe_gt_sriov_pf_config_push(gt, n, false)) + fail++; + } + + if (fail) + xe_gt_sriov_notice(gt, "Failed to push %u of %u VF%s configurations\n", + fail, total_vfs - skip, str_plural(total_vfs)); + + if (fail != total_vfs) + xe_gt_sriov_dbg(gt, "pushed %u skip %u of %u VF%s configurations\n", + total_vfs - skip - fail, skip, total_vfs, str_plural(total_vfs)); +} + +/** + * xe_gt_sriov_pf_config_print_ggtt - Print GGTT configurations. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Print GGTT configuration data for all VFs. + * VFs without provisioned GGTT are ignored. + * + * This function can only be called on PF. + */ +int xe_gt_sriov_pf_config_print_ggtt(struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt)); + const struct xe_gt_sriov_config *config; + char buf[10]; + + for (n = 1; n <= total_vfs; n++) { + config = >->sriov.pf.vfs[n].config; + if (!config->ggtt_region) + continue; + + string_get_size(xe_ggtt_node_size(config->ggtt_region), 1, STRING_UNITS_2, + buf, sizeof(buf)); + drm_printf(p, "VF%u:\t%#0llx-%#llx\t(%s)\n", + n, xe_ggtt_node_addr(config->ggtt_region), + xe_ggtt_node_addr(config->ggtt_region) + + xe_ggtt_node_size(config->ggtt_region) - 1, + buf); + } + + return 0; +} + +/** + * xe_gt_sriov_pf_config_print_ctxs - Print GuC context IDs configurations. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Print GuC context ID allocations across all VFs. + * VFs without GuC context IDs are skipped. + * + * This function can only be called on PF. + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_print_ctxs(struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt)); + const struct xe_gt_sriov_config *config; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 1; n <= total_vfs; n++) { + config = >->sriov.pf.vfs[n].config; + if (!config->num_ctxs) + continue; + + drm_printf(p, "VF%u:\t%u-%u\t(%u)\n", + n, + config->begin_ctx, + config->begin_ctx + config->num_ctxs - 1, + config->num_ctxs); + } + + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + return 0; +} + +/** + * xe_gt_sriov_pf_config_print_dbs - Print GuC doorbell ID configurations. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Print GuC doorbell IDs allocations across all VFs. + * VFs without GuC doorbell IDs are skipped. + * + * This function can only be called on PF. + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_print_dbs(struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt)); + const struct xe_gt_sriov_config *config; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 1; n <= total_vfs; n++) { + config = >->sriov.pf.vfs[n].config; + if (!config->num_dbs) + continue; + + drm_printf(p, "VF%u:\t%u-%u\t(%u)\n", + n, + config->begin_db, + config->begin_db + config->num_dbs - 1, + config->num_dbs); + } + + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + return 0; +} + +/** + * xe_gt_sriov_pf_config_print_lmem - Print LMEM configurations. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Print LMEM allocations across all VFs. + * VFs without LMEM allocation are skipped. + * + * This function can only be called on PF. + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_print_lmem(struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt)); + const struct xe_gt_sriov_config *config; + char buf[10]; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 1; n <= total_vfs; n++) { + config = >->sriov.pf.vfs[n].config; + if (!config->lmem_obj) + continue; + + string_get_size(xe_bo_size(config->lmem_obj), 1, STRING_UNITS_2, + buf, sizeof(buf)); + drm_printf(p, "VF%u:\t%zu\t(%s)\n", + n, xe_bo_size(config->lmem_obj), buf); + } + + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + return 0; +} + +/** + * xe_gt_sriov_pf_config_print_available_ggtt - Print available GGTT ranges. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Print GGTT ranges that are available for the provisioning. + * + * This function can only be called on PF. + */ +int xe_gt_sriov_pf_config_print_available_ggtt(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_ggtt *ggtt = gt_to_tile(gt)->mem.ggtt; + u64 alignment = pf_get_ggtt_alignment(gt); + u64 spare, avail, total; + char buf[10]; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + + spare = pf_get_spare_ggtt(gt); + total = xe_ggtt_print_holes(ggtt, alignment, p); + + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + string_get_size(total, 1, STRING_UNITS_2, buf, sizeof(buf)); + drm_printf(p, "total:\t%llu\t(%s)\n", total, buf); + + string_get_size(spare, 1, STRING_UNITS_2, buf, sizeof(buf)); + drm_printf(p, "spare:\t%llu\t(%s)\n", spare, buf); + + avail = total > spare ? total - spare : 0; + + string_get_size(avail, 1, STRING_UNITS_2, buf, sizeof(buf)); + drm_printf(p, "avail:\t%llu\t(%s)\n", avail, buf); + + return 0; +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_gt_sriov_pf_config_kunit.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h new file mode 100644 index 000000000..2ec62c12a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_CONFIG_H_ +#define _XE_GT_SRIOV_PF_CONFIG_H_ + +#include + +enum xe_guc_klv_threshold_index; +struct drm_printer; +struct xe_gt; + +u64 xe_gt_sriov_pf_config_get_ggtt(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size); +int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt, + unsigned int vfid, unsigned int num_vfs); +int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt, + unsigned int vfid, unsigned int num_vfs, u64 size); + +u32 xe_gt_sriov_pf_config_get_ctxs(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs); +int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); +int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs, + u32 num_ctxs); + +u32 xe_gt_sriov_pf_config_get_dbs(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs); +int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); +int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs, + u32 num_dbs); + +u64 xe_gt_sriov_pf_config_get_lmem(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size); +int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); +int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs, + u64 size); +u64 xe_gt_sriov_pf_config_get_lmem_locked(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_lmem_locked(struct xe_gt *gt, unsigned int vfid, u64 size); +int xe_gt_sriov_pf_config_bulk_set_lmem_locked(struct xe_gt *gt, unsigned int vfid, + unsigned int num_vfs, u64 size); +struct xe_bo *xe_gt_sriov_pf_config_get_lmem_obj(struct xe_gt *gt, unsigned int vfid); + +u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_exec_quantum(struct xe_gt *gt, unsigned int vfid, u32 exec_quantum); + +u32 xe_gt_sriov_pf_config_get_exec_quantum_locked(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_exec_quantum_locked(struct xe_gt *gt, unsigned int vfid, + u32 exec_quantum); +int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum); + +void xe_gt_sriov_pf_config_get_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, + u32 *exec_quantum, u32 max_count); +int xe_gt_sriov_pf_config_set_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, + u32 *exec_quantum, u32 count); + +u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_preempt_timeout(struct xe_gt *gt, unsigned int vfid, + u32 preempt_timeout); + +u32 xe_gt_sriov_pf_config_get_preempt_timeout_locked(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_preempt_timeout_locked(struct xe_gt *gt, unsigned int vfid, + u32 preempt_timeout); +int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout); + +void xe_gt_sriov_pf_config_get_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, + u32 *preempt_timeout, u32 max_count); +int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, + u32 *preempt_timeout, u32 count); + +u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority); +void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority); + +u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index); +int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid, + enum xe_guc_klv_threshold_index index, u32 value); + +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs); +int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); +int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout); +int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force); +int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh); + +ssize_t xe_gt_sriov_pf_config_save(struct xe_gt *gt, unsigned int vfid, void *buf, size_t size); +int xe_gt_sriov_pf_config_restore(struct xe_gt *gt, unsigned int vfid, + const void *buf, size_t size); + +ssize_t xe_gt_sriov_pf_config_ggtt_save(struct xe_gt *gt, unsigned int vfid, + void *buf, size_t size); +int xe_gt_sriov_pf_config_ggtt_restore(struct xe_gt *gt, unsigned int vfid, + const void *buf, size_t size); + +bool xe_gt_sriov_pf_config_is_empty(struct xe_gt *gt, unsigned int vfid); + +int xe_gt_sriov_pf_config_init(struct xe_gt *gt); +void xe_gt_sriov_pf_config_restart(struct xe_gt *gt); + +int xe_gt_sriov_pf_config_print_ggtt(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_pf_config_print_ctxs(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_pf_config_print_dbs(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_pf_config_print_lmem(struct xe_gt *gt, struct drm_printer *p); + +int xe_gt_sriov_pf_config_print_available_ggtt(struct xe_gt *gt, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h new file mode 100644 index 000000000..75a48d0fa --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_CONFIG_TYPES_H_ +#define _XE_GT_SRIOV_PF_CONFIG_TYPES_H_ + +#include "abi/guc_scheduler_abi.h" +#include "xe_ggtt_types.h" +#include "xe_guc_klv_thresholds_set_types.h" + +struct xe_bo; + +/** + * struct xe_gt_sriov_config - GT level per-VF configuration data. + * + * Used by the PF driver to maintain per-VF provisioning data. + */ +struct xe_gt_sriov_config { + /** @ggtt_region: GGTT region assigned to the VF. */ + struct xe_ggtt_node *ggtt_region; + /** @lmem_obj: LMEM allocation for use by the VF. */ + struct xe_bo *lmem_obj; + /** @num_ctxs: number of GuC contexts IDs. */ + u16 num_ctxs; + /** @begin_ctx: start index of GuC context ID range. */ + u16 begin_ctx; + /** @num_dbs: number of GuC doorbells IDs. */ + u16 num_dbs; + /** @begin_db: start index of GuC doorbell ID range. */ + u16 begin_db; + /** @exec_quantum: execution-quantum in milliseconds. */ + u32 exec_quantum[GUC_MAX_SCHED_GROUPS]; + /** @preempt_timeout: preemption timeout in microseconds. */ + u32 preempt_timeout[GUC_MAX_SCHED_GROUPS]; + /** @sched_priority: scheduling priority. */ + u32 sched_priority; + /** @thresholds: GuC thresholds for adverse events notifications. */ + u32 thresholds[XE_GUC_KLV_NUM_THRESHOLDS]; +}; + +/** + * struct xe_gt_sriov_spare_config - GT-level PF spare configuration data. + * + * Used by the PF driver to maintain it's own reserved (spare) provisioning + * data that is not applicable to be tracked in struct xe_gt_sriov_config. + */ +struct xe_gt_sriov_spare_config { + /** @ggtt_size: GGTT size. */ + u64 ggtt_size; + /** @lmem_size: LMEM size. */ + u64 lmem_size; + /** @num_ctxs: number of GuC submission contexts. */ + u16 num_ctxs; + /** @num_dbs: number of GuC doorbells. */ + u16 num_dbs; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c new file mode 100644 index 000000000..058585f06 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c @@ -0,0 +1,2203 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include + +#include "abi/guc_actions_sriov_abi.h" + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_sriov_pf.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_gt_sriov_pf_monitor.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc_ct.h" +#include "xe_sriov.h" +#include "xe_sriov_packet.h" +#include "xe_sriov_packet_types.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_migration.h" +#include "xe_sriov_pf_service.h" +#include "xe_tile.h" + +static const char *control_cmd_to_string(u32 cmd) +{ + switch (cmd) { + case GUC_PF_TRIGGER_VF_PAUSE: + return "PAUSE"; + case GUC_PF_TRIGGER_VF_RESUME: + return "RESUME"; + case GUC_PF_TRIGGER_VF_STOP: + return "STOP"; + case GUC_PF_TRIGGER_VF_FLR_START: + return "FLR_START"; + case GUC_PF_TRIGGER_VF_FLR_FINISH: + return "FLR_FINISH"; + default: + return ""; + } +} + +static int guc_action_vf_control_cmd(struct xe_guc *guc, u32 vfid, u32 cmd) +{ + u32 request[PF2GUC_VF_CONTROL_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_PF2GUC_VF_CONTROL), + FIELD_PREP(PF2GUC_VF_CONTROL_REQUEST_MSG_1_VFID, vfid), + FIELD_PREP(PF2GUC_VF_CONTROL_REQUEST_MSG_2_COMMAND, cmd), + }; + int ret; + + ret = xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request)); + return ret > 0 ? -EPROTO : ret; +} + +static int pf_send_vf_control_cmd(struct xe_gt *gt, unsigned int vfid, u32 cmd) +{ + int err; + + xe_gt_assert(gt, vfid != PFID); + xe_gt_sriov_dbg_verbose(gt, "sending VF%u control command %s\n", + vfid, control_cmd_to_string(cmd)); + + err = guc_action_vf_control_cmd(>->uc.guc, vfid, cmd); + if (unlikely(err)) + xe_gt_sriov_err(gt, "VF%u control command %s failed (%pe)\n", + vfid, control_cmd_to_string(cmd), ERR_PTR(err)); + return err; +} + +static int pf_send_vf_pause(struct xe_gt *gt, unsigned int vfid) +{ + return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_PAUSE); +} + +static int pf_send_vf_resume(struct xe_gt *gt, unsigned int vfid) +{ + return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_RESUME); +} + +static int pf_send_vf_stop(struct xe_gt *gt, unsigned int vfid) +{ + return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_STOP); +} + +static int pf_send_vf_flr_start(struct xe_gt *gt, unsigned int vfid) +{ + return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_FLR_START); +} + +static int pf_send_vf_flr_finish(struct xe_gt *gt, unsigned int vfid) +{ + return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_FLR_FINISH); +} + +/** + * DOC: The VF state machine + * + * The simplified VF state machine could be presented as:: + * + * pause--------------------------o + * / | + * / v + * (READY)<------------------resume-----(PAUSED) + * ^ \ / / + * | \ / / + * | stop---->(STOPPED)<----stop / + * | / / + * | / / + * o--------<-----flr / + * \ / + * o------<--------------------flr + * + * Where: + * + * * READY - represents a state in which VF is fully operable + * * PAUSED - represents a state in which VF activity is temporarily suspended + * * STOPPED - represents a state in which VF activity is definitely halted + * * pause - represents a request to temporarily suspend VF activity + * * resume - represents a request to resume VF activity + * * stop - represents a request to definitely halt VF activity + * * flr - represents a request to perform VF FLR to restore VF activity + * + * However, each state transition requires additional steps that involves + * communication with GuC that might fail or be interrupted by other requests:: + * + * .................................WIP.... + * : : + * pause--------------------->PAUSE_WIP----------------------------o + * / : / \ : | + * / : o----<---stop flr--o : | + * / : | \ / | : V + * (READY,RESUMED)<--------+------------RESUME_WIP<----+--<-----resume--(PAUSED) + * ^ \ \ : | | : / / + * | \ \ : | | : / / + * | \ \ : | | : / / + * | \ \ : o----<----------------------+--<-------stop / + * | \ \ : | | : / + * | \ \ : V | : / + * | \ stop----->STOP_WIP---------flr--->-----o : / + * | \ : | | : / + * | \ : | V : / + * | flr--------+----->----------------->FLR_WIP<-----flr + * | : | / ^ : + * | : | / | : + * o--------<-------:----+-----<----------------o | : + * : | | : + * :....|...........................|.....: + * | | + * V | + * (STOPPED)--------------------flr + * + * For details about each internal WIP state machine see: + * + * * `The VF PAUSE state machine`_ + * * `The VF RESUME state machine`_ + * * `The VF STOP state machine`_ + * * `The VF FLR state machine`_ + */ + +#ifdef CONFIG_DRM_XE_DEBUG_SRIOV +static const char *control_bit_to_string(enum xe_gt_sriov_control_bits bit) +{ + switch (bit) { +#define CASE2STR(_X) \ + case XE_GT_SRIOV_STATE_##_X: return #_X + CASE2STR(WIP); + CASE2STR(FLR_WIP); + CASE2STR(FLR_PREPARE); + CASE2STR(FLR_SEND_START); + CASE2STR(FLR_WAIT_GUC); + CASE2STR(FLR_GUC_DONE); + CASE2STR(FLR_SYNC); + CASE2STR(FLR_RESET_CONFIG); + CASE2STR(FLR_RESET_DATA); + CASE2STR(FLR_RESET_MMIO); + CASE2STR(FLR_SEND_FINISH); + CASE2STR(FLR_FAILED); + CASE2STR(PAUSE_WIP); + CASE2STR(PAUSE_SEND_PAUSE); + CASE2STR(PAUSE_WAIT_GUC); + CASE2STR(PAUSE_GUC_DONE); + CASE2STR(PAUSE_FAILED); + CASE2STR(PAUSED); + CASE2STR(SAVE_WIP); + CASE2STR(SAVE_PROCESS_DATA); + CASE2STR(SAVE_WAIT_DATA); + CASE2STR(SAVE_DATA_DONE); + CASE2STR(SAVE_FAILED); + CASE2STR(SAVED); + CASE2STR(RESTORE_WIP); + CASE2STR(RESTORE_PROCESS_DATA); + CASE2STR(RESTORE_WAIT_DATA); + CASE2STR(RESTORE_DATA_DONE); + CASE2STR(RESTORE_FAILED); + CASE2STR(RESTORED); + CASE2STR(RESUME_WIP); + CASE2STR(RESUME_SEND_RESUME); + CASE2STR(RESUME_FAILED); + CASE2STR(RESUMED); + CASE2STR(STOP_WIP); + CASE2STR(STOP_SEND_STOP); + CASE2STR(STOP_FAILED); + CASE2STR(STOPPED); + CASE2STR(MISMATCH); +#undef CASE2STR + default: return "?"; + } +} +#endif + +static unsigned long pf_get_default_timeout(enum xe_gt_sriov_control_bits bit) +{ + switch (bit) { + case XE_GT_SRIOV_STATE_FLR_WAIT_GUC: + case XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC: + return HZ / 2; + case XE_GT_SRIOV_STATE_FLR_WIP: + case XE_GT_SRIOV_STATE_FLR_RESET_CONFIG: + return 5 * HZ; + case XE_GT_SRIOV_STATE_RESTORE_WIP: + return 20 * HZ; + default: + return HZ; + } +} + +static struct xe_gt_sriov_control_state *pf_pick_vf_control(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid <= xe_gt_sriov_pf_get_totalvfs(gt)); + + return >->sriov.pf.vfs[vfid].control; +} + +static unsigned long *pf_peek_vf_state(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + return cs->state; +} + +static bool pf_check_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + return test_bit(bit, pf_peek_vf_state(gt, vfid)); +} + +static void pf_dump_vf_state(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long state = *pf_peek_vf_state(gt, vfid); + enum xe_gt_sriov_control_bits bit; + + if (state) { + xe_gt_sriov_dbg_verbose(gt, "VF%u state %#lx%s%*pbl\n", + vfid, state, state ? " bits " : "", + (int)BITS_PER_LONG, &state); + for_each_set_bit(bit, &state, BITS_PER_LONG) + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d)\n", + vfid, control_bit_to_string(bit), bit); + } else { + xe_gt_sriov_dbg_verbose(gt, "VF%u state READY\n", vfid); + } +} + +static bool pf_expect_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + bool result = pf_check_vf_state(gt, vfid, bit); + + if (unlikely(!result)) + pf_dump_vf_state(gt, vfid); + + return result; +} + +static bool pf_expect_vf_not_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + bool result = !pf_check_vf_state(gt, vfid, bit); + + if (unlikely(!result)) + pf_dump_vf_state(gt, vfid); + + return result; +} + +static void pf_track_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit, + const char *what) +{ + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d) %s\n", + vfid, control_bit_to_string(bit), bit, what); +} + +static bool pf_enter_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + if (!test_and_set_bit(bit, pf_peek_vf_state(gt, vfid))) { + pf_track_vf_state(gt, vfid, bit, "enter"); + return true; + } + return false; +} + +static bool pf_exit_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + if (test_and_clear_bit(bit, pf_peek_vf_state(gt, vfid))) { + pf_track_vf_state(gt, vfid, bit, "exit"); + return true; + } + return false; +} + +static void pf_escape_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + if (pf_exit_vf_state(gt, vfid, bit)) + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d) escaped by %ps\n", + vfid, control_bit_to_string(bit), bit, + __builtin_return_address(0)); +} + +static void pf_enter_vf_mismatch(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_MISMATCH)) { + xe_gt_sriov_dbg(gt, "VF%u state mismatch detected by %ps\n", + vfid, __builtin_return_address(0)); + pf_dump_vf_state(gt, vfid); + } +} + +static void pf_exit_vf_mismatch(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_MISMATCH)) + xe_gt_sriov_dbg(gt, "VF%u state mismatch cleared by %ps\n", + vfid, __builtin_return_address(0)); + + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED); +} + +#define pf_enter_vf_state_machine_bug(gt, vfid) ({ \ + pf_enter_vf_mismatch((gt), (vfid)); \ +}) + +static void pf_queue_control_worker(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + queue_work(xe->sriov.wq, >->sriov.pf.control.worker); +} + +static void pf_queue_vf(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_pf_control *pfc = >->sriov.pf.control; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + spin_lock(&pfc->lock); + list_move_tail(>->sriov.pf.vfs[vfid].control.link, &pfc->list); + spin_unlock(&pfc->lock); + + pf_queue_control_worker(gt); +} + +static void pf_exit_vf_flr_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_stop_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_save_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_restore_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_pause_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_resume_wip(struct xe_gt *gt, unsigned int vfid); + +static bool pf_enter_vf_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_WIP)) { + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + reinit_completion(&cs->done); + return true; + } + return false; +} + +static void pf_exit_vf_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_WIP)) { + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + pf_exit_vf_flr_wip(gt, vfid); + pf_exit_vf_stop_wip(gt, vfid); + pf_exit_vf_save_wip(gt, vfid); + pf_exit_vf_restore_wip(gt, vfid); + pf_exit_vf_pause_wip(gt, vfid); + pf_exit_vf_resume_wip(gt, vfid); + + complete_all(&cs->done); + } +} + +static int pf_wait_vf_wip_done(struct xe_gt *gt, unsigned int vfid, unsigned long timeout) +{ + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + return wait_for_completion_timeout(&cs->done, timeout) ? 0 : -ETIMEDOUT; +} + +static void pf_enter_vf_ready(struct xe_gt *gt, unsigned int vfid) +{ + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +/** + * DOC: The VF PAUSE state machine + * + * The VF PAUSE state machine looks like:: + * + * (READY,RESUMED)<-------------<---------------------o---------o + * | \ \ + * pause \ \ + * | \ \ + * ....V...........................PAUSE_WIP........ \ \ + * : \ : o \ + * : \ o------<-----busy : | \ + * : \ / / : | | + * : PAUSE_SEND_PAUSE ---failed--->----------o--->(PAUSE_FAILED) | + * : | \ : | | + * : acked rejected---->----------o--->(MISMATCH) / + * : | : / + * : v : / + * : PAUSE_WAIT_GUC : / + * : | : / + * : done : / + * : | : / + * : v : / + * : PAUSE_GUC_DONE o-----restart + * : | : + * : | o---<--busy : + * : / : + * : / : + * : / : + * :....o..............o...............o...........: + * | | | + * completed flr stop + * | | | + * V .....V..... ......V..... + * (PAUSED) : FLR_WIP : : STOP_WIP : + * :.........: :..........: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_pause_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WIP)) { + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_GUC_DONE); + } +} + +static void pf_enter_vf_paused(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_pause_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_paused(gt, vfid); +} + +static void pf_enter_vf_pause_failed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_FAILED); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_pause_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_pause_failed(gt, vfid); +} + +static bool pf_exit_vf_pause_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_GUC_DONE)) + return false; + + pf_enter_vf_pause_completed(gt, vfid); + return true; +} + +static void pf_enter_vf_pause_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_GUC_DONE)) + pf_queue_vf(gt, vfid); +} + +static void pf_enter_pause_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC)) + pf_enter_vf_state_machine_bug(gt, vfid); +} + +static bool pf_exit_pause_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + return pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC); +} + +static void pf_enter_vf_pause_send_pause(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_pause_send_pause(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE)) + return false; + + /* GuC may actually send a PAUSE_DONE before we get a RESPONSE */ + pf_enter_pause_wait_guc(gt, vfid); + + err = pf_send_vf_pause(gt, vfid); + if (err) { + /* send failed, so we shouldn't expect PAUSE_DONE from GuC */ + pf_exit_pause_wait_guc(gt, vfid); + + if (err == -EBUSY) + pf_enter_vf_pause_send_pause(gt, vfid); + else if (err == -EIO) + pf_enter_vf_pause_rejected(gt, vfid); + else + pf_enter_vf_pause_failed(gt, vfid); + } else { + /* + * we have already moved to WAIT_GUC, maybe even to GUC_DONE + * but since GuC didn't complain, we may clear MISMATCH + */ + pf_exit_vf_mismatch(gt, vfid); + } + + return true; +} + +static bool pf_enter_vf_pause_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_pause_send_pause(gt, vfid); + return true; + } + + return false; +} + +/** + * xe_gt_sriov_pf_control_pause_vf - Pause a VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_PAUSE_WIP); + int err; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u is stopped!\n", vfid); + return -EPERM; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u was already paused!\n", vfid); + return -ESTALE; + } + + if (!pf_enter_vf_pause_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u pause already in progress!\n", vfid); + return -EALREADY; + } + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) { + xe_gt_sriov_dbg(gt, "VF%u pause didn't finish in %u ms (%pe)\n", + vfid, jiffies_to_msecs(timeout), ERR_PTR(err)); + return err; + } + + if (pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u paused!\n", vfid); + return 0; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_FAILED)) { + xe_gt_sriov_dbg(gt, "VF%u pause failed!\n", vfid); + return -EIO; + } + + xe_gt_sriov_dbg(gt, "VF%u pause was canceled!\n", vfid); + return -ECANCELED; +} + +/** + * DOC: The VF RESUME state machine + * + * The VF RESUME state machine looks like:: + * + * (PAUSED)<-----------------<------------------------o + * | \ + * resume \ + * | \ + * ....V............................RESUME_WIP...... \ + * : \ : o + * : \ o-------<-----busy : | + * : \ / / : | + * : RESUME_SEND_RESUME ---failed--->--------o--->(RESUME_FAILED) + * : / \ : | + * : acked rejected---->---------o--->(MISMATCH) + * : / : + * :....o..............o...............o.....o.....: + * | | | \ + * completed flr stop restart-->(READY) + * | | | + * V .....V..... ......V..... + * (RESUMED) : FLR_WIP : : STOP_WIP : + * :.........: :..........: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_resume_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_WIP)) + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_SEND_RESUME); +} + +static void pf_enter_vf_resumed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_resume_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_resumed(gt, vfid); +} + +static void pf_enter_vf_resume_failed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_FAILED); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_resume_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_resume_failed(gt, vfid); +} + +static void pf_enter_vf_resume_send_resume(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_SEND_RESUME)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_resume_send_resume(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_SEND_RESUME)) + return false; + + err = pf_send_vf_resume(gt, vfid); + if (err == -EBUSY) + pf_enter_vf_resume_send_resume(gt, vfid); + else if (err == -EIO) + pf_enter_vf_resume_rejected(gt, vfid); + else if (err) + pf_enter_vf_resume_failed(gt, vfid); + else + pf_enter_vf_resume_completed(gt, vfid); + return true; +} + +static bool pf_enter_vf_resume_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_resume_send_resume(gt, vfid); + return true; + } + + return false; +} + +/** + * xe_gt_sriov_pf_control_resume_vf - Resume a VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_RESUME_WIP); + int err; + + if (!pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u is not paused!\n", vfid); + return -EPERM; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WIP)) { + xe_gt_sriov_dbg(gt, "VF%u save is in progress!\n", vfid); + return -EBUSY; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WIP)) { + xe_gt_sriov_dbg(gt, "VF%u restore is in progress!\n", vfid); + return -EBUSY; + } + + if (!pf_enter_vf_resume_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u resume already in progress!\n", vfid); + return -EALREADY; + } + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) + return err; + + if (pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED)) { + xe_gt_sriov_dbg(gt, "VF%u resumed!\n", vfid); + return 0; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_FAILED)) { + xe_gt_sriov_dbg(gt, "VF%u resume failed!\n", vfid); + return -EIO; + } + + xe_gt_sriov_dbg(gt, "VF%u resume was canceled!\n", vfid); + return -ECANCELED; +} + +/** + * DOC: The VF SAVE state machine + * + * SAVE extends the PAUSED state. + * + * The VF SAVE state machine looks like:: + * + * ....PAUSED.................................................... + * : : + * : (O)<---------o : + * : | \ : + * : save (SAVED) (SAVE_FAILED) : + * : | ^ ^ : + * : | | | : + * : ....V...............o...........o......SAVE_WIP......... : + * : : | | | : : + * : : | empty | : : + * : : | | | : : + * : : | | | : : + * : : | DATA_DONE | : : + * : : | ^ | : : + * : : | | error : : + * : : | no_data / : : + * : : | / / : : + * : : | / / : : + * : : | / / : : + * : : o---------->PROCESS_DATA<----consume : : + * : : \ \ : : + * : : \ \ : : + * : : \ \ : : + * : : ring_full----->WAIT_DATA : : + * : : : : + * : :......................................................: : + * :............................................................: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_save_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WIP)) { + xe_gt_sriov_pf_migration_ring_free(gt, vfid); + + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WAIT_DATA); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_DATA_DONE); + } +} + +static void pf_enter_vf_saved(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + xe_gt_sriov_dbg(gt, "VF%u saved!\n", vfid); + + pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_save_failed(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_FAILED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + wake_up_all(xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid)); + + pf_exit_vf_wip(gt, vfid); +} + +static int pf_handle_vf_save_data(struct xe_gt *gt, unsigned int vfid) +{ + int ret; + + if (xe_gt_sriov_pf_migration_save_data_pending(gt, vfid, + XE_SRIOV_PACKET_TYPE_GUC)) { + ret = xe_gt_sriov_pf_migration_guc_save(gt, vfid); + if (ret) + return ret; + + xe_gt_sriov_pf_migration_save_data_complete(gt, vfid, + XE_SRIOV_PACKET_TYPE_GUC); + + return -EAGAIN; + } + + if (xe_gt_sriov_pf_migration_save_data_pending(gt, vfid, + XE_SRIOV_PACKET_TYPE_GGTT)) { + ret = xe_gt_sriov_pf_migration_ggtt_save(gt, vfid); + if (ret) + return ret; + + xe_gt_sriov_pf_migration_save_data_complete(gt, vfid, + XE_SRIOV_PACKET_TYPE_GGTT); + + return -EAGAIN; + } + + if (xe_gt_sriov_pf_migration_save_data_pending(gt, vfid, + XE_SRIOV_PACKET_TYPE_MMIO)) { + ret = xe_gt_sriov_pf_migration_mmio_save(gt, vfid); + if (ret) + return ret; + + xe_gt_sriov_pf_migration_save_data_complete(gt, vfid, + XE_SRIOV_PACKET_TYPE_MMIO); + + return -EAGAIN; + } + + if (xe_gt_sriov_pf_migration_save_data_pending(gt, vfid, + XE_SRIOV_PACKET_TYPE_VRAM)) { + ret = xe_gt_sriov_pf_migration_vram_save(gt, vfid); + if (ret == -EAGAIN) + return -EAGAIN; + else if (ret) + return ret; + + xe_gt_sriov_pf_migration_save_data_complete(gt, vfid, + XE_SRIOV_PACKET_TYPE_VRAM); + + return -EAGAIN; + } + + return 0; +} + +static bool pf_handle_vf_save(struct xe_gt *gt, unsigned int vfid) +{ + int ret; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA)) + return false; + + if (xe_gt_sriov_pf_migration_ring_full(gt, vfid)) { + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WAIT_DATA); + return true; + } + + ret = pf_handle_vf_save_data(gt, vfid); + if (ret == -EAGAIN) + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA); + else if (ret) + pf_enter_vf_save_failed(gt, vfid); + else + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_DATA_DONE); + + return true; +} + +static void pf_exit_vf_save_wait_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WAIT_DATA)) + return; + + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA); + pf_queue_vf(gt, vfid); +} + +static bool pf_enter_vf_save_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WIP)) { + xe_gt_sriov_pf_migration_save_init(gt, vfid); + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA); + pf_queue_vf(gt, vfid); + return true; + } + + return false; +} + +/** + * xe_gt_sriov_pf_control_check_save_data_done() - Check if all save migration data was produced. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: true if all migration data was produced, false otherwise. + */ +bool xe_gt_sriov_pf_control_check_save_data_done(struct xe_gt *gt, unsigned int vfid) +{ + return pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_DATA_DONE); +} + +/** + * xe_gt_sriov_pf_control_check_save_failed() - Check if save processing has failed. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: true if save processing failed, false otherwise. + */ +bool xe_gt_sriov_pf_control_check_save_failed(struct xe_gt *gt, unsigned int vfid) +{ + return pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_FAILED); +} + +/** + * xe_gt_sriov_pf_control_process_save_data() - Queue VF save migration data processing. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_process_save_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_FAILED)) + return -EIO; + + pf_exit_vf_save_wait_data(gt, vfid); + + return 0; +} + +/** + * xe_gt_sriov_pf_control_trigger_save_vf() - Start an SR-IOV VF migration data save sequence. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_trigger_save_vf(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u is stopped!\n", vfid); + return -EPERM; + } + + if (!pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u is not paused!\n", vfid); + return -EPERM; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WIP)) { + xe_gt_sriov_dbg(gt, "VF%u restore is in progress!\n", vfid); + return -EBUSY; + } + + if (!pf_enter_vf_save_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u save already in progress!\n", vfid); + return -EALREADY; + } + + return 0; +} + +/** + * xe_gt_sriov_pf_control_finish_save_vf() - Complete a VF migration data save sequence. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_finish_save_vf(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_DATA_DONE)) { + xe_gt_sriov_err(gt, "VF%u save is still in progress!\n", vfid); + return -EIO; + } + + pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_DATA_DONE); + pf_enter_vf_saved(gt, vfid); + + return 0; +} + +/** + * DOC: The VF RESTORE state machine + * + * RESTORE extends the PAUSED state. + * + * The VF RESTORE state machine looks like:: + * + * ....PAUSED.................................................... + * : : + * : (O)<---------o : + * : | \ : + * : restore (RESTORED) (RESTORE_FAILED) : + * : | ^ ^ : + * : | | | : + * : ....V...............o...........o......RESTORE_WIP...... : + * : : | | | : : + * : : | empty | : : + * : : | | | : : + * : : | | | : : + * : : | DATA_DONE | : : + * : : | ^ | : : + * : : | | error : : + * : : | trailer / : : + * : : | / / : : + * : : | / / : : + * : : | / / : : + * : : o---------->PROCESS_DATA<----produce : : + * : : \ \ : : + * : : \ \ : : + * : : \ \ : : + * : : ring_empty---->WAIT_DATA : : + * : : : : + * : :......................................................: : + * :............................................................: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_restore_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WIP)) { + xe_gt_sriov_pf_migration_ring_free(gt, vfid); + + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_DATA_DONE); + } +} + +static void pf_enter_vf_restored(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + xe_gt_sriov_dbg(gt, "VF%u restored!\n", vfid); + + pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_restore_failed(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + wake_up_all(xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid)); + + pf_exit_vf_wip(gt, vfid); +} + +static int pf_handle_vf_restore_data(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_sriov_packet *data = xe_gt_sriov_pf_migration_restore_consume(gt, vfid); + int ret = 0; + + switch (data->hdr.type) { + case XE_SRIOV_PACKET_TYPE_GGTT: + ret = xe_gt_sriov_pf_migration_ggtt_restore(gt, vfid, data); + break; + case XE_SRIOV_PACKET_TYPE_MMIO: + ret = xe_gt_sriov_pf_migration_mmio_restore(gt, vfid, data); + break; + case XE_SRIOV_PACKET_TYPE_GUC: + ret = xe_gt_sriov_pf_migration_guc_restore(gt, vfid, data); + break; + case XE_SRIOV_PACKET_TYPE_VRAM: + ret = xe_gt_sriov_pf_migration_vram_restore(gt, vfid, data); + break; + default: + xe_gt_sriov_notice(gt, "Skipping VF%u unknown data type: %d\n", + vfid, data->hdr.type); + break; + } + + xe_sriov_packet_free(data); + + return ret; +} + +static bool pf_handle_vf_restore(struct xe_gt *gt, unsigned int vfid) +{ + int ret; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA)) + return false; + + if (xe_gt_sriov_pf_migration_ring_empty(gt, vfid)) { + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_DATA_DONE)) + pf_enter_vf_restored(gt, vfid); + else + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA); + + return true; + } + + ret = pf_handle_vf_restore_data(gt, vfid); + if (ret) + pf_enter_vf_restore_failed(gt, vfid); + else + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA); + + return true; +} + +static void pf_exit_vf_restore_wait_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA)) + return; + + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA); + pf_queue_vf(gt, vfid); +} + +static bool pf_enter_vf_restore_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA); + pf_queue_vf(gt, vfid); + return true; + } + + return false; +} + +/** + * xe_gt_sriov_pf_control_check_restore_failed() - Check if restore processing has failed. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: true if restore processing failed, false otherwise. + */ +bool xe_gt_sriov_pf_control_check_restore_failed(struct xe_gt *gt, unsigned int vfid) +{ + return pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED); +} + +/** + * xe_gt_sriov_pf_control_restore_data_done() - Indicate the end of VF migration data stream. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_restore_data_done(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_DATA_DONE)) { + pf_enter_vf_state_machine_bug(gt, vfid); + return -EIO; + } + + return xe_gt_sriov_pf_control_process_restore_data(gt, vfid); +} + +/** + * xe_gt_sriov_pf_control_process_restore_data() - Queue VF restore migration data processing. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_process_restore_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED)) { + xe_gt_sriov_pf_migration_ring_free(gt, vfid); + return -EIO; + } + + pf_exit_vf_restore_wait_data(gt, vfid); + + return 0; +} + +/** + * xe_gt_sriov_pf_control_trigger_restore_vf() - Start an SR-IOV VF migration data restore sequence. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_trigger_restore_vf(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u is stopped!\n", vfid); + return -EPERM; + } + + if (!pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u is not paused!\n", vfid); + return -EPERM; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WIP)) { + xe_gt_sriov_dbg(gt, "VF%u save is in progress!\n", vfid); + return -EBUSY; + } + + if (!pf_enter_vf_restore_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u restore already in progress!\n", vfid); + return -EALREADY; + } + + return 0; +} + +static int pf_wait_vf_restore_done(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_RESTORE_WIP); + int err; + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) { + xe_gt_sriov_notice(gt, "VF%u RESTORE didn't finish in %u ms (%pe)\n", + vfid, jiffies_to_msecs(timeout), ERR_PTR(err)); + return err; + } + + if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED)) + return -EIO; + + return 0; +} + +/** + * xe_gt_sriov_pf_control_finish_restore_vf() - Complete a VF migration data restore sequence. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_finish_restore_vf(struct xe_gt *gt, unsigned int vfid) +{ + int ret; + + ret = pf_wait_vf_restore_done(gt, vfid); + if (ret) + return ret; + + if (!pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORED)) { + pf_enter_vf_mismatch(gt, vfid); + return -EIO; + } + + pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + + return 0; +} + +/** + * DOC: The VF STOP state machine + * + * The VF STOP state machine looks like:: + * + * (READY,PAUSED,RESUMED)<-------<--------------------o + * | \ + * stop \ + * | \ + * ....V..............................STOP_WIP...... \ + * : \ : o + * : \ o----<----busy : | + * : \ / / : | + * : STOP_SEND_STOP--------failed--->--------o--->(STOP_FAILED) + * : / \ : | + * : acked rejected-------->--------o--->(MISMATCH) + * : / : + * :....o..............o...............o...........: + * | | | + * completed flr restart + * | | | + * V .....V..... V + * (STOPPED) : FLR_WIP : (READY) + * :.........: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_stop_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_WIP)) + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_SEND_STOP); +} + +static void pf_enter_vf_stopped(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_stop_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_stopped(gt, vfid); +} + +static void pf_enter_vf_stop_failed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_FAILED); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_stop_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_stop_failed(gt, vfid); +} + +static void pf_enter_vf_stop_send_stop(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_SEND_STOP)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_stop_send_stop(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_SEND_STOP)) + return false; + + err = pf_send_vf_stop(gt, vfid); + if (err == -EBUSY) + pf_enter_vf_stop_send_stop(gt, vfid); + else if (err == -EIO) + pf_enter_vf_stop_rejected(gt, vfid); + else if (err) + pf_enter_vf_stop_failed(gt, vfid); + else + pf_enter_vf_stop_completed(gt, vfid); + return true; +} + +static bool pf_enter_vf_stop_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_stop_send_stop(gt, vfid); + return true; + } + return false; +} + +/** + * xe_gt_sriov_pf_control_stop_vf - Stop a VF. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_STOP_WIP); + int err; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u was already stopped!\n", vfid); + return -ESTALE; + } + + if (!pf_enter_vf_stop_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u stop already in progress!\n", vfid); + return -EALREADY; + } + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) + return err; + + if (pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u stopped!\n", vfid); + return 0; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_FAILED)) { + xe_gt_sriov_dbg(gt, "VF%u stop failed!\n", vfid); + return -EIO; + } + + xe_gt_sriov_dbg(gt, "VF%u stop was canceled!\n", vfid); + return -ECANCELED; +} + +/** + * DOC: The VF FLR state machine + * + * The VF FLR state machine looks like:: + * + * (READY,PAUSED,STOPPED)<------------<--------------o + * | | \ + * flr prepare \ + * | | \ + * ....V.............V............FLR_WIP........... \ + * : | | : \ + * : | FLR_PREPARE : | + * : | / : | + * : \ flr : | + * : \ / : | + * : \ o----<----busy : | + * : \ / / : | + * : FLR_SEND_START---failed----->-----------o--->(FLR_FAILED)<---o + * : | \ : | | + * : acked rejected----->-----------o--->(MISMATCH) | + * : | : ^ | + * : v : | | + * : FLR_WAIT_GUC : | | + * : | : | | + * : done : | | + * : | : | | + * : v : | | + * : FLR_GUC_DONE : | | + * : | : | | + * : | o--<--sync : | | + * : |/ / : | | + * : FLR_SYNC--o : | | + * : | : | | + * : FLR_RESET_CONFIG---failed--->-----------o--------+-----------o + * : | : | | + * : FLR_RESET_DATA : | | + * : | : | | + * : FLR_RESET_MMIO : | | + * : | : | | + * : | o----<----busy : | | + * : |/ / : | | + * : FLR_SEND_FINISH----failed--->-----------o--------+-----------o + * : / \ : | + * : acked rejected----->-----------o--------o + * : / : + * :....o..............................o...........: + * | | + * completed restart + * | / + * V / + * (READY)<----------<------------o + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_enter_vf_flr_send_start(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_START)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_prepare(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_PREPARE)) + return false; + + pf_enter_vf_flr_send_start(gt, vfid); + return true; +} + +static bool pf_enter_vf_flr_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) + return false; + + pf_enter_vf_wip(gt, vfid); + return true; +} + +static void pf_exit_vf_flr_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) { + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_PREPARE); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_CONFIG); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_GUC_DONE); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_START); + + xe_sriov_pf_control_sync_flr(gt_to_xe(gt), vfid); + } +} + +static void pf_enter_vf_flr_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_ready(gt, vfid); +} + +static void pf_enter_vf_flr_failed(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED)) + xe_gt_sriov_notice(gt, "VF%u FLR failed!\n", vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_flr_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_flr_failed(gt, vfid); +} + +static void pf_enter_vf_flr_send_finish(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_send_finish(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH)) + return false; + + err = pf_send_vf_flr_finish(gt, vfid); + if (err == -EBUSY) + pf_enter_vf_flr_send_finish(gt, vfid); + else if (err == -EIO) + pf_enter_vf_flr_rejected(gt, vfid); + else if (err) + pf_enter_vf_flr_failed(gt, vfid); + else + pf_enter_vf_flr_completed(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_reset_mmio(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_reset_mmio(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO)) + return false; + + xe_gt_sriov_pf_sanitize_hw(gt, vfid); + + pf_enter_vf_flr_send_finish(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_reset_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_reset_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA)) + return false; + + if (xe_tile_is_root(gt->tile) && xe_gt_is_main_type(gt)) + xe_sriov_pf_service_reset_vf(gt_to_xe(gt), vfid); + + xe_gt_sriov_pf_monitor_flr(gt, vfid); + + pf_enter_vf_flr_reset_mmio(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_reset_config(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_CONFIG)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_reset_config(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_FLR_RESET_CONFIG); + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_CONFIG)) + return false; + + err = xe_gt_sriov_pf_config_sanitize(gt, vfid, timeout); + if (err) + pf_enter_vf_flr_failed(gt, vfid); + else + pf_enter_vf_flr_reset_data(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC)) + pf_enter_vf_state_machine_bug(gt, vfid); +} + +static bool pf_exit_vf_flr_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + return pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC); +} + +static bool pf_exit_vf_flr_send_start(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_START)) + return false; + + /* GuC may actually send a FLR_DONE before we get a RESPONSE */ + pf_enter_vf_flr_wait_guc(gt, vfid); + + err = pf_send_vf_flr_start(gt, vfid); + if (err) { + /* send failed, so we shouldn't expect FLR_DONE from GuC */ + pf_exit_vf_flr_wait_guc(gt, vfid); + + if (err == -EBUSY) + pf_enter_vf_flr_send_start(gt, vfid); + else if (err == -EIO) + pf_enter_vf_flr_rejected(gt, vfid); + else + pf_enter_vf_flr_failed(gt, vfid); + } else { + /* + * we have already moved to WAIT_GUC, maybe even to GUC_DONE + * but since GuC didn't complain, we may clear MISMATCH + */ + pf_exit_vf_mismatch(gt, vfid); + } + + return true; +} + +static bool pf_exit_vf_flr_sync(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SYNC)) + return false; + + pf_enter_vf_flr_reset_config(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_sync(struct xe_gt *gt, unsigned int vfid) +{ + int ret; + + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SYNC)) + pf_enter_vf_state_machine_bug(gt, vfid); + + ret = xe_sriov_pf_control_sync_flr(gt_to_xe(gt), vfid); + if (ret < 0) { + xe_gt_sriov_dbg_verbose(gt, "FLR checkpoint %pe\n", ERR_PTR(ret)); + pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SYNC); + } else { + xe_gt_sriov_dbg_verbose(gt, "FLR checkpoint pass\n"); + pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SYNC); + } +} + +static bool pf_exit_vf_flr_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_GUC_DONE)) + return false; + + pf_enter_vf_flr_sync(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_GUC_DONE)) + pf_queue_vf(gt, vfid); +} + +/** + * xe_gt_sriov_pf_control_prepare_flr() - Notify PF that VF FLR request was issued. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This is an optional early notification path used to mark pending FLR before + * the GuC notifies the PF with a FLR event. + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_prepare_flr(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_flr_wip(gt, vfid)) + return -EALREADY; + + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_PREPARE); + return 0; +} + +static int pf_begin_vf_flr(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_flr_wip(gt, vfid)) { + pf_enter_vf_flr_send_start(gt, vfid); + return 0; + } + + if (pf_exit_vf_flr_prepare(gt, vfid)) + return 0; + + xe_gt_sriov_dbg(gt, "VF%u FLR is already in progress\n", vfid); + return -EALREADY; +} + +/** + * xe_gt_sriov_pf_control_trigger_flr - Start a VF FLR sequence. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid) +{ + return pf_begin_vf_flr(gt, vfid); +} + +/** + * xe_gt_sriov_pf_control_sync_flr() - Synchronize on the VF FLR checkpoint. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @sync: if true it will allow to exit the checkpoint + * + * Return: non-zero if FLR checkpoint has been reached, zero if the is no FLR + * in progress, or a negative error code on the FLR busy or failed. + */ +int xe_gt_sriov_pf_control_sync_flr(struct xe_gt *gt, unsigned int vfid, bool sync) +{ + if (sync && pf_exit_vf_flr_sync(gt, vfid)) + return 1; + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SYNC)) + return 1; + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) + return -EBUSY; + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED)) + return -EIO; + return 0; +} + +/** + * xe_gt_sriov_pf_control_wait_flr() - Wait for a VF FLR to complete. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_wait_flr(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_FLR_WIP); + int err; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED)) + return -EIO; + + if (!pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) + return 0; + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) { + xe_gt_sriov_notice(gt, "VF%u FLR didn't finish in %u ms (%pe)\n", + vfid, jiffies_to_msecs(timeout), ERR_PTR(err)); + return err; + } + + if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED)) + return -EIO; + + return 0; +} + +/** + * DOC: The VF FLR Flow with GuC + * + * The VF FLR flow includes several steps:: + * + * PF GUC PCI + * ======================================================== + * | | | + * (1) | [ ] <----- FLR --| + * | [ ] : + * (2) [ ] <-------- NOTIFY FLR --[ ] + * [ ] | + * (3) [ ] | + * [ ] | + * [ ]-- START FLR ---------> [ ] + * | [ ] + * (4) | [ ] + * | [ ] + * [ ] <--------- FLR DONE -- [ ] + * [ ] | + * (5) [ ] | + * [ ] | + * [ ]-- FINISH FLR --------> [ ] + * | | + * + * * Step 1: PCI HW generates interrupt to the GuC about VF FLR + * * Step 2: GuC FW sends G2H notification to the PF about VF FLR + * * Step 2a: on some platforms G2H is only received from root GuC + * * Step 3: PF sends H2G request to the GuC to start VF FLR sequence + * * Step 3a: on some platforms PF must send H2G to all other GuCs + * * Step 4: GuC FW performs VF FLR cleanups and notifies the PF when done + * * Step 5: PF performs VF FLR cleanups and notifies the GuC FW when finished + */ + +static bool needs_dispatch_flr(struct xe_device *xe) +{ + return xe->info.platform == XE_PVC; +} + +static void pf_handle_vf_flr(struct xe_gt *gt, u32 vfid) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_gt *gtit; + unsigned int gtid; + + xe_gt_sriov_info(gt, "VF%u FLR\n", vfid); + + if (needs_dispatch_flr(xe)) { + for_each_gt(gtit, xe, gtid) + pf_begin_vf_flr(gtit, vfid); + } else { + pf_begin_vf_flr(gt, vfid); + } +} + +static void pf_handle_vf_flr_done(struct xe_gt *gt, u32 vfid) +{ + if (!pf_exit_vf_flr_wait_guc(gt, vfid)) { + xe_gt_sriov_dbg(gt, "Received out of order 'VF%u FLR done'\n", vfid); + pf_enter_vf_mismatch(gt, vfid); + return; + } + + pf_enter_vf_flr_guc_done(gt, vfid); +} + +static void pf_handle_vf_pause_done(struct xe_gt *gt, u32 vfid) +{ + if (!pf_exit_pause_wait_guc(gt, vfid)) { + xe_gt_sriov_dbg(gt, "Received out of order 'VF%u PAUSE done'\n", vfid); + pf_enter_vf_mismatch(gt, vfid); + return; + } + + pf_enter_vf_pause_guc_done(gt, vfid); +} + +static int pf_handle_vf_event(struct xe_gt *gt, u32 vfid, u32 eventid) +{ + xe_gt_sriov_dbg_verbose(gt, "received VF%u event %#x\n", vfid, eventid); + + if (vfid > xe_gt_sriov_pf_get_totalvfs(gt)) + return -EPROTO; + + switch (eventid) { + case GUC_PF_NOTIFY_VF_FLR: + pf_handle_vf_flr(gt, vfid); + break; + case GUC_PF_NOTIFY_VF_FLR_DONE: + pf_handle_vf_flr_done(gt, vfid); + break; + case GUC_PF_NOTIFY_VF_PAUSE_DONE: + pf_handle_vf_pause_done(gt, vfid); + break; + case GUC_PF_NOTIFY_VF_FIXUP_DONE: + break; + default: + return -ENOPKG; + } + return 0; +} + +static int pf_handle_pf_event(struct xe_gt *gt, u32 eventid) +{ + switch (eventid) { + case GUC_PF_NOTIFY_VF_ENABLE: + xe_gt_sriov_dbg_verbose(gt, "VFs %s/%s\n", + str_enabled_disabled(true), + str_enabled_disabled(false)); + break; + default: + return -ENOPKG; + } + return 0; +} + +/** + * xe_gt_sriov_pf_control_process_guc2pf - Handle VF state notification from GuC. + * @gt: the &xe_gt + * @msg: the G2H message + * @len: the length of the G2H message + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len) +{ + u32 vfid; + u32 eventid; + + xe_gt_assert(gt, len); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) == + GUC_ACTION_GUC2PF_VF_STATE_NOTIFY); + + if (unlikely(!xe_device_is_sriov_pf(gt_to_xe(gt)))) + return -EPROTO; + + if (unlikely(FIELD_GET(GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_0_MBZ, msg[0]))) + return -EPFNOSUPPORT; + + if (unlikely(len != GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_LEN)) + return -EPROTO; + + vfid = FIELD_GET(GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_1_VFID, msg[1]); + eventid = FIELD_GET(GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_2_EVENT, msg[2]); + + return vfid ? pf_handle_vf_event(gt, vfid, eventid) : pf_handle_pf_event(gt, eventid); +} + +static bool pf_process_vf_state_machine(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_flr_send_start(gt, vfid)) + return true; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC)) { + xe_gt_sriov_dbg_verbose(gt, "VF%u in %s\n", vfid, + control_bit_to_string(XE_GT_SRIOV_STATE_FLR_WAIT_GUC)); + return false; + } + + if (pf_exit_vf_flr_guc_done(gt, vfid)) + return true; + + if (pf_exit_vf_flr_reset_config(gt, vfid)) + return true; + + if (pf_exit_vf_flr_reset_data(gt, vfid)) + return true; + + if (pf_exit_vf_flr_reset_mmio(gt, vfid)) + return true; + + if (pf_exit_vf_flr_send_finish(gt, vfid)) + return true; + + if (pf_exit_vf_stop_send_stop(gt, vfid)) + return true; + + if (pf_exit_vf_pause_send_pause(gt, vfid)) + return true; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC)) { + xe_gt_sriov_dbg_verbose(gt, "VF%u in %s\n", vfid, + control_bit_to_string(XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC)); + return true; + } + + if (pf_exit_vf_pause_guc_done(gt, vfid)) + return true; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WAIT_DATA)) { + xe_gt_sriov_dbg_verbose(gt, "VF%u in %s\n", vfid, + control_bit_to_string(XE_GT_SRIOV_STATE_SAVE_WAIT_DATA)); + return false; + } + + if (pf_handle_vf_save(gt, vfid)) + return true; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA)) { + xe_gt_sriov_dbg_verbose(gt, "VF%u in %s\n", vfid, + control_bit_to_string(XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA)); + return false; + } + + if (pf_handle_vf_restore(gt, vfid)) + return true; + + if (pf_exit_vf_resume_send_resume(gt, vfid)) + return true; + + return false; +} + +static unsigned int pf_control_state_index(struct xe_gt *gt, + struct xe_gt_sriov_control_state *cs) +{ + return container_of(cs, struct xe_gt_sriov_metadata, control) - gt->sriov.pf.vfs; +} + +static void pf_worker_find_work(struct xe_gt *gt) +{ + struct xe_gt_sriov_pf_control *pfc = >->sriov.pf.control; + struct xe_gt_sriov_control_state *cs; + unsigned int vfid; + bool empty; + bool more; + + spin_lock(&pfc->lock); + cs = list_first_entry_or_null(&pfc->list, struct xe_gt_sriov_control_state, link); + if (cs) + list_del_init(&cs->link); + empty = list_empty(&pfc->list); + spin_unlock(&pfc->lock); + + if (!cs) + return; + + /* VF metadata structures are indexed by the VFID */ + vfid = pf_control_state_index(gt, cs); + xe_gt_assert(gt, vfid <= xe_gt_sriov_pf_get_totalvfs(gt)); + + more = pf_process_vf_state_machine(gt, vfid); + if (more) + pf_queue_vf(gt, vfid); + else if (!empty) + pf_queue_control_worker(gt); +} + +static void control_worker_func(struct work_struct *w) +{ + struct xe_gt *gt = container_of(w, struct xe_gt, sriov.pf.control.worker); + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + pf_worker_find_work(gt); +} + +static void pf_stop_worker(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + cancel_work_sync(>->sriov.pf.control.worker); +} + +static void control_fini_action(struct drm_device *dev, void *data) +{ + struct xe_gt *gt = data; + + pf_stop_worker(gt); +} + +/** + * xe_gt_sriov_pf_control_init() - Initialize PF's control data. + * @gt: the &xe_gt + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int n, totalvfs; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + totalvfs = xe_sriov_pf_get_totalvfs(xe); + for (n = 0; n <= totalvfs; n++) { + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, n); + + init_completion(&cs->done); + INIT_LIST_HEAD(&cs->link); + } + + spin_lock_init(>->sriov.pf.control.lock); + INIT_LIST_HEAD(>->sriov.pf.control.list); + INIT_WORK(>->sriov.pf.control.worker, control_worker_func); + + return drmm_add_action_or_reset(&xe->drm, control_fini_action, gt); +} + +/** + * xe_gt_sriov_pf_control_restart() - Restart SR-IOV control data after a GT reset. + * @gt: the &xe_gt + * + * Any per-VF status maintained by the PF or any ongoing VF control activity + * performed by the PF must be reset or cancelled when the GT is reset. + * + * This function is for PF only. + */ +void xe_gt_sriov_pf_control_restart(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int n, totalvfs; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + pf_stop_worker(gt); + + totalvfs = xe_sriov_pf_get_totalvfs(xe); + for (n = 1; n <= totalvfs; n++) + pf_enter_vf_ready(gt, n); +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h new file mode 100644 index 000000000..23182a5c5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_CONTROL_H_ +#define _XE_GT_SRIOV_PF_CONTROL_H_ + +#include +#include + +struct xe_gt; + +int xe_gt_sriov_pf_control_init(struct xe_gt *gt); +void xe_gt_sriov_pf_control_restart(struct xe_gt *gt); + +int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid); +bool xe_gt_sriov_pf_control_check_save_data_done(struct xe_gt *gt, unsigned int vfid); +bool xe_gt_sriov_pf_control_check_save_failed(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_process_save_data(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_trigger_save_vf(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_finish_save_vf(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_restore_data_done(struct xe_gt *gt, unsigned int vfid); +bool xe_gt_sriov_pf_control_check_restore_failed(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_process_restore_data(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_trigger_restore_vf(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_finish_restore_vf(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_prepare_flr(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_control_sync_flr(struct xe_gt *gt, unsigned int vfid, bool sync); +int xe_gt_sriov_pf_control_wait_flr(struct xe_gt *gt, unsigned int vfid); + +#ifdef CONFIG_PCI_IOV +int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len); +#else +static inline int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len) +{ + return -EPROTO; +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h new file mode 100644 index 000000000..e78c59e08 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_CONTROL_TYPES_H_ +#define _XE_GT_SRIOV_PF_CONTROL_TYPES_H_ + +#include +#include +#include + +/** + * enum xe_gt_sriov_control_bits - Various bits used by the PF to represent a VF state + * + * @XE_GT_SRIOV_STATE_WIP: indicates that some operations are in progress. + * @XE_GT_SRIOV_STATE_FLR_WIP: indicates that a VF FLR is in progress. + * @XE_GT_SRIOV_STATE_FLR_PREPARE: indicates that the PF received early VF FLR prepare notification. + * @XE_GT_SRIOV_STATE_FLR_SEND_START: indicates that the PF wants to send a FLR START command. + * @XE_GT_SRIOV_STATE_FLR_WAIT_GUC: indicates that the PF awaits for a response from the GuC. + * @XE_GT_SRIOV_STATE_FLR_GUC_DONE: indicates that the PF has received a response from the GuC. + * @XE_GT_SRIOV_STATE_FLR_SYNC: indicates that the PF awaits to synchronize with other GuCs. + * @XE_GT_SRIOV_STATE_FLR_RESET_CONFIG: indicates that the PF needs to clear VF's resources. + * @XE_GT_SRIOV_STATE_FLR_RESET_DATA: indicates that the PF needs to clear VF's data. + * @XE_GT_SRIOV_STATE_FLR_RESET_MMIO: indicates that the PF needs to reset VF's registers. + * @XE_GT_SRIOV_STATE_FLR_SEND_FINISH: indicates that the PF wants to send a FLR FINISH message. + * @XE_GT_SRIOV_STATE_FLR_FAILED: indicates that VF FLR sequence failed. + * @XE_GT_SRIOV_STATE_PAUSE_WIP: indicates that a VF pause operation is in progress. + * @XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE: indicates that the PF is about to send a PAUSE command. + * @XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC: indicates that the PF awaits for a response from the GuC. + * @XE_GT_SRIOV_STATE_PAUSE_GUC_DONE: indicates that the PF has received a response from the GuC. + * @XE_GT_SRIOV_STATE_PAUSE_FAILED: indicates that a VF pause operation has failed. + * @XE_GT_SRIOV_STATE_PAUSED: indicates that the VF is paused. + * @XE_GT_SRIOV_STATE_SAVE_WIP: indicates that VF save operation is in progress. + * @XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA: indicates that VF migration data is being produced. + * @XE_GT_SRIOV_STATE_SAVE_WAIT_DATA: indicates that PF awaits for space in migration data ring. + * @XE_GT_SRIOV_STATE_SAVE_DATA_DONE: indicates that all migration data was produced by Xe. + * @XE_GT_SRIOV_STATE_SAVE_FAILED: indicates that VF save operation has failed. + * @XE_GT_SRIOV_STATE_SAVED: indicates that VF data is saved. + * @XE_GT_SRIOV_STATE_RESTORE_WIP: indicates that VF restore operation is in progress. + * @XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA: indicates that VF migration data is being consumed. + * @XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA: indicates that PF awaits for data in migration data ring. + * @XE_GT_SRIOV_STATE_RESTORE_DATA_DONE: indicates that all migration data was produced by the user. + * @XE_GT_SRIOV_STATE_RESTORE_FAILED: indicates that VF restore operation has failed. + * @XE_GT_SRIOV_STATE_RESTORED: indicates that VF data is restored. + * @XE_GT_SRIOV_STATE_RESUME_WIP: indicates the a VF resume operation is in progress. + * @XE_GT_SRIOV_STATE_RESUME_SEND_RESUME: indicates that the PF is about to send RESUME command. + * @XE_GT_SRIOV_STATE_RESUME_FAILED: indicates that a VF resume operation has failed. + * @XE_GT_SRIOV_STATE_RESUMED: indicates that the VF was resumed. + * @XE_GT_SRIOV_STATE_STOP_WIP: indicates that a VF stop operation is in progress. + * @XE_GT_SRIOV_STATE_STOP_SEND_STOP: indicates that the PF wants to send a STOP command. + * @XE_GT_SRIOV_STATE_STOP_FAILED: indicates that the VF stop operation has failed + * @XE_GT_SRIOV_STATE_STOPPED: indicates that the VF was stopped. + * @XE_GT_SRIOV_STATE_MISMATCH: indicates that the PF has detected a VF state mismatch. + */ +enum xe_gt_sriov_control_bits { + XE_GT_SRIOV_STATE_WIP = 1, + + XE_GT_SRIOV_STATE_FLR_WIP, + XE_GT_SRIOV_STATE_FLR_PREPARE, + XE_GT_SRIOV_STATE_FLR_SEND_START, + XE_GT_SRIOV_STATE_FLR_WAIT_GUC, + XE_GT_SRIOV_STATE_FLR_GUC_DONE, + XE_GT_SRIOV_STATE_FLR_SYNC, + XE_GT_SRIOV_STATE_FLR_RESET_CONFIG, + XE_GT_SRIOV_STATE_FLR_RESET_DATA, + XE_GT_SRIOV_STATE_FLR_RESET_MMIO, + XE_GT_SRIOV_STATE_FLR_SEND_FINISH, + XE_GT_SRIOV_STATE_FLR_FAILED, + + XE_GT_SRIOV_STATE_PAUSE_WIP, + XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE, + XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC, + XE_GT_SRIOV_STATE_PAUSE_GUC_DONE, + XE_GT_SRIOV_STATE_PAUSE_FAILED, + XE_GT_SRIOV_STATE_PAUSED, + + XE_GT_SRIOV_STATE_SAVE_WIP, + XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA, + XE_GT_SRIOV_STATE_SAVE_WAIT_DATA, + XE_GT_SRIOV_STATE_SAVE_DATA_DONE, + XE_GT_SRIOV_STATE_SAVE_FAILED, + XE_GT_SRIOV_STATE_SAVED, + + XE_GT_SRIOV_STATE_RESTORE_WIP, + XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA, + XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA, + XE_GT_SRIOV_STATE_RESTORE_DATA_DONE, + XE_GT_SRIOV_STATE_RESTORE_FAILED, + XE_GT_SRIOV_STATE_RESTORED, + + XE_GT_SRIOV_STATE_RESUME_WIP, + XE_GT_SRIOV_STATE_RESUME_SEND_RESUME, + XE_GT_SRIOV_STATE_RESUME_FAILED, + XE_GT_SRIOV_STATE_RESUMED, + + XE_GT_SRIOV_STATE_STOP_WIP, + XE_GT_SRIOV_STATE_STOP_SEND_STOP, + XE_GT_SRIOV_STATE_STOP_FAILED, + XE_GT_SRIOV_STATE_STOPPED, + + XE_GT_SRIOV_STATE_MISMATCH, /* always keep as last */ +}; + +#define XE_GT_SRIOV_NUM_STATES (XE_GT_SRIOV_STATE_MISMATCH + 1) + +/** + * struct xe_gt_sriov_control_state - GT-level per-VF control state. + * + * Used by the PF driver to maintain per-VF control data. + */ +struct xe_gt_sriov_control_state { + /** @state: VF state bits */ + DECLARE_BITMAP(state, XE_GT_SRIOV_NUM_STATES); + + /** @done: completion of async operations */ + struct completion done; + + /** @link: link into worker list */ + struct list_head link; +}; + +/** + * struct xe_gt_sriov_pf_control - GT-level control data. + * + * Used by the PF driver to maintain its data. + */ +struct xe_gt_sriov_pf_control { + /** @worker: worker that executes a VF operations */ + struct work_struct worker; + + /** @list: list of VF entries that have a pending work */ + struct list_head list; + + /** @lock: protects VF pending list */ + spinlock_t lock; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c new file mode 100644 index 000000000..0f242db77 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c @@ -0,0 +1,916 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include + +#include +#include + +#include "xe_debugfs.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_debugfs.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_debugfs.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_gt_sriov_pf_monitor.h" +#include "xe_gt_sriov_pf_policy.h" +#include "xe_gt_sriov_pf_service.h" +#include "xe_guc.h" +#include "xe_pm.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_pf_provision.h" + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov # d_inode->i_private = (xe_device*) + * │ ├── pf # d_inode->i_private = (xe_device*) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ │ ├── gt0 # d_inode->i_private = (xe_gt*) + * │ │ │ ├── gt1 # d_inode->i_private = (xe_gt*) + * │ │ ├── tile1 + * │ │ │ : + * │ ├── vf1 # d_inode->i_private = VFID(1) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ │ ├── gt0 # d_inode->i_private = (xe_gt*) + * │ │ │ ├── gt1 # d_inode->i_private = (xe_gt*) + * │ │ ├── tile1 + * │ │ │ : + * : : + * │ ├── vfN # d_inode->i_private = VFID(N) + */ + +static void *extract_priv(struct dentry *d) +{ + return d->d_inode->i_private; +} + +static struct xe_gt *extract_gt(struct dentry *d) +{ + return extract_priv(d); +} + +static struct xe_device *extract_xe(struct dentry *d) +{ + return extract_priv(d->d_parent->d_parent->d_parent); +} + +static unsigned int extract_vfid(struct dentry *d) +{ + void *priv = extract_priv(d->d_parent->d_parent); + + return priv == extract_xe(d) ? PFID : (uintptr_t)priv; +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * : ├── tile0 + * : ├── gt0 + * : ├── contexts_provisioned + * ├── doorbells_provisioned + * ├── runtime_registers + * ├── adverse_events + */ + +static const struct drm_info_list pf_info[] = { + { + "contexts_provisioned", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_pf_config_print_ctxs, + }, + { + "doorbells_provisioned", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_pf_config_print_dbs, + }, + { + "runtime_registers", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_pf_service_print_runtime, + }, + { + "adverse_events", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_pf_monitor_print_events, + }, +}; + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * : ├── tile0 + * : ├── gt0 + * : ├── reset_engine + * ├── sample_period + * ├── sched_if_idle + */ + +#define DEFINE_SRIOV_GT_POLICY_DEBUGFS_ATTRIBUTE(POLICY, TYPE, FORMAT) \ + \ +static int POLICY##_set(void *data, u64 val) \ +{ \ + struct xe_gt *gt = extract_gt(data); \ + struct xe_device *xe = gt_to_xe(gt); \ + int err; \ + \ + if (val > (TYPE)~0ull) \ + return -EOVERFLOW; \ + \ + guard(xe_pm_runtime)(xe); \ + err = xe_gt_sriov_pf_policy_set_##POLICY(gt, val); \ + if (!err) \ + xe_sriov_pf_provision_set_custom_mode(xe); \ + \ + return err; \ +} \ + \ +static int POLICY##_get(void *data, u64 *val) \ +{ \ + struct xe_gt *gt = extract_gt(data); \ + \ + *val = xe_gt_sriov_pf_policy_get_##POLICY(gt); \ + return 0; \ +} \ + \ +DEFINE_DEBUGFS_ATTRIBUTE(POLICY##_fops, POLICY##_get, POLICY##_set, FORMAT) + +DEFINE_SRIOV_GT_POLICY_DEBUGFS_ATTRIBUTE(reset_engine, bool, "%llu\n"); +DEFINE_SRIOV_GT_POLICY_DEBUGFS_ATTRIBUTE(sched_if_idle, bool, "%llu\n"); +DEFINE_SRIOV_GT_POLICY_DEBUGFS_ATTRIBUTE(sample_period, u32, "%llu\n"); + +static void pf_add_policy_attrs(struct xe_gt *gt, struct dentry *parent) +{ + xe_gt_assert(gt, gt == extract_gt(parent)); + xe_gt_assert(gt, PFID == extract_vfid(parent)); + + debugfs_create_file_unsafe("reset_engine", 0644, parent, parent, &reset_engine_fops); + debugfs_create_file_unsafe("sched_if_idle", 0644, parent, parent, &sched_if_idle_fops); + debugfs_create_file_unsafe("sample_period_ms", 0644, parent, parent, &sample_period_fops); +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * : ├── tile0 + * : ├── gt0 + * : ├── sched_groups_mode + * ├── sched_groups_exec_quantums_ms + * ├── sched_groups_preempt_timeout_us + * ├── sched_groups + * : ├── group0 + * : + * : └── groupN + * ├── vf1 + * : ├── tile0 + * : ├── gt0 + * : ├── sched_groups_exec_quantums_ms + * ├── sched_groups_preempt_timeout_us + * : + */ + +static const char *sched_group_mode_to_string(enum xe_sriov_sched_group_modes mode) +{ + switch (mode) { + case XE_SRIOV_SCHED_GROUPS_DISABLED: + return "disabled"; + case XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES: + return "media_slices"; + case XE_SRIOV_SCHED_GROUPS_MODES_COUNT: + /* dummy mode to make the compiler happy */ + break; + } + + return "unknown"; +} + +static int sched_groups_info(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + struct xe_gt *gt = extract_gt(m->private); + enum xe_sriov_sched_group_modes current_mode = + gt->sriov.pf.policy.guc.sched_groups.current_mode; + enum xe_sriov_sched_group_modes mode; + + for (mode = XE_SRIOV_SCHED_GROUPS_DISABLED; + mode < XE_SRIOV_SCHED_GROUPS_MODES_COUNT; + mode++) { + if (!xe_sriov_gt_pf_policy_has_sched_group_mode(gt, mode)) + continue; + + drm_printf(&p, "%s%s%s%s", + mode == XE_SRIOV_SCHED_GROUPS_DISABLED ? "" : " ", + mode == current_mode ? "[" : "", + sched_group_mode_to_string(mode), + mode == current_mode ? "]" : ""); + } + + drm_puts(&p, "\n"); + + return 0; +} + +static int sched_groups_open(struct inode *inode, struct file *file) +{ + return single_open(file, sched_groups_info, inode->i_private); +} + +static ssize_t sched_groups_write(struct file *file, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_gt *gt = extract_gt(file_inode(file)->i_private); + enum xe_sriov_sched_group_modes mode; + char name[32]; + int ret; + + if (*pos) + return -ESPIPE; + + if (!size) + return -ENODATA; + + if (size > sizeof(name) - 1) + return -EINVAL; + + ret = simple_write_to_buffer(name, sizeof(name) - 1, pos, ubuf, size); + if (ret < 0) + return ret; + name[ret] = '\0'; + + for (mode = XE_SRIOV_SCHED_GROUPS_DISABLED; + mode < XE_SRIOV_SCHED_GROUPS_MODES_COUNT; + mode++) + if (sysfs_streq(name, sched_group_mode_to_string(mode))) + break; + + if (mode == XE_SRIOV_SCHED_GROUPS_MODES_COUNT) + return -EINVAL; + + guard(xe_pm_runtime)(gt_to_xe(gt)); + ret = xe_gt_sriov_pf_policy_set_sched_groups_mode(gt, mode); + + return ret < 0 ? ret : size; +} + +static const struct file_operations sched_groups_fops = { + .owner = THIS_MODULE, + .open = sched_groups_open, + .read = seq_read, + .write = sched_groups_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int sched_groups_config_show(struct seq_file *m, void *data, + void (*get)(struct xe_gt *, unsigned int, u32 *, u32)) +{ + struct drm_printer p = drm_seq_file_printer(m); + unsigned int vfid = extract_vfid(m->private); + struct xe_gt *gt = extract_gt(m->private); + u32 values[GUC_MAX_SCHED_GROUPS]; + bool first = true; + u8 group; + + get(gt, vfid, values, ARRAY_SIZE(values)); + + for (group = 0; group < ARRAY_SIZE(values); group++) { + drm_printf(&p, "%s%u", first ? "" : ",", values[group]); + + first = false; + } + + drm_puts(&p, "\n"); + + return 0; +} + +static ssize_t sched_groups_config_write(struct file *file, const char __user *ubuf, + size_t size, loff_t *pos, + int (*set)(struct xe_gt *, unsigned int, u32 *, u32)) +{ + struct dentry *parent = file_inode(file)->i_private; + unsigned int vfid = extract_vfid(parent); + struct xe_gt *gt = extract_gt(parent); + u32 values[GUC_MAX_SCHED_GROUPS]; + int *input __free(kfree) = NULL; + u32 count; + int ret; + int i; + + if (*pos) + return -ESPIPE; + + if (!size) + return -ENODATA; + + ret = parse_int_array_user(ubuf, min(size, GUC_MAX_SCHED_GROUPS * sizeof(u32)), &input); + if (ret) + return ret; + + count = input[0]; + if (count > GUC_MAX_SCHED_GROUPS) + return -E2BIG; + + for (i = 0; i < count; i++) { + if (input[i + 1] < 0 || input[i + 1] > S32_MAX) + return -EINVAL; + + values[i] = input[i + 1]; + } + + guard(xe_pm_runtime)(gt_to_xe(gt)); + ret = set(gt, vfid, values, count); + + return ret < 0 ? ret : size; +} + +#define DEFINE_SRIOV_GT_GRP_CFG_DEBUGFS_ATTRIBUTE(CONFIG) \ +static int sched_groups_##CONFIG##_show(struct seq_file *m, void *data) \ +{ \ + return sched_groups_config_show(m, data, \ + xe_gt_sriov_pf_config_get_groups_##CONFIG); \ +} \ + \ +static int sched_groups_##CONFIG##_open(struct inode *inode, struct file *file) \ +{ \ + return single_open(file, sched_groups_##CONFIG##_show, \ + inode->i_private); \ +} \ + \ +static ssize_t sched_groups_##CONFIG##_write(struct file *file, \ + const char __user *ubuf, \ + size_t size, loff_t *pos) \ +{ \ + return sched_groups_config_write(file, ubuf, size, pos, \ + xe_gt_sriov_pf_config_set_groups_##CONFIG); \ +} \ + \ +static const struct file_operations sched_groups_##CONFIG##_fops = { \ + .owner = THIS_MODULE, \ + .open = sched_groups_##CONFIG##_open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .write = sched_groups_##CONFIG##_write, \ + .release = single_release, \ +} + +DEFINE_SRIOV_GT_GRP_CFG_DEBUGFS_ATTRIBUTE(exec_quantums); +DEFINE_SRIOV_GT_GRP_CFG_DEBUGFS_ATTRIBUTE(preempt_timeouts); + +static ssize_t sched_group_engines_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dentry *dent = file_dentry(file); + struct xe_gt *gt = extract_gt(dent->d_parent->d_parent); + struct xe_gt_sriov_scheduler_groups *info = >->sriov.pf.policy.guc.sched_groups; + struct guc_sched_group *groups = info->modes[info->current_mode].groups; + u32 num_groups = info->modes[info->current_mode].num_groups; + unsigned int group = (uintptr_t)extract_priv(dent); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + char engines[128]; + + engines[0] = '\0'; + + if (group < num_groups) { + for_each_hw_engine(hwe, gt, id) { + u8 guc_class = xe_hwe_to_guc_class(hwe); + u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); + u32 mask = groups[group].engines[guc_class]; + + if (mask & BIT(guc_logical_instance)) { + strlcat(engines, hwe->name, sizeof(engines)); + strlcat(engines, " ", sizeof(engines)); + } + } + strlcat(engines, "\n", sizeof(engines)); + } + + return simple_read_from_buffer(buf, count, ppos, engines, strlen(engines)); +} + +static const struct file_operations sched_group_engines_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = sched_group_engines_read, + .llseek = default_llseek, +}; + +static void pf_add_sched_groups(struct xe_gt *gt, struct dentry *parent, unsigned int vfid) +{ + struct dentry *groups; + u8 group; + + xe_gt_assert(gt, gt == extract_gt(parent)); + xe_gt_assert(gt, vfid == extract_vfid(parent)); + + /* + * TODO: we currently call this function before we initialize scheduler + * groups, so at this point in time we don't know if there are any + * valid groups on the GT and we can't selectively register the debugfs + * only if there are any. Therefore, we always register the debugfs + * files if we're on a platform that has support for groups. + * We should rework the flow so that debugfs is registered after the + * policy init, so that we check if there are valid groups before + * adding the debugfs files. + * Similarly, instead of using GUC_MAX_SCHED_GROUPS we could use + * gt->sriov.pf.policy.guc.sched_groups.max_number_of_groups. + */ + if (!xe_sriov_gt_pf_policy_has_sched_groups_support(gt)) + return; + + debugfs_create_file("sched_groups_exec_quantums_ms", 0644, parent, parent, + &sched_groups_exec_quantums_fops); + debugfs_create_file("sched_groups_preempt_timeouts_us", 0644, parent, parent, + &sched_groups_preempt_timeouts_fops); + + if (vfid != PFID) + return; + + debugfs_create_file("sched_groups_mode", 0644, parent, parent, &sched_groups_fops); + + groups = debugfs_create_dir("sched_groups", parent); + if (IS_ERR(groups)) + return; + + for (group = 0; group < GUC_MAX_SCHED_GROUPS; group++) { + char name[10]; + + snprintf(name, sizeof(name), "group%u", group); + debugfs_create_file(name, 0644, groups, (void *)(uintptr_t)group, + &sched_group_engines_fops); + } +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * │ ├── tile0 + * │ : ├── gt0 + * │ : ├── doorbells_spare + * │ ├── contexts_spare + * │ ├── exec_quantum_ms + * │ ├── preempt_timeout_us + * │ ├── sched_priority + * ├── vf1 + * : ├── tile0 + * : ├── gt0 + * : ├── doorbells_quota + * ├── contexts_quota + * ├── exec_quantum_ms + * ├── preempt_timeout_us + * ├── sched_priority + */ + +#define DEFINE_SRIOV_GT_CONFIG_DEBUGFS_ATTRIBUTE(CONFIG, TYPE, FORMAT) \ + \ +static int CONFIG##_set(void *data, u64 val) \ +{ \ + struct xe_gt *gt = extract_gt(data); \ + unsigned int vfid = extract_vfid(data); \ + struct xe_device *xe = gt_to_xe(gt); \ + int err; \ + \ + if (val > (TYPE)~0ull) \ + return -EOVERFLOW; \ + \ + guard(xe_pm_runtime)(xe); \ + err = xe_sriov_pf_wait_ready(xe) ?: \ + xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ + if (!err) \ + xe_sriov_pf_provision_set_custom_mode(xe); \ + \ + return err; \ +} \ + \ +static int CONFIG##_get(void *data, u64 *val) \ +{ \ + struct xe_gt *gt = extract_gt(data); \ + unsigned int vfid = extract_vfid(data); \ + \ + *val = xe_gt_sriov_pf_config_get_##CONFIG(gt, vfid); \ + return 0; \ +} \ + \ +DEFINE_DEBUGFS_ATTRIBUTE(CONFIG##_fops, CONFIG##_get, CONFIG##_set, FORMAT) + +DEFINE_SRIOV_GT_CONFIG_DEBUGFS_ATTRIBUTE(ctxs, u32, "%llu\n"); +DEFINE_SRIOV_GT_CONFIG_DEBUGFS_ATTRIBUTE(dbs, u32, "%llu\n"); +DEFINE_SRIOV_GT_CONFIG_DEBUGFS_ATTRIBUTE(exec_quantum, u32, "%llu\n"); +DEFINE_SRIOV_GT_CONFIG_DEBUGFS_ATTRIBUTE(preempt_timeout, u32, "%llu\n"); +DEFINE_SRIOV_GT_CONFIG_DEBUGFS_ATTRIBUTE(sched_priority, u32, "%llu\n"); + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * │ ├── tile0 + * │ : ├── gt0 + * │ : ├── threshold_cat_error_count + * │ ├── threshold_doorbell_time_us + * │ ├── threshold_engine_reset_count + * │ ├── threshold_guc_time_us + * │ ├── threshold_irq_time_us + * │ ├── threshold_page_fault_count + * ├── vf1 + * : ├── tile0 + * : ├── gt0 + * : ├── threshold_cat_error_count + * ├── threshold_doorbell_time_us + * ├── threshold_engine_reset_count + * ├── threshold_guc_time_us + * ├── threshold_irq_time_us + * ├── threshold_page_fault_count + */ + +static int set_threshold(void *data, u64 val, enum xe_guc_klv_threshold_index index) +{ + struct xe_gt *gt = extract_gt(data); + unsigned int vfid = extract_vfid(data); + struct xe_device *xe = gt_to_xe(gt); + int err; + + if (val > (u32)~0ull) + return -EOVERFLOW; + + guard(xe_pm_runtime)(xe); + err = xe_gt_sriov_pf_config_set_threshold(gt, vfid, index, val); + if (!err) + xe_sriov_pf_provision_set_custom_mode(xe); + + return err; +} + +static int get_threshold(void *data, u64 *val, enum xe_guc_klv_threshold_index index) +{ + struct xe_gt *gt = extract_gt(data); + unsigned int vfid = extract_vfid(data); + + *val = xe_gt_sriov_pf_config_get_threshold(gt, vfid, index); + return 0; +} + +#define DEFINE_SRIOV_GT_THRESHOLD_DEBUGFS_ATTRIBUTE(THRESHOLD, INDEX) \ + \ +static int THRESHOLD##_set(void *data, u64 val) \ +{ \ + return set_threshold(data, val, INDEX); \ +} \ + \ +static int THRESHOLD##_get(void *data, u64 *val) \ +{ \ + return get_threshold(data, val, INDEX); \ +} \ + \ +DEFINE_DEBUGFS_ATTRIBUTE(THRESHOLD##_fops, THRESHOLD##_get, THRESHOLD##_set, "%llu\n") + +/* generate all threshold attributes */ +#define define_threshold_attribute(TAG, NAME, ...) \ + DEFINE_SRIOV_GT_THRESHOLD_DEBUGFS_ATTRIBUTE(NAME, MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)); +MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_threshold_attribute) +#undef define_threshold_attribute + +static void pf_add_config_attrs(struct xe_gt *gt, struct dentry *parent, unsigned int vfid) +{ + xe_gt_assert(gt, gt == extract_gt(parent)); + xe_gt_assert(gt, vfid == extract_vfid(parent)); + + debugfs_create_file_unsafe(vfid ? "doorbells_quota" : "doorbells_spare", + 0644, parent, parent, &dbs_fops); + debugfs_create_file_unsafe(vfid ? "contexts_quota" : "contexts_spare", + 0644, parent, parent, &ctxs_fops); + debugfs_create_file_unsafe("exec_quantum_ms", 0644, parent, parent, + &exec_quantum_fops); + debugfs_create_file_unsafe("preempt_timeout_us", 0644, parent, parent, + &preempt_timeout_fops); + debugfs_create_file_unsafe("sched_priority", 0644, parent, parent, + &sched_priority_fops); + + /* register all threshold attributes */ +#define register_threshold_attribute(TAG, NAME, VER...) ({ \ + if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, VER), true, VER)) \ + debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \ + &NAME##_fops); \ +}); + MAKE_XE_GUC_KLV_THRESHOLDS_SET(register_threshold_attribute) +#undef register_threshold_attribute +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── vf1 + * : ├── tile0 + * : ├── gt0 + * : ├── control { stop, pause, resume } + */ + +static const struct { + const char *cmd; + int (*fn)(struct xe_gt *gt, unsigned int vfid); +} control_cmds[] = { + { "stop", xe_gt_sriov_pf_control_stop_vf }, + { "pause", xe_gt_sriov_pf_control_pause_vf }, + { "resume", xe_gt_sriov_pf_control_resume_vf }, +}; + +static ssize_t control_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) +{ + struct dentry *dent = file_dentry(file); + struct dentry *parent = dent->d_parent; + struct xe_gt *gt = extract_gt(parent); + struct xe_device *xe = gt_to_xe(gt); + unsigned int vfid = extract_vfid(parent); + int ret = -EINVAL; + char cmd[32]; + size_t n; + + xe_gt_assert(gt, vfid); + xe_gt_sriov_pf_assert_vfid(gt, vfid); + + if (*pos) + return -ESPIPE; + + if (count > sizeof(cmd) - 1) + return -EINVAL; + + ret = simple_write_to_buffer(cmd, sizeof(cmd) - 1, pos, buf, count); + if (ret < 0) + return ret; + cmd[ret] = '\0'; + + for (n = 0; n < ARRAY_SIZE(control_cmds); n++) { + xe_gt_assert(gt, sizeof(cmd) > strlen(control_cmds[n].cmd)); + + if (sysfs_streq(cmd, control_cmds[n].cmd)) { + guard(xe_pm_runtime)(xe); + ret = control_cmds[n].fn ? (*control_cmds[n].fn)(gt, vfid) : 0; + break; + } + } + + return (ret < 0) ? ret : count; +} + +static ssize_t control_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) +{ + char help[128]; + size_t n; + + help[0] = '\0'; + for (n = 0; n < ARRAY_SIZE(control_cmds); n++) { + strlcat(help, control_cmds[n].cmd, sizeof(help)); + strlcat(help, "\n", sizeof(help)); + } + + return simple_read_from_buffer(buf, count, ppos, help, strlen(help)); +} + +static const struct file_operations control_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = control_write, + .read = control_read, + .llseek = default_llseek, +}; + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── vf1 + * : ├── tile0 + * : ├── gt0 + * : ├── config_blob + */ + +struct config_blob_data { + size_t size; + u8 blob[]; +}; + +static int config_blob_open(struct inode *inode, struct file *file) +{ + struct dentry *dent = file_dentry(file); + struct dentry *parent = dent->d_parent; + struct xe_gt *gt = extract_gt(parent); + unsigned int vfid = extract_vfid(parent); + struct config_blob_data *cbd; + ssize_t ret; + + ret = xe_gt_sriov_pf_config_save(gt, vfid, NULL, 0); + if (!ret) + return -ENODATA; + if (ret < 0) + return ret; + + cbd = kzalloc_flex(*cbd, blob, ret); + if (!cbd) + return -ENOMEM; + + ret = xe_gt_sriov_pf_config_save(gt, vfid, cbd->blob, ret); + if (ret < 0) { + kfree(cbd); + return ret; + } + + cbd->size = ret; + file->private_data = cbd; + return nonseekable_open(inode, file); +} + +static ssize_t config_blob_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct config_blob_data *cbd = file->private_data; + + return simple_read_from_buffer(buf, count, pos, cbd->blob, cbd->size); +} + +static ssize_t config_blob_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + struct dentry *dent = file_dentry(file); + struct dentry *parent = dent->d_parent; + struct xe_gt *gt = extract_gt(parent); + unsigned int vfid = extract_vfid(parent); + ssize_t ret; + void *tmp; + + if (*pos) + return -EINVAL; + + if (!count) + return -ENODATA; + + if (count > SZ_4K) + return -EINVAL; + + tmp = kzalloc(count, GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + if (copy_from_user(tmp, buf, count)) { + ret = -EFAULT; + } else { + ret = xe_gt_sriov_pf_config_restore(gt, vfid, tmp, count); + if (!ret) + ret = count; + } + kfree(tmp); + return ret; +} + +static int config_blob_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static const struct file_operations config_blob_ops = { + .owner = THIS_MODULE, + .open = config_blob_open, + .read = config_blob_read, + .write = config_blob_write, + .release = config_blob_release, +}; + +static void pf_add_compat_attrs(struct xe_gt *gt, struct dentry *dent, unsigned int vfid) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (!xe_gt_is_main_type(gt)) + return; + + if (vfid) { + debugfs_create_symlink("ggtt_quota", dent, "../ggtt_quota"); + if (xe_device_has_lmtt(xe)) + debugfs_create_symlink("lmem_quota", dent, "../vram_quota"); + } else { + debugfs_create_symlink("ggtt_spare", dent, "../ggtt_spare"); + debugfs_create_symlink("ggtt_available", dent, "../ggtt_available"); + debugfs_create_symlink("ggtt_provisioned", dent, "../ggtt_provisioned"); + if (xe_device_has_lmtt(xe)) { + debugfs_create_symlink("lmem_spare", dent, "../vram_spare"); + debugfs_create_symlink("lmem_provisioned", dent, "../vram_provisioned"); + } + } +} + +static void pf_populate_gt(struct xe_gt *gt, struct dentry *dent, unsigned int vfid) +{ + struct xe_device *xe = gt_to_xe(gt); + struct drm_minor *minor = xe->drm.primary; + + if (vfid) { + pf_add_config_attrs(gt, dent, vfid); + pf_add_sched_groups(gt, dent, vfid); + + debugfs_create_file("control", 0600, dent, NULL, &control_ops); + + /* for testing/debugging purposes only! */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + debugfs_create_file("config_blob", + IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ? 0600 : 0400, + dent, NULL, &config_blob_ops); + } + + } else { + pf_add_config_attrs(gt, dent, PFID); + pf_add_policy_attrs(gt, dent); + pf_add_sched_groups(gt, dent, PFID); + + drm_debugfs_create_files(pf_info, ARRAY_SIZE(pf_info), dent, minor); + } + + /* for backward compatibility only */ + pf_add_compat_attrs(gt, dent, vfid); +} + +/** + * xe_gt_sriov_pf_debugfs_populate() - Create SR-IOV GT-level debugfs directories and files. + * @gt: the &xe_gt to register + * @parent: the parent &dentry that represents a &xe_tile + * @vfid: the VF identifier + * + * Add to the @parent directory new debugfs directory that will represent a @gt and + * populate it with GT files that are related to the SR-IOV @vfid function. + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_debugfs_populate(struct xe_gt *gt, struct dentry *parent, unsigned int vfid) +{ + struct dentry *dent; + char name[8]; /* should be enough up to "gt%u\0" for 2^8 - 1 */ + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, extract_priv(parent) == gt->tile); + xe_gt_assert(gt, extract_priv(parent->d_parent) == gt_to_xe(gt) || + (uintptr_t)extract_priv(parent->d_parent) == vfid); + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * │ ├── pf + * │ │ ├── tile0 # parent + * │ │ │ ├── gt0 # d_inode->i_private = (xe_gt*) + * │ │ │ ├── gt1 + * │ │ : : + * │ ├── vf1 + * │ │ ├── tile0 # parent + * │ │ │ ├── gt0 # d_inode->i_private = (xe_gt*) + * │ │ │ ├── gt1 + * │ : : : + */ + snprintf(name, sizeof(name), "gt%u", gt->info.id); + dent = debugfs_create_dir(name, parent); + if (IS_ERR(dent)) + return; + dent->d_inode->i_private = gt; + + xe_gt_assert(gt, extract_gt(dent) == gt); + xe_gt_assert(gt, extract_vfid(dent) == vfid); + + pf_populate_gt(gt, dent, vfid); +} + +static void pf_add_links(struct xe_gt *gt, struct dentry *dent) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + unsigned int vfid; + char name[16]; /* should be more than enough for "vf%u\0" and VFID(UINT_MAX) */ + char symlink[64]; /* should be more enough for "../../sriov/vf%u/tile%u/gt%u\0" */ + + for (vfid = 0; vfid <= totalvfs; vfid++) { + if (vfid) + snprintf(name, sizeof(name), "vf%u", vfid); + else + snprintf(name, sizeof(name), "pf"); + snprintf(symlink, sizeof(symlink), "../../sriov/%s/tile%u/gt%u", + name, gt->tile->id, gt->info.id); + debugfs_create_symlink(name, dent, symlink); + } +} + +/** + * xe_gt_sriov_pf_debugfs_register - Register SR-IOV PF specific entries in GT debugfs. + * @gt: the &xe_gt to register + * @dent: the &dentry that represents the GT directory + * + * Instead of actual files, create symlinks for PF and each VF to their GT specific + * attributes that should be already exposed in the dedicated debugfs SR-IOV tree. + */ +void xe_gt_sriov_pf_debugfs_register(struct xe_gt *gt, struct dentry *dent) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, dent->d_inode->i_private == gt); + + pf_add_links(gt, dent); +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.h new file mode 100644 index 000000000..82ff3b7f0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_DEBUGFS_H_ +#define _XE_GT_SRIOV_PF_DEBUGFS_H_ + +struct xe_gt; +struct dentry; + +#ifdef CONFIG_PCI_IOV +void xe_gt_sriov_pf_debugfs_register(struct xe_gt *gt, struct dentry *root); +void xe_gt_sriov_pf_debugfs_populate(struct xe_gt *gt, struct dentry *parent, unsigned int vfid); +#else +static inline void xe_gt_sriov_pf_debugfs_register(struct xe_gt *gt, struct dentry *root) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h new file mode 100644 index 000000000..6af219d93 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_HELPERS_H_ +#define _XE_GT_SRIOV_PF_HELPERS_H_ + +#include "xe_gt_types.h" +#include "xe_sriov_pf_helpers.h" + +/** + * xe_gt_sriov_pf_assert_vfid() - warn if &id is not a supported VF number when debugging. + * @gt: the PF &xe_gt to assert on + * @vfid: the VF number to assert + * + * Assert that > belongs to the Physical Function (PF) device and provided &vfid + * is within a range of supported VF numbers (up to maximum number of VFs that + * driver can support, including VF0 that represents the PF itself). + * + * Note: Effective only on debug builds. See `Xe Asserts`_ for more information. + */ +#define xe_gt_sriov_pf_assert_vfid(gt, vfid) xe_sriov_pf_assert_vfid(gt_to_xe(gt), (vfid)) + +static inline int xe_gt_sriov_pf_get_totalvfs(struct xe_gt *gt) +{ + return xe_sriov_pf_get_totalvfs(gt_to_xe(gt)); +} + +static inline struct mutex *xe_gt_sriov_pf_master_mutex(struct xe_gt *gt) +{ + return xe_sriov_pf_master_mutex(gt_to_xe(gt)); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c new file mode 100644 index 000000000..01fe03b9e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -0,0 +1,1069 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include "regs/xe_guc_regs.h" + +#include "abi/guc_actions_sriov_abi.h" +#include "xe_bo.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_sriov_pf.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc.h" +#include "xe_guc_buf.h" +#include "xe_guc_ct.h" +#include "xe_migrate.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_sriov_packet.h" +#include "xe_sriov_packet_types.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" + +#define XE_GT_SRIOV_PF_MIGRATION_RING_SIZE 5 + +static struct xe_gt_sriov_migration_data *pf_pick_gt_migration(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return >->sriov.pf.vfs[vfid].migration; +} + +static void pf_dump_mig_data(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data, + const char *what) +{ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + + drm_printf(&p, "VF%u %s (%llu bytes)\n", vfid, what, data->hdr.size); + drm_print_hex_dump(&p, "mig_hdr: ", (void *)&data->hdr, sizeof(data->hdr)); + drm_print_hex_dump(&p, "mig_data: ", data->vaddr, min(SZ_64, data->hdr.size)); + } +} + +static ssize_t pf_migration_ggtt_size(struct xe_gt *gt, unsigned int vfid) +{ + if (!xe_gt_is_main_type(gt)) + return 0; + + return xe_gt_sriov_pf_config_ggtt_save(gt, vfid, NULL, 0); +} + +static int pf_save_vf_ggtt_mig_data(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_sriov_packet *data; + size_t size; + int ret; + + size = pf_migration_ggtt_size(gt, vfid); + xe_gt_assert(gt, size); + + data = xe_sriov_packet_alloc(gt_to_xe(gt)); + if (!data) + return -ENOMEM; + + ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, + XE_SRIOV_PACKET_TYPE_GGTT, 0, size); + if (ret) + goto fail; + + ret = xe_gt_sriov_pf_config_ggtt_save(gt, vfid, data->vaddr, size); + if (ret) + goto fail; + + pf_dump_mig_data(gt, vfid, data, "GGTT data save"); + + ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); + if (ret) + goto fail; + + return 0; + +fail: + xe_sriov_packet_free(data); + xe_gt_sriov_err(gt, "Failed to save VF%u GGTT data (%pe)\n", vfid, ERR_PTR(ret)); + return ret; +} + +static int pf_restore_vf_ggtt_mig_data(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + int ret; + + pf_dump_mig_data(gt, vfid, data, "GGTT data restore"); + + ret = xe_gt_sriov_pf_config_ggtt_restore(gt, vfid, data->vaddr, data->hdr.size); + if (ret) { + xe_gt_sriov_err(gt, "Failed to restore VF%u GGTT data (%pe)\n", + vfid, ERR_PTR(ret)); + return ret; + } + + return 0; +} + +/** + * xe_gt_sriov_pf_migration_ggtt_save() - Save VF GGTT migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_ggtt_save(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return pf_save_vf_ggtt_mig_data(gt, vfid); +} + +/** + * xe_gt_sriov_pf_migration_ggtt_restore() - Restore VF GGTT migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * @data: the &xe_sriov_packet containing migration data + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_ggtt_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return pf_restore_vf_ggtt_mig_data(gt, vfid, data); +} + +/* Return: number of dwords saved/restored/required or a negative error code on failure */ +static int guc_action_vf_save_restore(struct xe_guc *guc, u32 vfid, u32 opcode, + u64 addr, u32 ndwords) +{ + u32 request[PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_PF2GUC_SAVE_RESTORE_VF) | + FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_0_OPCODE, opcode), + FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_1_VFID, vfid), + FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_2_ADDR_LO, lower_32_bits(addr)), + FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_3_ADDR_HI, upper_32_bits(addr)), + FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_4_SIZE, ndwords), + }; + + return xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request)); +} + +/* Return: size of the state in dwords or a negative error code on failure */ +static int pf_send_guc_query_vf_mig_data_size(struct xe_gt *gt, unsigned int vfid) +{ + int ret; + + ret = guc_action_vf_save_restore(>->uc.guc, vfid, GUC_PF_OPCODE_VF_SAVE, 0, 0); + return ret ?: -ENODATA; +} + +/* Return: number of state dwords saved or a negative error code on failure */ +static int pf_send_guc_save_vf_mig_data(struct xe_gt *gt, unsigned int vfid, + void *dst, size_t size) +{ + const int ndwords = size / sizeof(u32); + struct xe_guc *guc = >->uc.guc; + CLASS(xe_guc_buf, buf)(&guc->buf, ndwords); + int ret; + + xe_gt_assert(gt, size % sizeof(u32) == 0); + xe_gt_assert(gt, size == ndwords * sizeof(u32)); + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + /* FW expects this buffer to be zero-initialized */ + memset(xe_guc_buf_cpu_ptr(buf), 0, size); + + ret = guc_action_vf_save_restore(guc, vfid, GUC_PF_OPCODE_VF_SAVE, + xe_guc_buf_flush(buf), ndwords); + if (!ret) + ret = -ENODATA; + else if (ret > ndwords) + ret = -EPROTO; + else if (ret > 0) + memcpy(dst, xe_guc_buf_sync_read(buf), ret * sizeof(u32)); + + return ret; +} + +/* Return: number of state dwords restored or a negative error code on failure */ +static int pf_send_guc_restore_vf_mig_data(struct xe_gt *gt, unsigned int vfid, + const void *src, size_t size) +{ + const int ndwords = size / sizeof(u32); + struct xe_guc *guc = >->uc.guc; + CLASS(xe_guc_buf_from_data, buf)(&guc->buf, src, size); + int ret; + + xe_gt_assert(gt, size % sizeof(u32) == 0); + xe_gt_assert(gt, size == ndwords * sizeof(u32)); + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + ret = guc_action_vf_save_restore(guc, vfid, GUC_PF_OPCODE_VF_RESTORE, + xe_guc_buf_flush(buf), ndwords); + if (!ret) + ret = -ENODATA; + else if (ret > ndwords) + ret = -EPROTO; + + return ret; +} + +static bool pf_migration_supported(struct xe_gt *gt) +{ + return xe_sriov_pf_migration_supported(gt_to_xe(gt)); +} + +static int pf_save_vf_guc_mig_data(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_sriov_packet *data; + size_t size; + int ret; + + ret = pf_send_guc_query_vf_mig_data_size(gt, vfid); + if (ret < 0) + goto fail; + + size = ret * sizeof(u32); + + data = xe_sriov_packet_alloc(gt_to_xe(gt)); + if (!data) { + ret = -ENOMEM; + goto fail; + } + + ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, + XE_SRIOV_PACKET_TYPE_GUC, 0, size); + if (ret) + goto fail_free; + + ret = pf_send_guc_save_vf_mig_data(gt, vfid, data->vaddr, size); + if (ret < 0) + goto fail_free; + size = ret * sizeof(u32); + xe_gt_assert(gt, size); + xe_gt_assert(gt, size <= data->hdr.size); + data->hdr.size = size; + data->remaining = size; + + pf_dump_mig_data(gt, vfid, data, "GuC data save"); + + ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); + if (ret) + goto fail_free; + + return 0; + +fail_free: + xe_sriov_packet_free(data); +fail: + xe_gt_sriov_err(gt, "Failed to save VF%u GuC data (%pe)\n", + vfid, ERR_PTR(ret)); + return ret; +} + +static ssize_t pf_migration_guc_size(struct xe_gt *gt, unsigned int vfid) +{ + ssize_t size; + + if (!pf_migration_supported(gt)) + return -ENOPKG; + + size = pf_send_guc_query_vf_mig_data_size(gt, vfid); + if (size >= 0) + size *= sizeof(u32); + + return size; +} + +/** + * xe_gt_sriov_pf_migration_guc_save() - Save VF GuC migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_guc_save(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + if (!pf_migration_supported(gt)) + return -ENOPKG; + + return pf_save_vf_guc_mig_data(gt, vfid); +} + +static int pf_restore_vf_guc_state(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + int ret; + + xe_gt_assert(gt, data->hdr.size); + + pf_dump_mig_data(gt, vfid, data, "GuC data restore"); + + ret = pf_send_guc_restore_vf_mig_data(gt, vfid, data->vaddr, data->hdr.size); + if (ret < 0) + goto fail; + + return 0; + +fail: + xe_gt_sriov_err(gt, "Failed to restore VF%u GuC data (%pe)\n", + vfid, ERR_PTR(ret)); + return ret; +} + +/** + * xe_gt_sriov_pf_migration_guc_restore() - Restore VF GuC migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @data: the &xe_sriov_packet containing migration data + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_guc_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + if (!pf_migration_supported(gt)) + return -ENOPKG; + + return pf_restore_vf_guc_state(gt, vfid, data); +} + +static ssize_t pf_migration_mmio_size(struct xe_gt *gt, unsigned int vfid) +{ + if (xe_gt_is_media_type(gt)) + return MED_VF_SW_FLAG_COUNT * sizeof(u32); + else + return VF_SW_FLAG_COUNT * sizeof(u32); +} + +static int pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid, void *buf, size_t size) +{ + struct xe_mmio mmio; + u32 *regs = buf; + int n; + + if (size != pf_migration_mmio_size(gt, vfid)) + return -EINVAL; + + xe_mmio_init_vf_view(&mmio, >->mmio, vfid); + + if (xe_gt_is_media_type(gt)) + for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) + regs[n] = xe_mmio_read32(&mmio, MED_VF_SW_FLAG(n)); + else + for (n = 0; n < VF_SW_FLAG_COUNT; n++) + regs[n] = xe_mmio_read32(&mmio, VF_SW_FLAG(n)); + + return 0; +} + +static int pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, + const void *buf, size_t size) +{ + const u32 *regs = buf; + struct xe_mmio mmio; + int n; + + if (size != pf_migration_mmio_size(gt, vfid)) + return -EINVAL; + + xe_mmio_init_vf_view(&mmio, >->mmio, vfid); + + if (xe_gt_is_media_type(gt)) + for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) + xe_mmio_write32(&mmio, MED_VF_SW_FLAG(n), regs[n]); + else + for (n = 0; n < VF_SW_FLAG_COUNT; n++) + xe_mmio_write32(&mmio, VF_SW_FLAG(n), regs[n]); + + return 0; +} + +static int pf_save_vf_mmio_mig_data(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_sriov_packet *data; + size_t size; + int ret; + + size = pf_migration_mmio_size(gt, vfid); + xe_gt_assert(gt, size); + + data = xe_sriov_packet_alloc(gt_to_xe(gt)); + if (!data) + return -ENOMEM; + + ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, + XE_SRIOV_PACKET_TYPE_MMIO, 0, size); + if (ret) + goto fail; + + ret = pf_migration_mmio_save(gt, vfid, data->vaddr, size); + if (ret) + goto fail; + + pf_dump_mig_data(gt, vfid, data, "MMIO data save"); + + ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); + if (ret) + goto fail; + + return 0; + +fail: + xe_sriov_packet_free(data); + xe_gt_sriov_err(gt, "Failed to save VF%u MMIO data (%pe)\n", vfid, ERR_PTR(ret)); + return ret; +} + +static int pf_restore_vf_mmio_mig_data(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + int ret; + + pf_dump_mig_data(gt, vfid, data, "MMIO data restore"); + + ret = pf_migration_mmio_restore(gt, vfid, data->vaddr, data->hdr.size); + if (ret) { + xe_gt_sriov_err(gt, "Failed to restore VF%u MMIO data (%pe)\n", + vfid, ERR_PTR(ret)); + + return ret; + } + + return 0; +} + +/** + * xe_gt_sriov_pf_migration_mmio_save() - Save VF MMIO migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return pf_save_vf_mmio_mig_data(gt, vfid); +} + +/** + * xe_gt_sriov_pf_migration_mmio_restore() - Restore VF MMIO migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * @data: the &xe_sriov_packet containing migration data + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return pf_restore_vf_mmio_mig_data(gt, vfid, data); +} + +static ssize_t pf_migration_vram_size(struct xe_gt *gt, unsigned int vfid) +{ + if (!xe_gt_is_main_type(gt)) + return 0; + + return xe_gt_sriov_pf_config_get_lmem(gt, vfid); +} + +static struct dma_fence *__pf_save_restore_vram(struct xe_gt *gt, unsigned int vfid, + struct xe_bo *vram, u64 vram_offset, + struct xe_bo *sysmem, u64 sysmem_offset, + size_t size, bool save) +{ + struct dma_fence *ret = NULL; + struct drm_exec exec; + int err; + + drm_exec_init(&exec, 0, 0); + drm_exec_until_all_locked(&exec) { + err = drm_exec_lock_obj(&exec, &vram->ttm.base); + drm_exec_retry_on_contention(&exec); + if (err) { + ret = ERR_PTR(err); + goto err; + } + + err = drm_exec_lock_obj(&exec, &sysmem->ttm.base); + drm_exec_retry_on_contention(&exec); + if (err) { + ret = ERR_PTR(err); + goto err; + } + } + + ret = xe_migrate_vram_copy_chunk(vram, vram_offset, sysmem, sysmem_offset, size, + save ? XE_MIGRATE_COPY_TO_SRAM : XE_MIGRATE_COPY_TO_VRAM); + +err: + drm_exec_fini(&exec); + + return ret; +} + +#define PF_VRAM_SAVE_RESTORE_TIMEOUT (5 * HZ) +static int pf_save_vram_chunk(struct xe_gt *gt, unsigned int vfid, + struct xe_bo *src_vram, u64 src_vram_offset, + size_t size) +{ + struct xe_sriov_packet *data; + struct dma_fence *fence; + int ret; + + data = xe_sriov_packet_alloc(gt_to_xe(gt)); + if (!data) + return -ENOMEM; + + ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, + XE_SRIOV_PACKET_TYPE_VRAM, src_vram_offset, + size); + if (ret) + goto fail; + + fence = __pf_save_restore_vram(gt, vfid, + src_vram, src_vram_offset, + data->bo, 0, size, true); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto fail; + } + + ret = dma_fence_wait_timeout(fence, false, PF_VRAM_SAVE_RESTORE_TIMEOUT); + dma_fence_put(fence); + if (!ret) { + ret = -ETIME; + goto fail; + } + + pf_dump_mig_data(gt, vfid, data, "VRAM data save"); + + ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); + if (ret) + goto fail; + + return 0; + +fail: + xe_sriov_packet_free(data); + return ret; +} + +#define VF_VRAM_STATE_CHUNK_MAX_SIZE SZ_512M +static int pf_save_vf_vram_mig_data(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); + loff_t *offset = &migration->save.vram_offset; + struct xe_bo *vram; + size_t vram_size, chunk_size; + int ret; + + vram = xe_gt_sriov_pf_config_get_lmem_obj(gt, vfid); + if (!vram) + return -ENXIO; + + vram_size = xe_bo_size(vram); + + xe_gt_assert(gt, *offset < vram_size); + + chunk_size = min(vram_size - *offset, VF_VRAM_STATE_CHUNK_MAX_SIZE); + + ret = pf_save_vram_chunk(gt, vfid, vram, *offset, chunk_size); + if (ret) + goto fail; + + *offset += chunk_size; + + xe_bo_put(vram); + + if (*offset < vram_size) + return -EAGAIN; + + return 0; + +fail: + xe_bo_put(vram); + xe_gt_sriov_err(gt, "Failed to save VF%u VRAM data (%pe)\n", vfid, ERR_PTR(ret)); + return ret; +} + +static int pf_restore_vf_vram_mig_data(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + u64 end = data->hdr.offset + data->hdr.size; + struct dma_fence *fence; + struct xe_bo *vram; + size_t size; + int ret = 0; + + vram = xe_gt_sriov_pf_config_get_lmem_obj(gt, vfid); + if (!vram) + return -ENXIO; + + size = xe_bo_size(vram); + + if (end > size || end < data->hdr.size) { + ret = -EINVAL; + goto err; + } + + pf_dump_mig_data(gt, vfid, data, "VRAM data restore"); + + fence = __pf_save_restore_vram(gt, vfid, vram, data->hdr.offset, + data->bo, 0, data->hdr.size, false); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto err; + } + + ret = dma_fence_wait_timeout(fence, false, PF_VRAM_SAVE_RESTORE_TIMEOUT); + dma_fence_put(fence); + if (!ret) { + ret = -ETIME; + goto err; + } + + xe_bo_put(vram); + + return 0; +err: + xe_bo_put(vram); + xe_gt_sriov_err(gt, "Failed to restore VF%u VRAM data (%pe)\n", vfid, ERR_PTR(ret)); + return ret; +} + +/** + * xe_gt_sriov_pf_migration_vram_save() - Save VF VRAM migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_vram_save(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return pf_save_vf_vram_mig_data(gt, vfid); +} + +/** + * xe_gt_sriov_pf_migration_vram_restore() - Restore VF VRAM migration data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * @data: the &xe_sriov_packet containing migration data + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_vram_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + return pf_restore_vf_vram_mig_data(gt, vfid, data); +} + +/** + * xe_gt_sriov_pf_migration_size() - Total size of migration data from all components within a GT. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * + * This function is for PF only. + * + * Return: total migration data size in bytes or a negative error code on failure. + */ +ssize_t xe_gt_sriov_pf_migration_size(struct xe_gt *gt, unsigned int vfid) +{ + ssize_t total = 0; + ssize_t size; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid != PFID); + xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); + + size = pf_migration_guc_size(gt, vfid); + if (size < 0) + return size; + if (size > 0) + size += sizeof(struct xe_sriov_packet_hdr); + total += size; + + size = pf_migration_ggtt_size(gt, vfid); + if (size < 0) + return size; + if (size > 0) + size += sizeof(struct xe_sriov_packet_hdr); + total += size; + + size = pf_migration_mmio_size(gt, vfid); + if (size < 0) + return size; + if (size > 0) + size += sizeof(struct xe_sriov_packet_hdr); + total += size; + + size = pf_migration_vram_size(gt, vfid); + if (size < 0) + return size; + if (size > 0) + size += sizeof(struct xe_sriov_packet_hdr); + total += size; + + return total; +} + +/** + * xe_gt_sriov_pf_migration_ring_empty() - Check if a migration ring is empty. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * Return: true if the ring is empty, otherwise false. + */ +bool xe_gt_sriov_pf_migration_ring_empty(struct xe_gt *gt, unsigned int vfid) +{ + return ptr_ring_empty(&pf_pick_gt_migration(gt, vfid)->ring); +} + +/** + * xe_gt_sriov_pf_migration_ring_full() - Check if a migration ring is full. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * Return: true if the ring is full, otherwise false. + */ +bool xe_gt_sriov_pf_migration_ring_full(struct xe_gt *gt, unsigned int vfid) +{ + return ptr_ring_full(&pf_pick_gt_migration(gt, vfid)->ring); +} + +/** + * xe_gt_sriov_pf_migration_ring_free() - Consume and free all data in migration ring + * @gt: the &xe_gt + * @vfid: the VF identifier + */ +void xe_gt_sriov_pf_migration_ring_free(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); + struct xe_sriov_packet *data; + + if (ptr_ring_empty(&migration->ring)) + return; + + xe_gt_sriov_notice(gt, "VF%u unprocessed migration data left in the ring!\n", vfid); + + while ((data = ptr_ring_consume(&migration->ring))) + xe_sriov_packet_free(data); +} + +static void pf_migration_save_data_todo(struct xe_gt *gt, unsigned int vfid, + enum xe_sriov_packet_type type) +{ + set_bit(type, &pf_pick_gt_migration(gt, vfid)->save.data_remaining); +} + +/** + * xe_gt_sriov_pf_migration_save_init() - Initialize per-GT migration related data. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + */ +void xe_gt_sriov_pf_migration_save_init(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); + + migration->save.data_remaining = 0; + migration->save.vram_offset = 0; + + xe_gt_assert(gt, pf_migration_guc_size(gt, vfid) > 0); + pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_GUC); + + if (pf_migration_ggtt_size(gt, vfid) > 0) + pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_GGTT); + + xe_gt_assert(gt, pf_migration_mmio_size(gt, vfid) > 0); + pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_MMIO); + + if (pf_migration_vram_size(gt, vfid) > 0) + pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_VRAM); +} + +/** + * xe_gt_sriov_pf_migration_save_data_pending() - Check if migration data type needs to be saved. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * @type: the &xe_sriov_packet_type of data to be checked + * + * Return: true if the data needs saving, otherwise false. + */ +bool xe_gt_sriov_pf_migration_save_data_pending(struct xe_gt *gt, unsigned int vfid, + enum xe_sriov_packet_type type) +{ + return test_bit(type, &pf_pick_gt_migration(gt, vfid)->save.data_remaining); +} + +/** + * xe_gt_sriov_pf_migration_save_data_complete() - Complete migration data type save. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be 0) + * @type: the &xe_sriov_packet_type to be marked as completed. + */ +void xe_gt_sriov_pf_migration_save_data_complete(struct xe_gt *gt, unsigned int vfid, + enum xe_sriov_packet_type type) +{ + clear_bit(type, &pf_pick_gt_migration(gt, vfid)->save.data_remaining); +} + +/** + * xe_gt_sriov_pf_migration_save_produce() - Add VF save data packet to migration ring. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @data: the &xe_sriov_packet + * + * Called by the save migration data producer (PF SR-IOV Control worker) when + * processing migration data. + * Wakes up the save migration data consumer (userspace), that is potentially + * waiting for data when the ring was empty. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_save_produce(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + int ret; + + ret = ptr_ring_produce(&pf_pick_gt_migration(gt, vfid)->ring, data); + if (ret) + return ret; + + wake_up_all(xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid)); + + return 0; +} + +/** + * xe_gt_sriov_pf_migration_restore_consume() - Get VF restore data packet from migration ring. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * Called by the restore migration data consumer (PF SR-IOV Control worker) when + * processing migration data. + * Wakes up the restore migration data producer (userspace), that is + * potentially waiting to add more data when the ring is full. + * + * Return: Pointer to &xe_sriov_packet on success, + * NULL if ring is empty. + */ +struct xe_sriov_packet * +xe_gt_sriov_pf_migration_restore_consume(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); + struct wait_queue_head *wq = xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid); + struct xe_sriov_packet *data; + + data = ptr_ring_consume(&migration->ring); + if (data) + wake_up_all(wq); + + return data; +} + +static bool pf_restore_data_ready(struct xe_gt *gt, unsigned int vfid) +{ + if (xe_gt_sriov_pf_control_check_restore_failed(gt, vfid) || + !ptr_ring_full(&pf_pick_gt_migration(gt, vfid)->ring)) + return true; + + return false; +} + +/** + * xe_gt_sriov_pf_migration_restore_produce() - Add VF restore data packet to migration ring. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @data: the &xe_sriov_packet + * + * Called by the restore migration data producer (userspace) when processing + * migration data. + * If the ring is full, waits until there is space. + * Queues the restore migration data consumer (PF SR-IOV Control worker), that + * is potentially waiting for data when the ring was empty. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_restore_produce(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data) +{ + int ret; + + xe_gt_assert(gt, data->hdr.tile_id == gt->tile->id); + xe_gt_assert(gt, data->hdr.gt_id == gt->info.id); + + for (;;) { + if (xe_gt_sriov_pf_control_check_restore_failed(gt, vfid)) + return -EIO; + + ret = ptr_ring_produce(&pf_pick_gt_migration(gt, vfid)->ring, data); + if (!ret) + break; + + ret = wait_event_interruptible(*xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid), + pf_restore_data_ready(gt, vfid)); + if (ret) + return ret; + } + + return xe_gt_sriov_pf_control_process_restore_data(gt, vfid); +} + +/** + * xe_gt_sriov_pf_migration_save_consume() - Get VF save data packet from migration ring. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * Called by the save migration data consumer (userspace) when + * processing migration data. + * Queues the save migration data producer (PF SR-IOV Control worker), that is + * potentially waiting to add more data when the ring is full. + * + * Return: Pointer to &xe_sriov_packet on success, + * NULL if ring is empty and there's no more data available, + * ERR_PTR(-EAGAIN) if the ring is empty, but data is still produced. + */ +struct xe_sriov_packet * +xe_gt_sriov_pf_migration_save_consume(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); + struct xe_sriov_packet *data; + int ret; + + data = ptr_ring_consume(&migration->ring); + if (data) { + ret = xe_gt_sriov_pf_control_process_save_data(gt, vfid); + if (ret) { + xe_sriov_packet_free(data); + return ERR_PTR(ret); + } + + return data; + } + + if (xe_gt_sriov_pf_control_check_save_data_done(gt, vfid)) + return NULL; + + if (xe_gt_sriov_pf_control_check_save_failed(gt, vfid)) + return ERR_PTR(-EIO); + + return ERR_PTR(-EAGAIN); +} + +static void destroy_pf_packet(void *ptr) +{ + struct xe_sriov_packet *data = ptr; + + xe_sriov_packet_free(data); +} + +static void action_ring_cleanup(void *arg) +{ + struct ptr_ring *r = arg; + + ptr_ring_cleanup(r, destroy_pf_packet); +} + +static void pf_gt_migration_check_support(struct xe_gt *gt) +{ + if (!GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 54)) + xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0"); +} + +/** + * xe_gt_sriov_pf_migration_init() - Initialize support for VF migration. + * @gt: the &xe_gt + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_migration_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int n, totalvfs; + int err; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + pf_gt_migration_check_support(gt); + + if (!pf_migration_supported(gt)) + return 0; + + totalvfs = xe_sriov_pf_get_totalvfs(xe); + for (n = 1; n <= totalvfs; n++) { + struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, n); + + err = ptr_ring_init(&migration->ring, + XE_GT_SRIOV_PF_MIGRATION_RING_SIZE, GFP_KERNEL); + if (err) + return err; + + err = devm_add_action_or_reset(xe->drm.dev, action_ring_cleanup, &migration->ring); + if (err) + return err; + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h new file mode 100644 index 000000000..181207a63 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_MIGRATION_H_ +#define _XE_GT_SRIOV_PF_MIGRATION_H_ + +#include + +struct xe_gt; +struct xe_sriov_packet; +enum xe_sriov_packet_type; + +/* TODO: get this information by querying GuC in the future */ +#define XE_GT_SRIOV_PF_MIGRATION_GUC_DATA_MAX_SIZE SZ_8M + +int xe_gt_sriov_pf_migration_init(struct xe_gt *gt); +int xe_gt_sriov_pf_migration_guc_save(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_migration_guc_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data); +int xe_gt_sriov_pf_migration_ggtt_save(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_migration_ggtt_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data); +int xe_gt_sriov_pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data); +int xe_gt_sriov_pf_migration_vram_save(struct xe_gt *gt, unsigned int vfid); +int xe_gt_sriov_pf_migration_vram_restore(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data); + +ssize_t xe_gt_sriov_pf_migration_size(struct xe_gt *gt, unsigned int vfid); + +bool xe_gt_sriov_pf_migration_ring_empty(struct xe_gt *gt, unsigned int vfid); +bool xe_gt_sriov_pf_migration_ring_full(struct xe_gt *gt, unsigned int vfid); +void xe_gt_sriov_pf_migration_ring_free(struct xe_gt *gt, unsigned int vfid); + +void xe_gt_sriov_pf_migration_save_init(struct xe_gt *gt, unsigned int vfid); +bool xe_gt_sriov_pf_migration_save_data_pending(struct xe_gt *gt, unsigned int vfid, + enum xe_sriov_packet_type type); +void xe_gt_sriov_pf_migration_save_data_complete(struct xe_gt *gt, unsigned int vfid, + enum xe_sriov_packet_type type); + +int xe_gt_sriov_pf_migration_save_produce(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data); +struct xe_sriov_packet * +xe_gt_sriov_pf_migration_restore_consume(struct xe_gt *gt, unsigned int vfid); + +int xe_gt_sriov_pf_migration_restore_produce(struct xe_gt *gt, unsigned int vfid, + struct xe_sriov_packet *data); +struct xe_sriov_packet * +xe_gt_sriov_pf_migration_save_consume(struct xe_gt *gt, unsigned int vfid); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration_types.h new file mode 100644 index 000000000..f50c64241 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration_types.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_MIGRATION_TYPES_H_ +#define _XE_GT_SRIOV_PF_MIGRATION_TYPES_H_ + +#include + +/** + * struct xe_gt_sriov_migration_data - GT-level per-VF migration data. + * + * Used by the PF driver to maintain per-VF migration data. + */ +struct xe_gt_sriov_migration_data { + /** @ring: queue containing VF save / restore migration data */ + struct ptr_ring ring; + /** @save: structure for currently processed save migration data */ + struct { + /** @save.data_remaining: bitmap of migration types that need to be saved */ + unsigned long data_remaining; + /** @save.vram_offset: last saved offset within VRAM, used for chunked VRAM save */ + loff_t vram_offset; + } save; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c new file mode 100644 index 000000000..a85ba4435 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include "abi/guc_actions_sriov_abi.h" +#include "abi/guc_messages_abi.h" + +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_monitor.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_klv_thresholds_set.h" + +/** + * xe_gt_sriov_pf_monitor_flr - Cleanup VF data after VF FLR. + * @gt: the &xe_gt + * @vfid: the VF identifier + * + * On FLR this function will reset all event data related to the VF. + * This function is for PF only. + */ +void xe_gt_sriov_pf_monitor_flr(struct xe_gt *gt, u32 vfid) +{ + int e; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_sriov_pf_assert_vfid(gt, vfid); + + for (e = 0; e < XE_GUC_KLV_NUM_THRESHOLDS; e++) + gt->sriov.pf.vfs[vfid].monitor.guc.events[e] = 0; +} + +static void pf_update_event_counter(struct xe_gt *gt, u32 vfid, + enum xe_guc_klv_threshold_index e) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, e < XE_GUC_KLV_NUM_THRESHOLDS); + + gt->sriov.pf.vfs[vfid].monitor.guc.events[e]++; +} + +static int pf_handle_vf_threshold_event(struct xe_gt *gt, u32 vfid, u32 threshold) +{ + char origin[8]; + int e; + + e = xe_guc_klv_threshold_key_to_index(threshold); + xe_sriov_function_name(vfid, origin, sizeof(origin)); + + /* was there a new KEY added that we missed? */ + if (unlikely(e < 0)) { + xe_gt_sriov_notice(gt, "unknown threshold key %#x reported for %s\n", + threshold, origin); + return -ENOTCONN; + } + + xe_gt_sriov_dbg(gt, "%s exceeded threshold %u %s\n", + origin, xe_gt_sriov_pf_config_get_threshold(gt, vfid, e), + xe_guc_klv_key_to_string(threshold)); + + pf_update_event_counter(gt, vfid, e); + + return 0; +} + +/** + * xe_gt_sriov_pf_monitor_process_guc2pf - Handle adverse event notification from the GuC. + * @gt: the &xe_gt + * @msg: G2H event message + * @len: length of the message + * + * This function is intended for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 vfid; + u32 threshold; + + xe_gt_assert(gt, len >= GUC_HXG_MSG_MIN_LEN); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) == + GUC_ACTION_GUC2PF_ADVERSE_EVENT); + + if (unlikely(!IS_SRIOV_PF(xe))) + return -EPROTO; + + if (unlikely(FIELD_GET(GUC2PF_ADVERSE_EVENT_EVENT_MSG_0_MBZ, msg[0]))) + return -EPFNOSUPPORT; + + if (unlikely(len < GUC2PF_ADVERSE_EVENT_EVENT_MSG_LEN)) + return -EPROTO; + + vfid = FIELD_GET(GUC2PF_ADVERSE_EVENT_EVENT_MSG_1_VFID, msg[1]); + threshold = FIELD_GET(GUC2PF_ADVERSE_EVENT_EVENT_MSG_2_THRESHOLD, msg[2]); + + if (unlikely(vfid > xe_gt_sriov_pf_get_totalvfs(gt))) + return -EINVAL; + + return pf_handle_vf_threshold_event(gt, vfid, threshold); +} + +/** + * xe_gt_sriov_pf_monitor_print_events - Print adverse events counters. + * @gt: the &xe_gt to print events from + * @p: the &drm_printer + * + * Print adverse events counters for all VFs. + * VFs with no events are not printed. + * + * This function can only be called on PF. + * + * Return: always 0 + */ +int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int n, total_vfs = xe_gt_sriov_pf_get_totalvfs(gt); + const struct xe_gt_sriov_monitor *data; + int e; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + for (n = 1; n <= total_vfs; n++) { + data = >->sriov.pf.vfs[n].monitor; + + for (e = 0; e < XE_GUC_KLV_NUM_THRESHOLDS; e++) + if (data->guc.events[e]) + break; + + /* skip empty unless in debug mode */ + if (e >= XE_GUC_KLV_NUM_THRESHOLDS && + !IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) + continue; + +#define __format(...) "%s:%u " +#define __value(TAG, NAME, ...) , #NAME, data->guc.events[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)] + + drm_printf(p, "VF%u:\t" MAKE_XE_GUC_KLV_THRESHOLDS_SET(__format) "\n", + n MAKE_XE_GUC_KLV_THRESHOLDS_SET(__value)); + +#undef __format +#undef __value + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h new file mode 100644 index 000000000..0b8f088d3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_MONITOR_H_ +#define _XE_GT_SRIOV_PF_MONITOR_H_ + +#include +#include + +struct drm_printer; +struct xe_gt; + +void xe_gt_sriov_pf_monitor_flr(struct xe_gt *gt, u32 vfid); +int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p); + +#ifdef CONFIG_PCI_IOV +int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len); +#else +static inline int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len) +{ + return -EPROTO; +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor_types.h new file mode 100644 index 000000000..e27c0308c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor_types.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_MONITOR_TYPES_H_ +#define _XE_GT_SRIOV_PF_MONITOR_TYPES_H_ + +#include "xe_guc_klv_thresholds_set_types.h" + +/** + * struct xe_gt_sriov_monitor - GT level per-VF monitoring data. + */ +struct xe_gt_sriov_monitor { + /** @guc: monitoring data related to the GuC. */ + struct { + /** @guc.events: number of adverse events reported by the GuC. */ + unsigned int events[XE_GUC_KLV_NUM_THRESHOLDS]; + } guc; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c new file mode 100644 index 000000000..cdfe19492 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c @@ -0,0 +1,796 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include + +#include "abi/guc_actions_sriov_abi.h" + +#include "xe_gt.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_policy.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc.h" +#include "xe_guc_buf.h" +#include "xe_guc_ct.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_submit.h" + +/* + * Return: number of KLVs that were successfully parsed and saved, + * negative error code on failure. + */ +static int guc_action_update_vgt_policy(struct xe_guc *guc, u64 addr, u32 size) +{ + u32 request[] = { + GUC_ACTION_PF2GUC_UPDATE_VGT_POLICY, + lower_32_bits(addr), + upper_32_bits(addr), + size, + }; + + return xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request)); +} + +/* + * Return: number of KLVs that were successfully parsed and saved, + * negative error code on failure. + */ +static int pf_send_policy_klvs(struct xe_gt *gt, struct xe_guc_buf buf, u32 num_dwords) +{ + struct xe_guc *guc = >->uc.guc; + + return guc_action_update_vgt_policy(guc, xe_guc_buf_flush(buf), num_dwords); +} + +/* + * Return: 0 on success, -ENOKEY if some KLVs were not updated, -EPROTO if reply was malformed, + * negative error code on failure. + */ +static int pf_push_policy_buf_klvs(struct xe_gt *gt, u32 num_klvs, + struct xe_guc_buf buf, u32 num_dwords) +{ + int ret; + + ret = pf_send_policy_klvs(gt, buf, num_dwords); + + if (ret != num_klvs) { + int err = ret < 0 ? ret : ret < num_klvs ? -ENOKEY : -EPROTO; + struct drm_printer p = xe_gt_info_printer(gt); + void *klvs = xe_guc_buf_cpu_ptr(buf); + + xe_gt_sriov_notice(gt, "Failed to push %u policy KLV%s (%pe)\n", + num_klvs, str_plural(num_klvs), ERR_PTR(err)); + xe_guc_klv_print(klvs, num_dwords, &p); + return err; + } + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + void *klvs = xe_guc_buf_cpu_ptr(buf); + + xe_gt_sriov_dbg(gt, "pushed policy update with %u KLV%s:\n", + num_klvs, str_plural(num_klvs)); + xe_guc_klv_print(klvs, num_dwords, &p); + } + + return 0; +} + +/* + * Return: 0 on success, -ENOBUFS if there is no free buffer for the indirect data, + * negative error code on failure. + */ +static int pf_push_policy_klvs(struct xe_gt *gt, u32 num_klvs, + const u32 *klvs, u32 num_dwords) +{ + CLASS(xe_guc_buf_from_data, buf)(>->uc.guc.buf, klvs, num_dwords * sizeof(u32)); + + xe_gt_assert(gt, num_klvs == xe_guc_klv_count(klvs, num_dwords)); + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + return pf_push_policy_buf_klvs(gt, num_klvs, buf, num_dwords); +} + +static int pf_push_policy_u32(struct xe_gt *gt, u16 key, u32 value) +{ + u32 klv[] = { + PREP_GUC_KLV(key, 1), + value, + }; + + return pf_push_policy_klvs(gt, 1, klv, ARRAY_SIZE(klv)); +} + +static int pf_push_policy_payload(struct xe_gt *gt, u16 key, void *payload, u32 num_dwords) +{ + CLASS(xe_guc_buf, buf)(>->uc.guc.buf, GUC_KLV_LEN_MIN + num_dwords); + u32 *klv; + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + klv = xe_guc_buf_cpu_ptr(buf); + + klv[0] = PREP_GUC_KLV(key, num_dwords); + if (num_dwords) + memcpy(&klv[1], payload, num_dwords * sizeof(u32)); + + return pf_push_policy_buf_klvs(gt, 1, buf, GUC_KLV_LEN_MIN + num_dwords); +} + +static int pf_update_policy_bool(struct xe_gt *gt, u16 key, bool *policy, bool value) +{ + int err; + + err = pf_push_policy_u32(gt, key, value); + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "Failed to update policy %#x '%s' to '%s' (%pe)\n", + key, xe_guc_klv_key_to_string(key), + str_enabled_disabled(value), ERR_PTR(err)); + return err; + } + + xe_gt_sriov_dbg(gt, "policy key %#x '%s' updated to '%s'\n", + key, xe_guc_klv_key_to_string(key), + str_enabled_disabled(value)); + + *policy = value; + return 0; +} + +static int pf_update_policy_u32(struct xe_gt *gt, u16 key, u32 *policy, u32 value) +{ + int err; + + err = pf_push_policy_u32(gt, key, value); + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "Failed to update policy %#x '%s' to '%s' (%pe)\n", + key, xe_guc_klv_key_to_string(key), + str_enabled_disabled(value), ERR_PTR(err)); + return err; + } + + xe_gt_sriov_dbg(gt, "policy key %#x '%s' updated to %u\n", + key, xe_guc_klv_key_to_string(key), value); + + *policy = value; + return 0; +} + +static int pf_provision_sched_if_idle(struct xe_gt *gt, bool enable) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY, + >->sriov.pf.policy.guc.sched_if_idle, + enable); +} + +static int pf_reprovision_sched_if_idle(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (!gt->sriov.pf.policy.guc.sched_if_idle) + return 0; + + return pf_provision_sched_if_idle(gt, gt->sriov.pf.policy.guc.sched_if_idle); +} + +static void pf_sanitize_sched_if_idle(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + gt->sriov.pf.policy.guc.sched_if_idle = false; +} + +/** + * xe_gt_sriov_pf_policy_set_sched_if_idle_locked() - Control the 'sched_if_idle' policy. + * @gt: the &xe_gt where to apply the policy + * @enable: the value of the 'sched_if_idle' policy + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable) +{ + u32 priority; + int err; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_provision_sched_if_idle(gt, enable); + if (err) + return err; + + /* + * As of GuC 70.12 a change of this policy impacts individual configs + * of all VFs. See `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ for details. + */ + xe_gt_assert(gt, GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 12)); + + priority = enable ? GUC_SCHED_PRIORITY_NORMAL : GUC_SCHED_PRIORITY_LOW; + xe_gt_sriov_pf_config_force_sched_priority_locked(gt, priority); + + return 0; +} + +/** + * xe_gt_sriov_pf_policy_set_sched_if_idle() - Control the 'sched_if_idle' policy. + * @gt: the &xe_gt where to apply the policy + * @enable: the value of the 'sched_if_idle' policy + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, enable); +} + +/** + * xe_gt_sriov_pf_policy_get_sched_if_idle_locked() - Retrieve value of 'sched_if_idle' policy. + * @gt: the &xe_gt where to read the policy from + * + * This function can only be called on PF. + * + * Return: last value of 'sched_if_idle' policy applied. + */ +bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return gt->sriov.pf.policy.guc.sched_if_idle; +} + +/** + * xe_gt_sriov_pf_policy_get_sched_if_idle - Retrieve value of 'sched_if_idle' policy. + * @gt: the &xe_gt where to read the policy from + * + * This function can only be called on PF. + * + * Return: value of 'sched_if_idle' policy. + */ +bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt); +} + +static int pf_provision_reset_engine(struct xe_gt *gt, bool enable) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY, + >->sriov.pf.policy.guc.reset_engine, enable); +} + +static int pf_reprovision_reset_engine(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (!gt->sriov.pf.policy.guc.reset_engine) + return 0; + + return pf_provision_reset_engine(gt, gt->sriov.pf.policy.guc.reset_engine); +} + +static void pf_sanitize_reset_engine(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + gt->sriov.pf.policy.guc.reset_engine = false; +} + +/** + * xe_gt_sriov_pf_policy_set_reset_engine - Control the 'reset_engine' policy. + * @gt: the &xe_gt where to apply the policy + * @enable: the value of the 'reset_engine' policy + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable) +{ + int err; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_provision_reset_engine(gt, enable); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return err; +} + +/** + * xe_gt_sriov_pf_policy_get_reset_engine - Retrieve value of 'reset_engine' policy. + * @gt: the &xe_gt where to read the policy from + * + * This function can only be called on PF. + * + * Return: value of 'reset_engine' policy. + */ +bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt) +{ + bool enable; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + enable = gt->sriov.pf.policy.guc.reset_engine; + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return enable; +} + +static int pf_provision_sample_period(struct xe_gt *gt, u32 value) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return pf_update_policy_u32(gt, GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY, + >->sriov.pf.policy.guc.sample_period, value); +} + +static int pf_reprovision_sample_period(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (!gt->sriov.pf.policy.guc.sample_period) + return 0; + + return pf_provision_sample_period(gt, gt->sriov.pf.policy.guc.sample_period); +} + +static void pf_sanitize_sample_period(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + gt->sriov.pf.policy.guc.sample_period = 0; +} + +/** + * xe_gt_sriov_pf_policy_set_sample_period - Control the 'sample_period' policy. + * @gt: the &xe_gt where to apply the policy + * @value: the value of the 'sample_period' policy + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value) +{ + int err; + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_provision_sample_period(gt, value); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return err; +} + +/** + * xe_gt_sriov_pf_policy_get_sample_period - Retrieve value of 'sample_period' policy. + * @gt: the &xe_gt where to read the policy from + * + * This function can only be called on PF. + * + * Return: value of 'sample_period' policy. + */ +u32 xe_gt_sriov_pf_policy_get_sample_period(struct xe_gt *gt) +{ + u32 value; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + value = gt->sriov.pf.policy.guc.sample_period; + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return value; +} + +static void pf_sched_group_media_slices(struct xe_gt *gt, struct guc_sched_group **groups, + u32 *num_groups) +{ + u8 slice_to_group[MAX_MEDIA_SLICES]; + u32 vecs_mask = VECS_INSTANCES(gt); + u32 gsc_mask = GSCCS_INSTANCES(gt); + u32 vcs_mask = VCS_INSTANCES(gt); + struct guc_sched_group *values; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int group = 0; + int slice; + + xe_gt_assert(gt, xe_gt_is_media_type(gt)); + + /* + * Post-BMG the matching of video engines to slices changes, so for now + * we don't allow this mode on those platforms. + */ + if (gt_to_xe(gt)->info.platform > XE_BATTLEMAGE) + return; + + /* + * On BMG and older platforms a media slice has 2 VCS and a VECS. We + * bundle the GSC with the first slice. + */ + for (slice = 0; slice < MAX_MEDIA_SLICES; slice++) { + if ((vcs_mask & 0x3) || (vecs_mask & 0x1) || (gsc_mask & 0x1)) + slice_to_group[slice] = group++; + + vcs_mask >>= 2; + vecs_mask >>= 1; + gsc_mask >>= 1; + } + + xe_gt_assert(gt, !vcs_mask); + xe_gt_assert(gt, !vecs_mask); + xe_gt_assert(gt, !gsc_mask); + + /* We need at least 2 slices to split them up */ + if (group < 2) + return; + + /* + * If we have more groups than the GuC can support then we don't want to + * expose this specific mode, because the GuC will return an error if we + * try to enable it. + */ + if (group > gt->sriov.pf.policy.guc.sched_groups.max_groups) { + xe_gt_sriov_notice(gt, "media_slice mode has too many groups: %u vs %u\n", + group, gt->sriov.pf.policy.guc.sched_groups.max_groups); + return; + } + + /* The GuC expects an array with a guc_sched_group entry for each group */ + values = drmm_kcalloc(>_to_xe(gt)->drm, group, sizeof(struct guc_sched_group), + GFP_KERNEL); + if (!values) + return; + + for_each_hw_engine(hwe, gt, id) { + u8 guc_class = xe_hwe_to_guc_class(hwe); + u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); + + switch (hwe->class) { + case XE_ENGINE_CLASS_VIDEO_DECODE: + slice = hwe->instance / 2; + break; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + slice = hwe->instance; + break; + case XE_ENGINE_CLASS_OTHER: + slice = 0; + break; + default: + xe_gt_assert_msg(gt, false, + "unknown media gt class %u (%s) during EGS setup\n", + hwe->class, hwe->name); + slice = 0; + } + + values[slice_to_group[slice]].engines[guc_class] |= BIT(guc_logical_instance); + } + + *groups = values; + *num_groups = group; +} + +/** + * xe_sriov_gt_pf_policy_has_sched_groups_support() - Checks whether scheduler + * groups are supported. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: true if scheduler groups are supported, false otherwise. + */ +bool xe_sriov_gt_pf_policy_has_sched_groups_support(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + /* + * The GuC supports scheduler groups from v70.53.0, but a fix for it has + * been merged in v70.55.1, so we require the latter. The feature is + * also only enabled on BMG and newer FW. + */ + return GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 55, 1) && + gt_to_xe(gt)->info.platform >= XE_BATTLEMAGE; +} + +static void pf_init_sched_groups(struct xe_gt *gt) +{ + enum xe_sriov_sched_group_modes m; + + if (!xe_sriov_gt_pf_policy_has_sched_groups_support(gt)) + return; + + /* + * The GuC interface supports up to 8 groups. However, the GuC only + * fully allocates resources for a subset of groups, based on the number + * of engines and expected usage. The plan is for this to become + * queryable via H2G, but for now GuC FW for all devices supports a + * maximum of 2 groups so we can just hardcode that. + */ + gt->sriov.pf.policy.guc.sched_groups.max_groups = 2; + + for (m = XE_SRIOV_SCHED_GROUPS_DISABLED + 1; m < XE_SRIOV_SCHED_GROUPS_MODES_COUNT; m++) { + u32 *num_groups = >->sriov.pf.policy.guc.sched_groups.modes[m].num_groups; + struct guc_sched_group **groups = + >->sriov.pf.policy.guc.sched_groups.modes[m].groups; + + switch (m) { + case XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES: + /* this mode only has groups on the media GT */ + if (xe_gt_is_media_type(gt)) + pf_sched_group_media_slices(gt, groups, num_groups); + break; + case XE_SRIOV_SCHED_GROUPS_DISABLED: + case XE_SRIOV_SCHED_GROUPS_MODES_COUNT: + /* + * By defining m of type enum xe_sriov_sched_group_modes + * we can get the compiler to automatically flag + * missing cases if new enum entries are added. However, + * to keep the compiler happy we also need to add the + * cases that are excluded from the loop. + */ + xe_gt_assert(gt, false); + break; + } + + xe_gt_assert(gt, *num_groups < GUC_MAX_SCHED_GROUPS); + + if (*num_groups) + gt->sriov.pf.policy.guc.sched_groups.supported_modes |= BIT(m); + } +} + +/** + * xe_sriov_gt_pf_policy_has_multi_group_modes() - check whether the GT supports + * any scheduler modes that have multiple groups + * @gt: the &xe_gt to check + * + * This function can only be called on PF. + * + * Return: true if the GT supports modes with multiple groups, false otherwise. + */ +bool xe_sriov_gt_pf_policy_has_multi_group_modes(struct xe_gt *gt) +{ + return gt->sriov.pf.policy.guc.sched_groups.supported_modes; +} + +/** + * xe_sriov_gt_pf_policy_has_sched_group_mode() - check whether the GT supports + * a specific scheduler group mode + * @gt: the &xe_gt to check + * @mode: the mode to check + * + * This function can only be called on PF. + * + * Return: true if the GT supports the specified mode, false otherwise. + */ +bool xe_sriov_gt_pf_policy_has_sched_group_mode(struct xe_gt *gt, + enum xe_sriov_sched_group_modes mode) +{ + if (mode == XE_SRIOV_SCHED_GROUPS_DISABLED) + return true; + + return gt->sriov.pf.policy.guc.sched_groups.supported_modes & BIT(mode); +} + +static int __pf_provision_sched_groups(struct xe_gt *gt, enum xe_sriov_sched_group_modes mode) +{ + struct guc_sched_group *groups = gt->sriov.pf.policy.guc.sched_groups.modes[mode].groups; + u32 num_groups = gt->sriov.pf.policy.guc.sched_groups.modes[mode].num_groups; + + return pf_push_policy_payload(gt, GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_KEY, + groups, num_groups * GUC_MAX_ENGINE_CLASSES); +} + +static int pf_provision_sched_groups(struct xe_gt *gt, enum xe_sriov_sched_group_modes mode) +{ + int err; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + if (!xe_sriov_gt_pf_policy_has_sched_group_mode(gt, mode)) + return -EINVAL; + + /* already in the desired mode */ + if (gt->sriov.pf.policy.guc.sched_groups.current_mode == mode) + return 0; + + /* + * We don't allow changing this with VFs active since it is hard for + * VFs to check. + */ + if (xe_sriov_pf_num_vfs(gt_to_xe(gt))) + return -EBUSY; + + /* + * The GuC silently ignores the setting if any MLRC contexts are + * registered. We expect the admin to make sure that all apps that use + * MLRC are terminated before scheduler groups are enabled, so this + * check is just to make sure that the exec_queue destruction has been + * completed. + */ + if (mode != XE_SRIOV_SCHED_GROUPS_DISABLED && + xe_guc_has_registered_mlrc_queues(>->uc.guc)) { + xe_gt_sriov_notice(gt, "can't enable sched groups with active MLRC queues\n"); + return -EPERM; + } + + err = __pf_provision_sched_groups(gt, mode); + if (err) + return err; + + gt->sriov.pf.policy.guc.sched_groups.current_mode = mode; + + return 0; +} + +static int pf_reprovision_sched_groups(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + /* We only have something to provision if we have possible groups */ + if (!xe_sriov_gt_pf_policy_has_multi_group_modes(gt)) + return 0; + + return __pf_provision_sched_groups(gt, gt->sriov.pf.policy.guc.sched_groups.current_mode); +} + +static void pf_sanitize_sched_groups(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + gt->sriov.pf.policy.guc.sched_groups.current_mode = XE_SRIOV_SCHED_GROUPS_DISABLED; +} + +/** + * xe_gt_sriov_pf_policy_set_sched_groups_mode() - Control the 'sched_groups' policy. + * @gt: the &xe_gt where to apply the policy + * @mode: the sched_group mode to be activated + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_sched_groups_mode(struct xe_gt *gt, + enum xe_sriov_sched_group_modes mode) +{ + if (!xe_sriov_gt_pf_policy_has_multi_group_modes(gt)) + return -ENODEV; + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + return pf_provision_sched_groups(gt, mode); +} + +/** + * xe_gt_sriov_pf_policy_sched_groups_enabled() - check whether the GT has + * multiple scheduler groups enabled + * @gt: the &xe_gt to check + * + * This function can only be called on PF. + * + * Return: true if the GT has multiple groups enabled, false otherwise. + */ +bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt) +{ + return gt->sriov.pf.policy.guc.sched_groups.current_mode != XE_SRIOV_SCHED_GROUPS_DISABLED; +} + +static void pf_sanitize_guc_policies(struct xe_gt *gt) +{ + pf_sanitize_sched_if_idle(gt); + pf_sanitize_reset_engine(gt); + pf_sanitize_sample_period(gt); + pf_sanitize_sched_groups(gt); +} + +/** + * xe_gt_sriov_pf_policy_sanitize - Reset policy settings. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt) +{ + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + pf_sanitize_guc_policies(gt); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); +} + +/** + * xe_gt_sriov_pf_policy_restart() - Reprovision policy settings. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt) +{ + int err = 0; + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + err |= pf_reprovision_sched_if_idle(gt); + err |= pf_reprovision_reset_engine(gt); + err |= pf_reprovision_sample_period(gt); + err |= pf_reprovision_sched_groups(gt); + + return err ? -ENXIO : 0; +} + +/** + * xe_gt_sriov_pf_policy_init() - Initializes the SW state of the PF policies. + * @gt: the &xe_gt + * + * This function can only be called on PF. This function does not touch the HW, + * but must be called after the engines have been initialized. + */ +void xe_gt_sriov_pf_policy_init(struct xe_gt *gt) +{ + pf_init_sched_groups(gt); +} + +static void print_guc_policies(struct drm_printer *p, struct xe_gt_sriov_guc_policies *policy) +{ + drm_printf(p, "%s:\t%s\n", + xe_guc_klv_key_to_string(GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY), + str_enabled_disabled(policy->sched_if_idle)); + drm_printf(p, "%s:\t%s\n", + xe_guc_klv_key_to_string(GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY), + str_enabled_disabled(policy->reset_engine)); + drm_printf(p, "%s:\t%u %s\n", + xe_guc_klv_key_to_string(GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY), + policy->sample_period, policy->sample_period ? "ms" : "(disabled)"); +} + +/** + * xe_gt_sriov_pf_policy_print - Dump actual policy values. + * @gt: the &xe_gt where to read the policy from + * @p: the &drm_printer + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + print_guc_policies(p, >->sriov.pf.policy.guc); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h new file mode 100644 index 000000000..d667d6689 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_POLICY_H_ +#define _XE_GT_SRIOV_PF_POLICY_H_ + +#include + +#include "xe_gt_sriov_pf_policy_types.h" + +struct drm_printer; +struct xe_gt; + +int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable); +int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable); +bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt); +bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt); +int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable); +bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt); +int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value); +u32 xe_gt_sriov_pf_policy_get_sample_period(struct xe_gt *gt); +bool xe_sriov_gt_pf_policy_has_sched_groups_support(struct xe_gt *gt); +bool xe_sriov_gt_pf_policy_has_multi_group_modes(struct xe_gt *gt); +bool xe_sriov_gt_pf_policy_has_sched_group_mode(struct xe_gt *gt, + enum xe_sriov_sched_group_modes mode); +int xe_gt_sriov_pf_policy_set_sched_groups_mode(struct xe_gt *gt, + enum xe_sriov_sched_group_modes mode); +bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt); + +void xe_gt_sriov_pf_policy_init(struct xe_gt *gt); +void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt); +int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt); +int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h new file mode 100644 index 000000000..97d278190 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_POLICY_TYPES_H_ +#define _XE_GT_SRIOV_PF_POLICY_TYPES_H_ + +#include + +#include "abi/guc_scheduler_abi.h" + +/** + * enum xe_sriov_sched_group_modes - list of possible scheduler group modes + * @XE_SRIOV_SCHED_GROUPS_DISABLED: no separate groups (i.e., all engines in group 0) + * @XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES: separate groups for each media slice + * @XE_SRIOV_SCHED_GROUPS_MODES_COUNT: number of valid modes + */ +enum xe_sriov_sched_group_modes { + XE_SRIOV_SCHED_GROUPS_DISABLED = 0, + XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES, + XE_SRIOV_SCHED_GROUPS_MODES_COUNT /* must be last */ +}; + +/** + * struct xe_gt_sriov_scheduler_groups - Scheduler groups policy info + * @max_groups: max number of groups supported by the GuC for the platform + * @supported_modes: mask of supported modes + * @current_mode: active scheduler groups mode + * @modes: array of masks and their number for each mode + * @modes.groups: array of engine instance groups in given mode, with each group + * consisting of GUC_MAX_ENGINE_CLASSES engine instances masks. A + * A NULL value indicates that all the engines are in the same + * group for this mode on this GT. + * @modes.num_groups: number of groups in given mode, zero if all the engines + * are in the same group. + */ +struct xe_gt_sriov_scheduler_groups { + u8 max_groups; + u32 supported_modes; + enum xe_sriov_sched_group_modes current_mode; + struct { + struct guc_sched_group *groups; + u32 num_groups; + } modes[XE_SRIOV_SCHED_GROUPS_MODES_COUNT]; +}; + +/** + * struct xe_gt_sriov_guc_policies - GuC SR-IOV policies. + * @sched_if_idle: controls strict scheduling policy. + * @reset_engine: controls engines reset on VF switch policy. + * @sample_period: adverse events sampling period (in milliseconds). + * @sched_groups: available scheduling group configurations. + */ +struct xe_gt_sriov_guc_policies { + bool sched_if_idle; + bool reset_engine; + u32 sample_period; + struct xe_gt_sriov_scheduler_groups sched_groups; +}; + +/** + * struct xe_gt_sriov_pf_policy - PF policy data. + * @guc: GuC scheduling policies. + */ +struct xe_gt_sriov_pf_policy { + struct xe_gt_sriov_guc_policies guc; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c new file mode 100644 index 000000000..6586df2fc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include + +#include "abi/guc_relay_actions_abi.h" + +#include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" + +#include "xe_assert.h" +#include "xe_mmio.h" +#include "xe_gt_sriov_printk.h" +#include "xe_gt_sriov_pf_service.h" +#include "xe_gt_sriov_pf_service_types.h" +#include "xe_guc_ct.h" +#include "xe_guc_hxg_helpers.h" +#include "xe_sriov.h" +#include "xe_sriov_pf_service.h" + +static const struct xe_reg tgl_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_SLICE_ENABLE, /* _MMIO(0x9138) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ +}; + +static const struct xe_reg ats_m_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + MIRROR_FUSE1, /* _MMIO(0x911c) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ + HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ +}; + +static const struct xe_reg pvc_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ + HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ +}; + +static const struct xe_reg ver_1270_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + XEHP_FUSE4, /* _MMIO(0x9114) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + MIRROR_FUSE1, /* _MMIO(0x911c) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ + HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ +}; + +static const struct xe_reg ver_2000_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + XEHP_FUSE4, /* _MMIO(0x9114) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + MIRROR_FUSE1, /* _MMIO(0x911c) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ + XE2_GT_COMPUTE_DSS_2, /* _MMIO(0x914c) */ + XE2_GT_GEOMETRY_DSS_1, /* _MMIO(0x9150) */ + XE2_GT_GEOMETRY_DSS_2, /* _MMIO(0x9154) */ + HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ +}; + +static const struct xe_reg ver_3000_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + XEHP_FUSE4, /* _MMIO(0x9114) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + MIRROR_FUSE1, /* _MMIO(0x911c) */ + MIRROR_L3BANK_ENABLE, /* _MMIO(0x9130) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ + XE2_GT_COMPUTE_DSS_2, /* _MMIO(0x914c) */ + XE2_GT_GEOMETRY_DSS_1, /* _MMIO(0x9150) */ + XE2_GT_GEOMETRY_DSS_2, /* _MMIO(0x9154) */ + HUC_KERNEL_LOAD_INFO, /* _MMIO(0xc1dc) */ +}; + +static const struct xe_reg ver_35_runtime_regs[] = { + RPM_CONFIG0, /* _MMIO(0x0d00) */ + XEHP_FUSE4, /* _MMIO(0x9114) */ + MIRROR_FUSE3, /* _MMIO(0x9118) */ + MIRROR_L3BANK_ENABLE, /* _MMIO(0x9130) */ + XELP_EU_ENABLE, /* _MMIO(0x9134) */ + XELP_GT_GEOMETRY_DSS_ENABLE, /* _MMIO(0x913c) */ + GT_VEBOX_VDBOX_DISABLE, /* _MMIO(0x9140) */ + XEHP_GT_COMPUTE_DSS_ENABLE, /* _MMIO(0x9144) */ + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,/* _MMIO(0x9148) */ + XE2_GT_COMPUTE_DSS_2, /* _MMIO(0x914c) */ + XE2_GT_GEOMETRY_DSS_1, /* _MMIO(0x9150) */ + XE2_GT_GEOMETRY_DSS_2, /* _MMIO(0x9154) */ + XE3P_XPC_GT_GEOMETRY_DSS_3, /* _MMIO(0x915c) */ + XE3P_XPC_GT_COMPUTE_DSS_3, /* _MMIO(0x9160) */ + SERVICE_COPY_ENABLE, /* _MMIO(0x9170) */ +}; + +static const struct xe_reg *pick_runtime_regs(struct xe_device *xe, unsigned int *count) +{ + const struct xe_reg *regs; + + if (GRAPHICS_VER(xe) >= 35) { + *count = ARRAY_SIZE(ver_35_runtime_regs); + regs = ver_35_runtime_regs; + } else if (GRAPHICS_VERx100(xe) >= 3000) { + *count = ARRAY_SIZE(ver_3000_runtime_regs); + regs = ver_3000_runtime_regs; + } else if (GRAPHICS_VERx100(xe) >= 2000) { + *count = ARRAY_SIZE(ver_2000_runtime_regs); + regs = ver_2000_runtime_regs; + } else if (GRAPHICS_VERx100(xe) >= 1270) { + *count = ARRAY_SIZE(ver_1270_runtime_regs); + regs = ver_1270_runtime_regs; + } else if (GRAPHICS_VERx100(xe) == 1260) { + *count = ARRAY_SIZE(pvc_runtime_regs); + regs = pvc_runtime_regs; + } else if (GRAPHICS_VERx100(xe) == 1255) { + *count = ARRAY_SIZE(ats_m_runtime_regs); + regs = ats_m_runtime_regs; + } else if (GRAPHICS_VERx100(xe) == 1200) { + *count = ARRAY_SIZE(tgl_runtime_regs); + regs = tgl_runtime_regs; + } else { + regs = ERR_PTR(-ENOPKG); + *count = 0; + } + + return regs; +} + +static int pf_alloc_runtime_info(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + const struct xe_reg *regs; + unsigned int size; + u32 *values; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + xe_gt_assert(gt, !gt->sriov.pf.service.runtime.size); + xe_gt_assert(gt, !gt->sriov.pf.service.runtime.regs); + xe_gt_assert(gt, !gt->sriov.pf.service.runtime.values); + + regs = pick_runtime_regs(xe, &size); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + if (unlikely(!size)) + return 0; + + values = drmm_kcalloc(&xe->drm, size, sizeof(u32), GFP_KERNEL); + if (!values) + return -ENOMEM; + + gt->sriov.pf.service.runtime.size = size; + gt->sriov.pf.service.runtime.regs = regs; + gt->sriov.pf.service.runtime.values = values; + + return 0; +} + +static void read_many(struct xe_gt *gt, unsigned int count, + const struct xe_reg *regs, u32 *values) +{ + while (count--) + *values++ = xe_mmio_read32(>->mmio, *regs++); +} + +static void pf_prepare_runtime_info(struct xe_gt *gt) +{ + const struct xe_reg *regs; + unsigned int size; + u32 *values; + + if (!gt->sriov.pf.service.runtime.size) + return; + + size = gt->sriov.pf.service.runtime.size; + regs = gt->sriov.pf.service.runtime.regs; + values = gt->sriov.pf.service.runtime.values; + + read_many(gt, size, regs, values); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + + xe_gt_sriov_pf_service_print_runtime(gt, &p); + } +} + +/** + * xe_gt_sriov_pf_service_init - Early initialization of the GT SR-IOV PF services. + * @gt: the &xe_gt to initialize + * + * Performs early initialization of the GT SR-IOV PF services, including preparation + * of the runtime info that will be shared with VFs. + * + * This function can only be called on PF. + */ +int xe_gt_sriov_pf_service_init(struct xe_gt *gt) +{ + int err; + + err = pf_alloc_runtime_info(gt); + if (unlikely(err)) + goto failed; + + return 0; +failed: + xe_gt_sriov_err(gt, "Failed to initialize service (%pe)\n", ERR_PTR(err)); + return err; +} + +/** + * xe_gt_sriov_pf_service_update - Update PF SR-IOV services. + * @gt: the &xe_gt to update + * + * Updates runtime data shared with VFs. + * + * This function can be called more than once. + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_service_update(struct xe_gt *gt) +{ + pf_prepare_runtime_info(gt); +} + +/* Return: length of the response message or a negative error code on failure. */ +static int pf_process_handshake_msg(struct xe_gt *gt, u32 origin, + const u32 *request, u32 len, u32 *response, u32 size) +{ + u32 wanted_major, wanted_minor; + u32 major, minor; + u32 mbz; + int err; + + if (unlikely(len != VF2PF_HANDSHAKE_REQUEST_MSG_LEN)) + return -EMSGSIZE; + + mbz = FIELD_GET(VF2PF_HANDSHAKE_REQUEST_MSG_0_MBZ, request[0]); + if (unlikely(mbz)) + return -EPFNOSUPPORT; + + wanted_major = FIELD_GET(VF2PF_HANDSHAKE_REQUEST_MSG_1_MAJOR, request[1]); + wanted_minor = FIELD_GET(VF2PF_HANDSHAKE_REQUEST_MSG_1_MINOR, request[1]); + + err = xe_sriov_pf_service_handshake_vf(gt_to_xe(gt), origin, wanted_major, wanted_minor, + &major, &minor); + if (err < 0) + return err; + + xe_gt_assert(gt, major || minor); + xe_gt_assert(gt, size >= VF2PF_HANDSHAKE_RESPONSE_MSG_LEN); + + response[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_SUCCESS) | + FIELD_PREP(GUC_HXG_RESPONSE_MSG_0_DATA0, 0); + response[1] = FIELD_PREP(VF2PF_HANDSHAKE_RESPONSE_MSG_1_MAJOR, major) | + FIELD_PREP(VF2PF_HANDSHAKE_RESPONSE_MSG_1_MINOR, minor); + + return VF2PF_HANDSHAKE_RESPONSE_MSG_LEN; +} + +struct reg_data { + u32 offset; + u32 value; +} __packed; +static_assert(hxg_sizeof(struct reg_data) == 2); + +/* Return: number of entries copied or negative error code on failure. */ +static int pf_service_runtime_query(struct xe_gt *gt, u32 start, u32 limit, + struct reg_data *data, u32 *remaining) +{ + struct xe_gt_sriov_pf_service_runtime_regs *runtime; + unsigned int count, i; + u32 addr; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + runtime = >->sriov.pf.service.runtime; + + if (start > runtime->size) + return -ERANGE; + + count = min_t(u32, runtime->size - start, limit); + + for (i = 0; i < count; ++i, ++data) { + addr = runtime->regs[start + i].addr; + data->offset = xe_mmio_adjusted_addr(>->mmio, addr); + data->value = runtime->values[start + i]; + } + + *remaining = runtime->size - start - count; + return count; +} + +/* Return: length of the response message or a negative error code on failure. */ +static int pf_process_runtime_query_msg(struct xe_gt *gt, u32 origin, + const u32 *msg, u32 msg_len, u32 *response, u32 resp_size) +{ + const u32 chunk_size = hxg_sizeof(struct reg_data); + struct reg_data *reg_data_buf; + u32 limit, start, max_chunks; + u32 remaining = 0; + int ret; + + /* this action is available from ABI 1.0 */ + if (!xe_sriov_pf_service_is_negotiated(gt_to_xe(gt), origin, 1, 0)) + return -EACCES; + + if (unlikely(msg_len > VF2PF_QUERY_RUNTIME_REQUEST_MSG_LEN)) + return -EMSGSIZE; + if (unlikely(msg_len < VF2PF_QUERY_RUNTIME_REQUEST_MSG_LEN)) + return -EPROTO; + if (unlikely(resp_size < VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN)) + return -EINVAL; + + limit = FIELD_GET(VF2PF_QUERY_RUNTIME_REQUEST_MSG_0_LIMIT, msg[0]); + start = FIELD_GET(VF2PF_QUERY_RUNTIME_REQUEST_MSG_1_START, msg[1]); + + resp_size = min_t(u32, resp_size, VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MAX_LEN); + max_chunks = (resp_size - VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN) / chunk_size; + limit = limit == VF2PF_QUERY_RUNTIME_NO_LIMIT ? max_chunks : min_t(u32, max_chunks, limit); + reg_data_buf = (void *)(response + VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN); + + ret = pf_service_runtime_query(gt, start, limit, reg_data_buf, &remaining); + if (ret < 0) + return ret; + + response[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_SUCCESS) | + FIELD_PREP(VF2PF_QUERY_RUNTIME_RESPONSE_MSG_0_COUNT, ret); + response[1] = FIELD_PREP(VF2PF_QUERY_RUNTIME_RESPONSE_MSG_1_REMAINING, remaining); + + return VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN + ret * hxg_sizeof(struct reg_data); +} + +/** + * xe_gt_sriov_pf_service_process_request - Service GT level SR-IOV request message from the VF. + * @gt: the &xe_gt that provides the service + * @origin: VF number that is requesting the service + * @msg: request message + * @msg_len: length of the request message (in dwords) + * @response: placeholder for the response message + * @resp_size: length of the response message buffer (in dwords) + * + * This function processes `Relay Message`_ request from the VF. + * + * Return: length of the response message or a negative error code on failure. + */ +int xe_gt_sriov_pf_service_process_request(struct xe_gt *gt, u32 origin, + const u32 *msg, u32 msg_len, + u32 *response, u32 resp_size) +{ + u32 action, data __maybe_unused; + int ret; + + xe_gt_assert(gt, msg_len >= GUC_HXG_MSG_MIN_LEN); + xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_REQUEST); + + action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); + data = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]); + xe_gt_sriov_dbg_verbose(gt, "service action %#x:%u from VF%u\n", + action, data, origin); + + switch (action) { + case GUC_RELAY_ACTION_VF2PF_HANDSHAKE: + ret = pf_process_handshake_msg(gt, origin, msg, msg_len, response, resp_size); + break; + case GUC_RELAY_ACTION_VF2PF_QUERY_RUNTIME: + ret = pf_process_runtime_query_msg(gt, origin, msg, msg_len, response, resp_size); + break; + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +/** + * xe_gt_sriov_pf_service_print_runtime - Print PF runtime data shared with VFs. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * This function is for PF use only. + */ +int xe_gt_sriov_pf_service_print_runtime(struct xe_gt *gt, struct drm_printer *p) +{ + const struct xe_reg *regs; + unsigned int size; + u32 *values; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + size = gt->sriov.pf.service.runtime.size; + regs = gt->sriov.pf.service.runtime.regs; + values = gt->sriov.pf.service.runtime.values; + + for (; size--; regs++, values++) { + drm_printf(p, "reg[%#x] = %#x\n", + xe_mmio_adjusted_addr(>->mmio, regs->addr), *values); + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.h new file mode 100644 index 000000000..10b02c9b6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_SERVICE_H_ +#define _XE_GT_SRIOV_PF_SERVICE_H_ + +#include +#include + +struct drm_printer; +struct xe_gt; + +int xe_gt_sriov_pf_service_init(struct xe_gt *gt); +void xe_gt_sriov_pf_service_update(struct xe_gt *gt); + +int xe_gt_sriov_pf_service_print_runtime(struct xe_gt *gt, struct drm_printer *p); + +#ifdef CONFIG_PCI_IOV +int xe_gt_sriov_pf_service_process_request(struct xe_gt *gt, u32 origin, + const u32 *msg, u32 msg_len, + u32 *response, u32 resp_size); +#else +static inline int +xe_gt_sriov_pf_service_process_request(struct xe_gt *gt, u32 origin, + const u32 *msg, u32 msg_len, + u32 *response, u32 resp_size) +{ + return -EPROTO; +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service_types.h new file mode 100644 index 000000000..ad6dd75f0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service_types.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_SERVICE_TYPES_H_ +#define _XE_GT_SRIOV_PF_SERVICE_TYPES_H_ + +#include + +struct xe_reg; + +/** + * struct xe_gt_sriov_pf_service_version - VF/PF ABI Version. + * @major: the major version of the VF/PF ABI + * @minor: the minor version of the VF/PF ABI + * + * See `GuC Relay Communication`_. + */ +struct xe_gt_sriov_pf_service_version { + u16 major; + u16 minor; +}; + +/** + * struct xe_gt_sriov_pf_service_runtime_regs - Runtime data shared with VFs. + * @regs: pointer to static array with register offsets. + * @values: pointer to array with captured register values. + * @size: size of the regs and value arrays. + */ +struct xe_gt_sriov_pf_service_runtime_regs { + const struct xe_reg *regs; + u32 *values; + u32 size; +}; + +/** + * struct xe_gt_sriov_pf_service - Data used by the PF service. + * @version: information about VF/PF ABI versions for current platform. + * @version.base: lowest VF/PF ABI version that could be negotiated with VF. + * @version.latest: latest VF/PF ABI version supported by the PF driver. + * @runtime: runtime data shared with VFs. + */ +struct xe_gt_sriov_pf_service { + struct { + struct xe_gt_sriov_pf_service_version base; + struct xe_gt_sriov_pf_service_version latest; + } version; + struct xe_gt_sriov_pf_service_runtime_regs runtime; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h new file mode 100644 index 000000000..6296483c4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_TYPES_H_ +#define _XE_GT_SRIOV_PF_TYPES_H_ + +#include + +#include "xe_gt_sriov_pf_config_types.h" +#include "xe_gt_sriov_pf_control_types.h" +#include "xe_gt_sriov_pf_migration_types.h" +#include "xe_gt_sriov_pf_monitor_types.h" +#include "xe_gt_sriov_pf_policy_types.h" +#include "xe_gt_sriov_pf_service_types.h" + +/** + * struct xe_gt_sriov_metadata - GT level per-VF metadata. + */ +struct xe_gt_sriov_metadata { + /** @config: per-VF provisioning data. */ + struct xe_gt_sriov_config config; + + /** @monitor: per-VF monitoring data. */ + struct xe_gt_sriov_monitor monitor; + + /** @control: per-VF control data. */ + struct xe_gt_sriov_control_state control; + + /** @version: negotiated VF/PF ABI version */ + struct xe_gt_sriov_pf_service_version version; + + /** @migration: per-VF migration data. */ + struct xe_gt_sriov_migration_data migration; +}; + +/** + * struct xe_gt_sriov_pf_workers - GT level workers used by the PF. + */ +struct xe_gt_sriov_pf_workers { + /** @restart: worker that executes actions post GT reset */ + struct work_struct restart; +}; + +/** + * struct xe_gt_sriov_pf - GT level PF virtualization data. + * @workers: workers data. + * @service: service data. + * @control: control data. + * @policy: policy data. + * @spare: PF-only provisioning configuration. + * @vfs: metadata for all VFs. + */ +struct xe_gt_sriov_pf { + struct xe_gt_sriov_pf_workers workers; + struct xe_gt_sriov_pf_service service; + struct xe_gt_sriov_pf_control control; + struct xe_gt_sriov_pf_policy policy; + struct xe_gt_sriov_spare_config spare; + struct xe_gt_sriov_metadata *vfs; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_printk.h b/drivers/gpu/drm/xe/xe_gt_sriov_printk.h new file mode 100644 index 000000000..d3457d608 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_printk.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PRINTK_H_ +#define _XE_GT_SRIOV_PRINTK_H_ + +#include "xe_gt_printk.h" +#include "xe_tile_sriov_printk.h" + +#define __XE_GT_SRIOV_PRINTK_FMT(_gt, _fmt, ...) \ + __XE_TILE_SRIOV_PRINTK_FMT((_gt)->tile, __XE_GT_PRINTK_FMT((_gt), _fmt, ##__VA_ARGS__)) + +#define __xe_gt_sriov_printk(gt, _level, fmt, ...) \ + xe_sriov_##_level(gt_to_xe(gt), __XE_GT_SRIOV_PRINTK_FMT((gt), fmt, ##__VA_ARGS__)) + +#define xe_gt_sriov_err(_gt, _fmt, ...) \ + __xe_gt_sriov_printk(_gt, err, _fmt, ##__VA_ARGS__) + +#define xe_gt_sriov_notice(_gt, _fmt, ...) \ + __xe_gt_sriov_printk(_gt, notice, _fmt, ##__VA_ARGS__) + +#define xe_gt_sriov_info(_gt, _fmt, ...) \ + __xe_gt_sriov_printk(_gt, info, _fmt, ##__VA_ARGS__) + +#define xe_gt_sriov_dbg(_gt, _fmt, ...) \ + __xe_gt_sriov_printk(_gt, dbg, _fmt, ##__VA_ARGS__) + +/* for low level noisy debug messages */ +#ifdef CONFIG_DRM_XE_DEBUG_SRIOV +#define xe_gt_sriov_dbg_verbose(_gt, _fmt, ...) xe_gt_sriov_dbg(_gt, _fmt, ##__VA_ARGS__) +#else +#define xe_gt_sriov_dbg_verbose(_gt, _fmt, ...) typecheck(struct xe_gt *, (_gt)) +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c new file mode 100644 index 000000000..37899fcf5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -0,0 +1,1625 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include +#include + +#include +#include + +#include "abi/guc_actions_sriov_abi.h" +#include "abi/guc_communication_mmio_abi.h" +#include "abi/guc_klvs_abi.h" +#include "abi/guc_relay_actions_abi.h" +#include "regs/xe_gt_regs.h" + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_ggtt.h" +#include "xe_gt_sriov_printk.h" +#include "xe_gt_sriov_vf.h" +#include "xe_gt_sriov_vf_types.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_hxg_helpers.h" +#include "xe_guc_relay.h" +#include "xe_guc_submit.h" +#include "xe_irq.h" +#include "xe_lrc.h" +#include "xe_memirq.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_sriov_vf.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_tile_sriov_vf.h" +#include "xe_tlb_inval.h" +#include "xe_uc_fw.h" +#include "xe_wopcm.h" + +#define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) + +#ifdef CONFIG_DRM_XE_DEBUG +enum VF_MIGRATION_WAIT_POINTS { + VF_MIGRATION_WAIT_RESFIX_START = BIT(0), + VF_MIGRATION_WAIT_FIXUPS = BIT(1), + VF_MIGRATION_WAIT_RESTART_JOBS = BIT(2), + VF_MIGRATION_WAIT_RESFIX_DONE = BIT(3), +}; + +#define VF_MIGRATION_WAIT_DELAY_IN_MS 1000 +static void vf_post_migration_inject_wait(struct xe_gt *gt, + enum VF_MIGRATION_WAIT_POINTS wait) +{ + while (gt->sriov.vf.migration.debug.resfix_stoppers & wait) { + xe_gt_dbg(gt, + "*TESTING* injecting %u ms delay due to resfix_stoppers=%#x, to continue clear %#x\n", + VF_MIGRATION_WAIT_DELAY_IN_MS, + gt->sriov.vf.migration.debug.resfix_stoppers, wait); + + msleep(VF_MIGRATION_WAIT_DELAY_IN_MS); + } +} + +#define VF_MIGRATION_INJECT_WAIT(gt, _POS) ({ \ + struct xe_gt *__gt = (gt); \ + vf_post_migration_inject_wait(__gt, VF_MIGRATION_WAIT_##_POS); \ + }) + +#else +#define VF_MIGRATION_INJECT_WAIT(_gt, ...) typecheck(struct xe_gt *, (_gt)) +#endif + +static int guc_action_vf_reset(struct xe_guc *guc) +{ + u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_VF_RESET), + }; + int ret; + + ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +#define GUC_RESET_VF_STATE_RETRY_MAX 10 +static int vf_reset_guc_state(struct xe_gt *gt) +{ + unsigned int retry = GUC_RESET_VF_STATE_RETRY_MAX; + struct xe_guc *guc = >->uc.guc; + int err; + + do { + err = guc_action_vf_reset(guc); + if (!err || err != -ETIMEDOUT) + break; + } while (--retry); + + if (unlikely(err)) + xe_gt_sriov_err(gt, "Failed to reset GuC state (%pe)\n", ERR_PTR(err)); + return err; +} + +/** + * xe_gt_sriov_vf_reset - Reset GuC VF internal state. + * @gt: the &xe_gt + * + * It requires functional `GuC MMIO based communication`_. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_vf_reset(struct xe_gt *gt) +{ + if (!xe_device_uc_enabled(gt_to_xe(gt))) + return -ENODEV; + + return vf_reset_guc_state(gt); +} + +static int guc_action_match_version(struct xe_guc *guc, + struct xe_uc_fw_version *wanted, + struct xe_uc_fw_version *found) +{ + u32 request[VF2GUC_MATCH_VERSION_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_ACTION_VF2GUC_MATCH_VERSION), + FIELD_PREP(VF2GUC_MATCH_VERSION_REQUEST_MSG_1_BRANCH, wanted->branch) | + FIELD_PREP(VF2GUC_MATCH_VERSION_REQUEST_MSG_1_MAJOR, wanted->major) | + FIELD_PREP(VF2GUC_MATCH_VERSION_REQUEST_MSG_1_MINOR, wanted->minor), + }; + u32 response[GUC_MAX_MMIO_MSG_LEN]; + int ret; + + BUILD_BUG_ON(VF2GUC_MATCH_VERSION_RESPONSE_MSG_LEN > GUC_MAX_MMIO_MSG_LEN); + + ret = xe_guc_mmio_send_recv(guc, request, ARRAY_SIZE(request), response); + if (unlikely(ret < 0)) + return ret; + + if (unlikely(FIELD_GET(VF2GUC_MATCH_VERSION_RESPONSE_MSG_0_MBZ, response[0]))) + return -EPROTO; + + memset(found, 0, sizeof(struct xe_uc_fw_version)); + found->branch = FIELD_GET(VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_BRANCH, response[1]); + found->major = FIELD_GET(VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_MAJOR, response[1]); + found->minor = FIELD_GET(VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_MINOR, response[1]); + found->patch = FIELD_GET(VF2GUC_MATCH_VERSION_RESPONSE_MSG_1_PATCH, response[1]); + + return 0; +} + +static int guc_action_match_version_any(struct xe_guc *guc, + struct xe_uc_fw_version *found) +{ + struct xe_uc_fw_version wanted = { + .branch = GUC_VERSION_BRANCH_ANY, + .major = GUC_VERSION_MAJOR_ANY, + .minor = GUC_VERSION_MINOR_ANY, + .patch = 0 + }; + + return guc_action_match_version(guc, &wanted, found); +} + +static void vf_minimum_guc_version(struct xe_gt *gt, struct xe_uc_fw_version *ver) +{ + struct xe_device *xe = gt_to_xe(gt); + + memset(ver, 0, sizeof(struct xe_uc_fw_version)); + + switch (xe->info.platform) { + case XE_TIGERLAKE ... XE_PVC: + /* 1.1 this is current baseline for Xe driver */ + ver->branch = 0; + ver->major = 1; + ver->minor = 1; + break; + default: + /* 1.2 has support for the GMD_ID KLV */ + ver->branch = 0; + ver->major = 1; + ver->minor = 2; + break; + } +} + +static void vf_wanted_guc_version(struct xe_gt *gt, struct xe_uc_fw_version *ver) +{ + /* for now it's the same as minimum */ + return vf_minimum_guc_version(gt, ver); +} + +static int vf_handshake_with_guc(struct xe_gt *gt) +{ + struct xe_uc_fw_version *guc_version = >->sriov.vf.guc_version; + struct xe_uc_fw_version wanted = {0}; + struct xe_guc *guc = >->uc.guc; + bool old = false; + int err; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + /* select wanted version - prefer previous (if any) */ + if (guc_version->major || guc_version->minor) { + wanted = *guc_version; + old = true; + } else { + vf_wanted_guc_version(gt, &wanted); + xe_gt_assert(gt, wanted.major != GUC_VERSION_MAJOR_ANY); + + /* First time we handshake, so record the minimum wanted */ + gt->sriov.vf.wanted_guc_version = wanted; + } + + err = guc_action_match_version(guc, &wanted, guc_version); + if (unlikely(err)) + goto fail; + + if (old) { + /* we don't support interface version change */ + if (MAKE_GUC_VER_STRUCT(*guc_version) != MAKE_GUC_VER_STRUCT(wanted)) { + xe_gt_sriov_err(gt, "New GuC interface version detected: %u.%u.%u.%u\n", + guc_version->branch, guc_version->major, + guc_version->minor, guc_version->patch); + xe_gt_sriov_info(gt, "Previously used version was: %u.%u.%u.%u\n", + wanted.branch, wanted.major, + wanted.minor, wanted.patch); + err = -EREMCHG; + goto fail; + } else { + /* version is unchanged, no need to re-verify it */ + return 0; + } + } + + /* illegal */ + if (guc_version->major > wanted.major) { + err = -EPROTO; + goto unsupported; + } + + /* there's no fallback on major version. */ + if (guc_version->major != wanted.major) { + err = -ENOPKG; + goto unsupported; + } + + /* check against minimum version supported by us */ + vf_minimum_guc_version(gt, &wanted); + xe_gt_assert(gt, wanted.major != GUC_VERSION_MAJOR_ANY); + if (MAKE_GUC_VER_STRUCT(*guc_version) < MAKE_GUC_VER_STRUCT(wanted)) { + err = -ENOKEY; + goto unsupported; + } + + xe_gt_sriov_dbg(gt, "using GuC interface version %u.%u.%u.%u\n", + guc_version->branch, guc_version->major, + guc_version->minor, guc_version->patch); + + return 0; + +unsupported: + xe_gt_sriov_err(gt, "Unsupported GuC version %u.%u.%u.%u (%pe)\n", + guc_version->branch, guc_version->major, + guc_version->minor, guc_version->patch, + ERR_PTR(err)); +fail: + xe_gt_sriov_err(gt, "Unable to confirm GuC version %u.%u (%pe)\n", + wanted.major, wanted.minor, ERR_PTR(err)); + + /* try again with *any* just to query which version is supported */ + if (!guc_action_match_version_any(guc, &wanted)) + xe_gt_sriov_notice(gt, "GuC reports interface version %u.%u.%u.%u\n", + wanted.branch, wanted.major, wanted.minor, wanted.patch); + return err; +} + +/** + * xe_gt_sriov_vf_bootstrap - Query and setup GuC ABI interface version. + * @gt: the &xe_gt + * + * This function is for VF use only. + * It requires functional `GuC MMIO based communication`_. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_vf_bootstrap(struct xe_gt *gt) +{ + int err; + + if (!xe_device_uc_enabled(gt_to_xe(gt))) + return -ENODEV; + + err = vf_reset_guc_state(gt); + if (unlikely(err)) + return err; + + err = vf_handshake_with_guc(gt); + if (unlikely(err)) + return err; + + return 0; +} + +/** + * xe_gt_sriov_vf_guc_versions - Minimum required and found GuC ABI versions + * @gt: the &xe_gt + * @wanted: pointer to the xe_uc_fw_version to be filled with the wanted version + * @found: pointer to the xe_uc_fw_version to be filled with the found version + * + * This function is for VF use only and it can only be used after successful + * version handshake with the GuC. + */ +void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt, + struct xe_uc_fw_version *wanted, + struct xe_uc_fw_version *found) +{ + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, gt->sriov.vf.guc_version.major); + + if (wanted) + *wanted = gt->sriov.vf.wanted_guc_version; + + if (found) + *found = gt->sriov.vf.guc_version; +} + +static int guc_action_vf_resfix_start(struct xe_guc *guc, u16 marker) +{ + u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_START) | + FIELD_PREP(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER, marker), + }; + int ret; + + ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static int vf_resfix_start(struct xe_gt *gt, u16 marker) +{ + struct xe_guc *guc = >->uc.guc; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + VF_MIGRATION_INJECT_WAIT(gt, RESFIX_START); + + xe_gt_sriov_dbg_verbose(gt, "Sending resfix start marker %u\n", marker); + + return guc_action_vf_resfix_start(guc, marker); +} + +static int guc_action_vf_resfix_done(struct xe_guc *guc, u16 marker) +{ + u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_DONE) | + FIELD_PREP(VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER, marker), + }; + int ret; + + ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static int vf_resfix_done(struct xe_gt *gt, u16 marker) +{ + struct xe_guc *guc = >->uc.guc; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + xe_gt_sriov_dbg_verbose(gt, "Sending resfix done marker %u\n", marker); + + return guc_action_vf_resfix_done(guc, marker); +} + +static int guc_action_query_single_klv(struct xe_guc *guc, u32 key, + u32 *value, u32 value_len) +{ + u32 request[VF2GUC_QUERY_SINGLE_KLV_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_ACTION_VF2GUC_QUERY_SINGLE_KLV), + FIELD_PREP(VF2GUC_QUERY_SINGLE_KLV_REQUEST_MSG_1_KEY, key), + }; + u32 response[GUC_MAX_MMIO_MSG_LEN]; + u32 length; + int ret; + + BUILD_BUG_ON(VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_MAX_LEN > GUC_MAX_MMIO_MSG_LEN); + ret = xe_guc_mmio_send_recv(guc, request, ARRAY_SIZE(request), response); + if (unlikely(ret < 0)) + return ret; + + if (unlikely(FIELD_GET(VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_0_MBZ, response[0]))) + return -EPROTO; + + length = FIELD_GET(VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_0_LENGTH, response[0]); + if (unlikely(length > value_len)) + return -EOVERFLOW; + if (unlikely(length < value_len)) + return -ENODATA; + + switch (value_len) { + default: + xe_gt_WARN_ON(guc_to_gt(guc), value_len > 3); + fallthrough; + case 3: + value[2] = FIELD_GET(VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_3_VALUE96, response[3]); + fallthrough; + case 2: + value[1] = FIELD_GET(VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_2_VALUE64, response[2]); + fallthrough; + case 1: + value[0] = FIELD_GET(VF2GUC_QUERY_SINGLE_KLV_RESPONSE_MSG_1_VALUE32, response[1]); + fallthrough; + case 0: + break; + } + + return 0; +} + +static int guc_action_query_single_klv32(struct xe_guc *guc, u32 key, u32 *value32) +{ + return guc_action_query_single_klv(guc, key, value32, hxg_sizeof(u32)); +} + +static int guc_action_query_single_klv64(struct xe_guc *guc, u32 key, u64 *value64) +{ + u32 value[2]; + int err; + + err = guc_action_query_single_klv(guc, key, value, hxg_sizeof(value)); + if (unlikely(err)) + return err; + + *value64 = make_u64_from_u32(value[1], value[0]); + return 0; +} + +static bool has_gmdid(struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) >= 1270; +} + +/** + * xe_gt_sriov_vf_gmdid - Query GMDID over MMIO. + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: value of GMDID KLV on success or 0 on failure. + */ +u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt) +{ + const char *type = xe_gt_is_media_type(gt) ? "media" : "graphics"; + struct xe_guc *guc = >->uc.guc; + u32 value; + int err; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, !GRAPHICS_VERx100(gt_to_xe(gt)) || has_gmdid(gt_to_xe(gt))); + xe_gt_assert(gt, gt->sriov.vf.guc_version.major > 1 || gt->sriov.vf.guc_version.minor >= 2); + + err = guc_action_query_single_klv32(guc, GUC_KLV_GLOBAL_CFG_GMD_ID_KEY, &value); + if (unlikely(err)) { + xe_gt_sriov_err(gt, "Failed to obtain %s GMDID (%pe)\n", + type, ERR_PTR(err)); + return 0; + } + + xe_gt_sriov_dbg(gt, "%s GMDID = %#x\n", type, value); + return value; +} + +static int vf_get_ggtt_info(struct xe_gt *gt) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_guc *guc = >->uc.guc; + u64 start, size, ggtt_size; + int err; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + err = guc_action_query_single_klv64(guc, GUC_KLV_VF_CFG_GGTT_START_KEY, &start); + if (unlikely(err)) + return err; + + err = guc_action_query_single_klv64(guc, GUC_KLV_VF_CFG_GGTT_SIZE_KEY, &size); + if (unlikely(err)) + return err; + + if (!size) + return -ENODATA; + + xe_tile_sriov_vf_ggtt_base_store(tile, start); + ggtt_size = xe_tile_sriov_vf_ggtt(tile); + if (!ggtt_size) { + /* + * This function is called once during xe_guc_init_noalloc(), + * at which point ggtt_size = 0 and we have to initialize everything, + * and GGTT is not yet initialized. + * + * Return early as there's nothing to fixup. + */ + xe_tile_sriov_vf_ggtt_store(tile, size); + return 0; + } + + if (ggtt_size != size) { + xe_gt_sriov_err(gt, "Unexpected GGTT reassignment: %lluK != %lluK\n", + size / SZ_1K, ggtt_size / SZ_1K); + return -EREMCHG; + } + + xe_gt_sriov_dbg_verbose(gt, "GGTT %#llx-%#llx = %lluK\n", + start, start + size - 1, size / SZ_1K); + + /* + * This function can be called repeatedly from post migration fixups, + * at which point we inform the GGTT of the new base address. + * xe_ggtt_shift_nodes() may be called multiple times for each migration, + * but will be a noop if the base is unchanged. + */ + xe_ggtt_shift_nodes(tile->mem.ggtt, start); + + return 0; +} + +static int vf_get_lmem_info(struct xe_gt *gt) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_guc *guc = >->uc.guc; + char size_str[10]; + u64 size, lmem_size; + int err; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + err = guc_action_query_single_klv64(guc, GUC_KLV_VF_CFG_LMEM_SIZE_KEY, &size); + if (unlikely(err)) + return err; + + lmem_size = xe_tile_sriov_vf_lmem(tile); + if (lmem_size && lmem_size != size) { + xe_gt_sriov_err(gt, "Unexpected LMEM reassignment: %lluM != %lluM\n", + size / SZ_1M, lmem_size / SZ_1M); + return -EREMCHG; + } + + string_get_size(size, 1, STRING_UNITS_2, size_str, sizeof(size_str)); + xe_gt_sriov_dbg_verbose(gt, "LMEM %lluM %s\n", size / SZ_1M, size_str); + + xe_tile_sriov_vf_lmem_store(tile, size); + + return size ? 0 : -ENODATA; +} + +static int vf_get_submission_cfg(struct xe_gt *gt) +{ + struct xe_gt_sriov_vf_selfconfig *config = >->sriov.vf.self_config; + struct xe_guc *guc = >->uc.guc; + u32 num_ctxs, num_dbs; + int err; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + err = guc_action_query_single_klv32(guc, GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY, &num_ctxs); + if (unlikely(err)) + return err; + + err = guc_action_query_single_klv32(guc, GUC_KLV_VF_CFG_NUM_DOORBELLS_KEY, &num_dbs); + if (unlikely(err)) + return err; + + if (config->num_ctxs && config->num_ctxs != num_ctxs) { + xe_gt_sriov_err(gt, "Unexpected CTXs reassignment: %u != %u\n", + num_ctxs, config->num_ctxs); + return -EREMCHG; + } + if (config->num_dbs && config->num_dbs != num_dbs) { + xe_gt_sriov_err(gt, "Unexpected DBs reassignment: %u != %u\n", + num_dbs, config->num_dbs); + return -EREMCHG; + } + + xe_gt_sriov_dbg_verbose(gt, "CTXs %u DBs %u\n", num_ctxs, num_dbs); + + config->num_ctxs = num_ctxs; + config->num_dbs = num_dbs; + + return config->num_ctxs ? 0 : -ENODATA; +} + +static void vf_cache_gmdid(struct xe_gt *gt) +{ + xe_gt_assert(gt, has_gmdid(gt_to_xe(gt))); + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + gt->sriov.vf.runtime.gmdid = xe_gt_sriov_vf_gmdid(gt); +} + +static int vf_query_sched_groups(struct xe_gt *gt) +{ + struct xe_guc *guc = >->uc.guc; + struct xe_uc_fw_version guc_version; + u32 value = 0; + int err; + + xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version); + + if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 26, 0)) + return 0; + + err = guc_action_query_single_klv32(guc, + GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY, + &value); + if (unlikely(err)) { + xe_gt_sriov_err(gt, "Failed to obtain sched groups status (%pe)\n", + ERR_PTR(err)); + return err; + } + + /* valid values are 0 (disabled) and 1 (enabled) */ + if (value > 1) { + xe_gt_sriov_err(gt, "Invalid sched groups status %u\n", value); + return -EPROTO; + } + + xe_gt_sriov_dbg(gt, "sched groups %s\n", str_enabled_disabled(value)); + return value; +} + +static int vf_cache_sched_groups_status(struct xe_gt *gt) +{ + int ret; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + ret = vf_query_sched_groups(gt); + if (ret < 0) + return ret; + + gt->sriov.vf.runtime.uses_sched_groups = ret; + + return 0; +} + +static int vf_cache_num_paging_engines(struct xe_gt *gt) +{ + struct xe_guc *guc = >->uc.guc; + struct xe_uc_fw_version guc_version; + u32 value = 0; + int err; + + xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version); + + if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 36, 0)) + return 0; + + err = guc_action_query_single_klv32(guc, GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_KEY, + &value); + if (unlikely(err)) { + xe_gt_sriov_err(gt, + "Failed to obtain the number of paging instances (%pe)\n", + ERR_PTR(err)); + return err; + } + + gt->sriov.vf.runtime.num_paging_engine_instances = value; + + xe_gt_sriov_dbg(gt, "num_paging_engines %u\n", value); + return 0; +} + +/** + * xe_gt_sriov_vf_query_config - Query SR-IOV config data over MMIO. + * @gt: the &xe_gt + * + * This function is for VF use only. This function may shift the GGTT and is + * performed under GGTT lock, making this step visible to all GTs that share a + * GGTT. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_vf_query_config(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + err = vf_get_ggtt_info(gt); + if (unlikely(err)) + return err; + + if (IS_DGFX(xe) && xe_gt_is_main_type(gt)) { + err = vf_get_lmem_info(gt); + if (unlikely(err)) + return err; + } + + err = vf_get_submission_cfg(gt); + if (unlikely(err)) + return err; + + err = vf_cache_sched_groups_status(gt); + if (unlikely(err)) + return err; + + if (has_gmdid(xe)) + vf_cache_gmdid(gt); + + err = vf_cache_num_paging_engines(gt); + if (unlikely(err)) + return err; + + return 0; +} + +/** + * xe_gt_sriov_vf_sched_groups_enabled() - Check if PF has enabled multiple + * scheduler groups + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: true if shed groups were enabled, false otherwise. + */ +bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, gt->sriov.vf.guc_version.major); + + return gt->sriov.vf.runtime.uses_sched_groups; +} + +/** + * xe_gt_sriov_vf_guc_ids - VF GuC context IDs configuration. + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: number of GuC context IDs assigned to VF. + */ +u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, gt->sriov.vf.guc_version.major); + xe_gt_assert(gt, gt->sriov.vf.self_config.num_ctxs); + + return gt->sriov.vf.self_config.num_ctxs; +} + +/** + * xe_gt_sriov_vf_paging_engines - Return the number of paging engine instances + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: number of GuC paging engine instances configured by the PF. + */ +u32 xe_gt_sriov_vf_paging_engines(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, gt->sriov.vf.guc_version.major); + + return gt->sriov.vf.runtime.num_paging_engine_instances; +} + +static int relay_action_handshake(struct xe_gt *gt, u32 *major, u32 *minor) +{ + u32 request[VF2PF_HANDSHAKE_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_RELAY_ACTION_VF2PF_HANDSHAKE), + FIELD_PREP(VF2PF_HANDSHAKE_REQUEST_MSG_1_MAJOR, *major) | + FIELD_PREP(VF2PF_HANDSHAKE_REQUEST_MSG_1_MINOR, *minor), + }; + u32 response[VF2PF_HANDSHAKE_RESPONSE_MSG_LEN]; + int ret; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + ret = xe_guc_relay_send_to_pf(>->uc.guc.relay, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + if (unlikely(ret < 0)) + return ret; + + if (unlikely(ret != VF2PF_HANDSHAKE_RESPONSE_MSG_LEN)) + return -EPROTO; + + if (unlikely(FIELD_GET(VF2PF_HANDSHAKE_RESPONSE_MSG_0_MBZ, response[0]))) + return -EPROTO; + + *major = FIELD_GET(VF2PF_HANDSHAKE_RESPONSE_MSG_1_MAJOR, response[1]); + *minor = FIELD_GET(VF2PF_HANDSHAKE_RESPONSE_MSG_1_MINOR, response[1]); + + return 0; +} + +static void vf_connect_pf(struct xe_device *xe, u16 major, u16 minor) +{ + xe_assert(xe, IS_SRIOV_VF(xe)); + + xe->sriov.vf.pf_version.major = major; + xe->sriov.vf.pf_version.minor = minor; +} + +static void vf_disconnect_pf(struct xe_device *xe) +{ + vf_connect_pf(xe, 0, 0); +} + +static int vf_handshake_with_pf(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 major_wanted = GUC_RELAY_VERSION_LATEST_MAJOR; + u32 minor_wanted = GUC_RELAY_VERSION_LATEST_MINOR; + u32 major = major_wanted, minor = minor_wanted; + int err; + + err = relay_action_handshake(gt, &major, &minor); + if (unlikely(err)) + goto failed; + + if (!major && !minor) { + err = -ENODATA; + goto failed; + } + + xe_gt_sriov_dbg(gt, "using VF/PF ABI %u.%u\n", major, minor); + vf_connect_pf(xe, major, minor); + return 0; + +failed: + xe_gt_sriov_err(gt, "Unable to confirm VF/PF ABI version %u.%u (%pe)\n", + major, minor, ERR_PTR(err)); + vf_disconnect_pf(xe); + return err; +} + +/** + * xe_gt_sriov_vf_connect - Establish connection with the PF driver. + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_vf_connect(struct xe_gt *gt) +{ + int err; + + err = vf_handshake_with_pf(gt); + if (unlikely(err)) + goto failed; + + return 0; + +failed: + xe_gt_sriov_err(gt, "Failed to get version info (%pe)\n", ERR_PTR(err)); + return err; +} + +/** + * xe_gt_sriov_vf_default_lrcs_hwsp_rebase - Update GGTT references in HWSP of default LRCs. + * @gt: the &xe_gt struct instance + */ +static void xe_gt_sriov_vf_default_lrcs_hwsp_rebase(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) + xe_default_lrc_update_memirq_regs_with_address(hwe); +} + +static void vf_post_migration_mark_fixups_done(struct xe_gt *gt) +{ + WRITE_ONCE(gt->sriov.vf.migration.ggtt_need_fixes, false); + smp_wmb(); /* Ensure above write visible before wake */ + wake_up_all(>->sriov.vf.migration.wq); +} + +static void vf_start_migration_recovery(struct xe_gt *gt) +{ + bool started; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + spin_lock(>->sriov.vf.migration.lock); + + if (!gt->sriov.vf.migration.recovery_queued && + !gt->sriov.vf.migration.recovery_teardown) { + gt->sriov.vf.migration.recovery_queued = true; + WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, true); + WRITE_ONCE(gt->sriov.vf.migration.ggtt_need_fixes, true); + smp_wmb(); /* Ensure above writes visible before wake */ + + xe_guc_ct_wake_waiters(>->uc.guc.ct); + + started = queue_work(gt->ordered_wq, >->sriov.vf.migration.worker); + xe_gt_sriov_info(gt, "VF migration recovery %s\n", started ? + "scheduled" : "already in progress"); + } + + spin_unlock(>->sriov.vf.migration.lock); +} + +/** + * xe_gt_sriov_vf_migrated_event_handler - Start a VF migration recovery, + * or just mark that a GuC is ready for it. + * @gt: the &xe_gt struct instance linked to target GuC + * + * This function shall be called only by VF. + */ +void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_VF(xe)); + xe_gt_assert(gt, xe_gt_sriov_vf_recovery_pending(gt)); + + if (!xe_sriov_vf_migration_supported(xe)) { + xe_gt_sriov_err(gt, "migration not supported\n"); + return; + } + + xe_gt_sriov_info(gt, "ready for recovery after migration\n"); + vf_start_migration_recovery(gt); +} + +static bool vf_is_negotiated(struct xe_gt *gt, u16 major, u16 minor) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_VF(xe)); + + return major == xe->sriov.vf.pf_version.major && + minor <= xe->sriov.vf.pf_version.minor; +} + +static int vf_prepare_runtime_info(struct xe_gt *gt, unsigned int num_regs) +{ + struct vf_runtime_reg *regs = gt->sriov.vf.runtime.regs; + unsigned int regs_size = round_up(num_regs, 4); + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_VF(xe)); + + if (regs) { + if (num_regs <= gt->sriov.vf.runtime.regs_size) { + memset(regs, 0, num_regs * sizeof(*regs)); + gt->sriov.vf.runtime.num_regs = num_regs; + return 0; + } + + drmm_kfree(&xe->drm, regs); + gt->sriov.vf.runtime.regs = NULL; + gt->sriov.vf.runtime.num_regs = 0; + gt->sriov.vf.runtime.regs_size = 0; + } + + regs = drmm_kcalloc(&xe->drm, regs_size, sizeof(*regs), GFP_KERNEL); + if (unlikely(!regs)) + return -ENOMEM; + + gt->sriov.vf.runtime.regs = regs; + gt->sriov.vf.runtime.num_regs = num_regs; + gt->sriov.vf.runtime.regs_size = regs_size; + return 0; +} + +static int vf_query_runtime_info(struct xe_gt *gt) +{ + u32 request[VF2PF_QUERY_RUNTIME_REQUEST_MSG_LEN]; + u32 response[VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN + 32]; /* up to 16 regs */ + u32 limit = (ARRAY_SIZE(response) - VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN) / 2; + u32 count, remaining, num, i; + u32 start = 0; + int ret; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, limit); + + /* this is part of the 1.0 PF/VF ABI */ + if (!vf_is_negotiated(gt, 1, 0)) + return -ENOPKG; + + request[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_RELAY_ACTION_VF2PF_QUERY_RUNTIME) | + FIELD_PREP(VF2PF_QUERY_RUNTIME_REQUEST_MSG_0_LIMIT, limit); + +repeat: + request[1] = FIELD_PREP(VF2PF_QUERY_RUNTIME_REQUEST_MSG_1_START, start); + ret = xe_guc_relay_send_to_pf(>->uc.guc.relay, + request, ARRAY_SIZE(request), + response, ARRAY_SIZE(response)); + if (unlikely(ret < 0)) + goto failed; + + if (unlikely(ret < VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN)) { + ret = -EPROTO; + goto failed; + } + if (unlikely((ret - VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN) % 2)) { + ret = -EPROTO; + goto failed; + } + + num = (ret - VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN) / 2; + count = FIELD_GET(VF2PF_QUERY_RUNTIME_RESPONSE_MSG_0_COUNT, response[0]); + remaining = FIELD_GET(VF2PF_QUERY_RUNTIME_RESPONSE_MSG_1_REMAINING, response[1]); + + xe_gt_sriov_dbg_verbose(gt, "count=%u num=%u ret=%d start=%u remaining=%u\n", + count, num, ret, start, remaining); + + if (unlikely(count != num)) { + ret = -EPROTO; + goto failed; + } + + if (start == 0) { + ret = vf_prepare_runtime_info(gt, num + remaining); + if (unlikely(ret < 0)) + goto failed; + } else if (unlikely(start + num > gt->sriov.vf.runtime.num_regs)) { + ret = -EPROTO; + goto failed; + } + + for (i = 0; i < num; ++i) { + struct vf_runtime_reg *reg = >->sriov.vf.runtime.regs[start + i]; + + reg->offset = response[VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN + 2 * i]; + reg->value = response[VF2PF_QUERY_RUNTIME_RESPONSE_MSG_MIN_LEN + 2 * i + 1]; + } + + if (remaining) { + start += num; + goto repeat; + } + + return 0; + +failed: + vf_prepare_runtime_info(gt, 0); + return ret; +} + +static void vf_show_runtime_info(struct xe_gt *gt) +{ + struct vf_runtime_reg *vf_regs = gt->sriov.vf.runtime.regs; + unsigned int size = gt->sriov.vf.runtime.num_regs; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + for (; size--; vf_regs++) + xe_gt_sriov_dbg(gt, "runtime(%#x) = %#x\n", + vf_regs->offset, vf_regs->value); +} + +/** + * xe_gt_sriov_vf_query_runtime - Query SR-IOV runtime data. + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_vf_query_runtime(struct xe_gt *gt) +{ + int err; + + err = vf_query_runtime_info(gt); + if (unlikely(err)) + goto failed; + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) + vf_show_runtime_info(gt); + + return 0; + +failed: + xe_gt_sriov_err(gt, "Failed to get runtime info (%pe)\n", + ERR_PTR(err)); + return err; +} + +static int vf_runtime_reg_cmp(const void *a, const void *b) +{ + const struct vf_runtime_reg *ra = a; + const struct vf_runtime_reg *rb = b; + + return (int)ra->offset - (int)rb->offset; +} + +static struct vf_runtime_reg *vf_lookup_reg(struct xe_gt *gt, u32 addr) +{ + struct xe_gt_sriov_vf_runtime *runtime = >->sriov.vf.runtime; + struct vf_runtime_reg key = { .offset = addr }; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + return bsearch(&key, runtime->regs, runtime->num_regs, sizeof(key), + vf_runtime_reg_cmp); +} + +/** + * xe_gt_sriov_vf_read32 - Get a register value from the runtime data. + * @gt: the &xe_gt + * @reg: the register to read + * + * This function is for VF use only. + * This function shall be called after VF has connected to PF. + * This function is dedicated for registers that VFs can't read directly. + * + * Return: register value obtained from the PF or 0 if not found. + */ +u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg) +{ + u32 addr = xe_mmio_adjusted_addr(>->mmio, reg.addr); + struct vf_runtime_reg *rr; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, !reg.vf); + + if (reg.addr == GMD_ID.addr) { + xe_gt_sriov_dbg_verbose(gt, "gmdid(%#x) = %#x\n", + addr, gt->sriov.vf.runtime.gmdid); + return gt->sriov.vf.runtime.gmdid; + } + + rr = vf_lookup_reg(gt, addr); + if (!rr) { + xe_gt_WARN(gt, IS_ENABLED(CONFIG_DRM_XE_DEBUG), + "VF is trying to read an inaccessible register %#x+%#x\n", + reg.addr, addr - reg.addr); + return 0; + } + + xe_gt_sriov_dbg_verbose(gt, "runtime[%#x] = %#x\n", addr, rr->value); + return rr->value; +} + +/** + * xe_gt_sriov_vf_write32 - Handle a write to an inaccessible register. + * @gt: the &xe_gt + * @reg: the register to write + * @val: value to write + * + * This function is for VF use only. + * Currently it will trigger a WARN if running on debug build. + */ +void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val) +{ + u32 addr = xe_mmio_adjusted_addr(>->mmio, reg.addr); + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + xe_gt_assert(gt, !reg.vf); + + /* + * In the future, we may want to handle selected writes to inaccessible + * registers in some custom way, but for now let's just log a warning + * about such attempt, as likely we might be doing something wrong. + */ + xe_gt_WARN(gt, IS_ENABLED(CONFIG_DRM_XE_DEBUG), + "VF is trying to write %#x to an inaccessible register %#x+%#x\n", + val, reg.addr, addr - reg.addr); +} + +/** + * xe_gt_sriov_vf_print_config() - Print VF self config. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * This function is for VF use only. + * + * Return: always 0. + */ +int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_gt_sriov_vf_selfconfig *config = >->sriov.vf.self_config; + struct xe_device *xe = gt_to_xe(gt); + u64 lmem_size; + char buf[10]; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + if (xe_gt_is_main_type(gt)) { + u64 ggtt_size = xe_tile_sriov_vf_ggtt(gt_to_tile(gt)); + u64 ggtt_base = xe_tile_sriov_vf_ggtt_base(gt_to_tile(gt)); + + drm_printf(p, "GGTT range:\t%#llx-%#llx\n", + ggtt_base, ggtt_base + ggtt_size - 1); + string_get_size(ggtt_size, 1, STRING_UNITS_2, buf, sizeof(buf)); + drm_printf(p, "GGTT size:\t%llu (%s)\n", ggtt_size, buf); + + if (IS_DGFX(xe)) { + lmem_size = xe_tile_sriov_vf_lmem(gt_to_tile(gt)); + string_get_size(lmem_size, 1, STRING_UNITS_2, buf, sizeof(buf)); + drm_printf(p, "LMEM size:\t%llu (%s)\n", lmem_size, buf); + } + } + + drm_printf(p, "GuC contexts:\t%u\n", config->num_ctxs); + drm_printf(p, "GuC doorbells:\t%u\n", config->num_dbs); + + return 0; +} + +/** + * xe_gt_sriov_vf_print_runtime() - Print VF's runtime regs received from PF. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * This function is for VF use only. + * + * Return: always 0. + */ +int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p) +{ + struct vf_runtime_reg *vf_regs = gt->sriov.vf.runtime.regs; + unsigned int size = gt->sriov.vf.runtime.num_regs; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + for (; size--; vf_regs++) + drm_printf(p, "%#x = %#x\n", vf_regs->offset, vf_regs->value); + + return 0; +} + +/** + * xe_gt_sriov_vf_print_version() - Print VF ABI versions. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * This function is for VF use only. + * + * Return: always 0. + */ +int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_uc_fw_version *guc_version = >->sriov.vf.guc_version; + struct xe_uc_fw_version *wanted = >->sriov.vf.wanted_guc_version; + struct xe_sriov_vf_relay_version *pf_version = &xe->sriov.vf.pf_version; + struct xe_uc_fw_version ver; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + drm_printf(p, "GuC ABI:\n"); + + vf_minimum_guc_version(gt, &ver); + drm_printf(p, "\tbase:\t%u.%u.%u.*\n", ver.branch, ver.major, ver.minor); + + drm_printf(p, "\twanted:\t%u.%u.%u.*\n", + wanted->branch, wanted->major, wanted->minor); + + drm_printf(p, "\thandshake:\t%u.%u.%u.%u\n", + guc_version->branch, guc_version->major, + guc_version->minor, guc_version->patch); + + drm_printf(p, "PF ABI:\n"); + + drm_printf(p, "\tbase:\t%u.%u\n", + GUC_RELAY_VERSION_BASE_MAJOR, GUC_RELAY_VERSION_BASE_MINOR); + drm_printf(p, "\twanted:\t%u.%u\n", + GUC_RELAY_VERSION_LATEST_MAJOR, GUC_RELAY_VERSION_LATEST_MINOR); + drm_printf(p, "\thandshake:\t%u.%u\n", + pf_version->major, pf_version->minor); + + return 0; +} + +static bool vf_post_migration_shutdown(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + /* + * On platforms where CCS must be restored by the primary GT, the media + * GT's VF post-migration recovery must run afterward. Detect this case + * and re-queue the media GT's restore work item if necessary. + */ + if (xe->info.needs_shared_vf_gt_wq && xe_gt_is_media_type(gt)) { + struct xe_gt *primary_gt = gt_to_tile(gt)->primary_gt; + + if (xe_gt_sriov_vf_recovery_pending(primary_gt)) + return true; + } + + xe_guc_ct_flush_and_stop(>->uc.guc.ct); + xe_guc_submit_pause_vf(>->uc.guc); + xe_tlb_inval_reset(>->tlb_inval); + + return false; +} + +static size_t post_migration_scratch_size(struct xe_device *xe) +{ + return max(xe_lrc_reg_size(xe), LRC_WA_BB_SIZE); +} + +static int vf_post_migration_fixups(struct xe_gt *gt) +{ + void *buf = gt->sriov.vf.migration.scratch; + int err; + + VF_MIGRATION_INJECT_WAIT(gt, FIXUPS); + + /* xe_gt_sriov_vf_query_config will fixup the GGTT addresses */ + err = xe_gt_sriov_vf_query_config(gt); + if (err) + return err; + + if (xe_gt_is_main_type(gt)) + xe_sriov_vf_ccs_rebase(gt_to_xe(gt)); + + xe_gt_sriov_vf_default_lrcs_hwsp_rebase(gt); + err = xe_guc_contexts_hwsp_rebase(>->uc.guc, buf); + if (err) + return err; + + atomic_inc(>->sriov.vf.migration.fixups_complete_count); + + return 0; +} + +static void vf_post_migration_rearm(struct xe_gt *gt) +{ + VF_MIGRATION_INJECT_WAIT(gt, RESTART_JOBS); + + /* + * Make sure interrupts on the new HW are properly set. The GuC IRQ + * must be working at this point, since the recovery did started, + * but the rest was not enabled using the procedure from spec. + */ + xe_irq_resume(gt_to_xe(gt)); + + xe_guc_ct_restart(>->uc.guc.ct); + xe_guc_submit_unpause_prepare_vf(>->uc.guc); +} + +static void vf_post_migration_kickstart(struct xe_gt *gt) +{ + xe_guc_submit_unpause_vf(>->uc.guc); +} + +static void vf_post_migration_abort(struct xe_gt *gt) +{ + spin_lock_irq(>->sriov.vf.migration.lock); + WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false); + WRITE_ONCE(gt->sriov.vf.migration.ggtt_need_fixes, false); + spin_unlock_irq(>->sriov.vf.migration.lock); + + wake_up_all(>->sriov.vf.migration.wq); + + xe_guc_submit_pause_abort(>->uc.guc); +} + +static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker) +{ + VF_MIGRATION_INJECT_WAIT(gt, RESFIX_DONE); + + spin_lock_irq(>->sriov.vf.migration.lock); + if (gt->sriov.vf.migration.recovery_queued) + xe_gt_sriov_dbg(gt, "another recovery imminent\n"); + else + WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false); + spin_unlock_irq(>->sriov.vf.migration.lock); + + return vf_resfix_done(gt, marker); +} + +static int vf_post_migration_resfix_start(struct xe_gt *gt, u16 marker) +{ + int err; + + err = vf_resfix_start(gt, marker); + + guard(spinlock_irq) (>->sriov.vf.migration.lock); + gt->sriov.vf.migration.recovery_queued = false; + + return err; +} + +static u16 vf_post_migration_next_resfix_marker(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + BUILD_BUG_ON(1 + ((typeof(gt->sriov.vf.migration.resfix_marker))~0) > + FIELD_MAX(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER)); + + /* add 1 to avoid zero-marker */ + return 1 + gt->sriov.vf.migration.resfix_marker++; +} + +static void vf_post_migration_recovery(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u16 marker; + bool retry; + int err; + + xe_gt_sriov_dbg(gt, "migration recovery in progress\n"); + + retry = vf_post_migration_shutdown(gt); + if (retry) + goto queue; + + if (!xe_sriov_vf_migration_supported(xe)) { + xe_gt_sriov_err(gt, "migration is not supported\n"); + err = -ENOTRECOVERABLE; + goto fail; + } + + marker = vf_post_migration_next_resfix_marker(gt); + + err = vf_post_migration_resfix_start(gt, marker); + if (unlikely(err)) { + xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_START step (%pe)\n", + ERR_PTR(err)); + goto fail; + } + + err = vf_post_migration_fixups(gt); + if (err) + goto fail; + + vf_post_migration_mark_fixups_done(gt); + vf_post_migration_rearm(gt); + + err = vf_post_migration_resfix_done(gt, marker); + if (err) { + if (err == -EREMCHG) + goto queue; + + xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_DONE step (%pe)\n", + ERR_PTR(err)); + goto fail; + } + + vf_post_migration_kickstart(gt); + + xe_gt_sriov_notice(gt, "migration recovery ended\n"); + return; +fail: + vf_post_migration_abort(gt); + xe_gt_sriov_err(gt, "migration recovery failed (%pe)\n", ERR_PTR(err)); + xe_device_declare_wedged(xe); + return; + +queue: + xe_gt_sriov_info(gt, "Re-queuing migration recovery\n"); + queue_work(gt->ordered_wq, >->sriov.vf.migration.worker); +} + +static void migration_worker_func(struct work_struct *w) +{ + struct xe_gt *gt = container_of(w, struct xe_gt, + sriov.vf.migration.worker); + + vf_post_migration_recovery(gt); +} + +static void vf_migration_fini(void *arg) +{ + struct xe_gt *gt = arg; + + spin_lock_irq(>->sriov.vf.migration.lock); + gt->sriov.vf.migration.recovery_teardown = true; + spin_unlock_irq(>->sriov.vf.migration.lock); + + cancel_work_sync(>->sriov.vf.migration.worker); +} + +/** + * xe_gt_sriov_vf_init_early() - GT VF init early + * @gt: the &xe_gt + * + * Return 0 on success, errno on failure + */ +int xe_gt_sriov_vf_init_early(struct xe_gt *gt) +{ + void *buf; + + if (!xe_sriov_vf_migration_supported(gt_to_xe(gt))) + return 0; + + buf = drmm_kmalloc(>_to_xe(gt)->drm, + post_migration_scratch_size(gt_to_xe(gt)), + GFP_KERNEL); + if (!buf) + return -ENOMEM; + + gt->sriov.vf.migration.scratch = buf; + spin_lock_init(>->sriov.vf.migration.lock); + INIT_WORK(>->sriov.vf.migration.worker, migration_worker_func); + init_waitqueue_head(>->sriov.vf.migration.wq); + + return 0; +} + +/** + * xe_gt_sriov_vf_init() - GT VF init + * @gt: the &xe_gt + * + * Return 0 on success, errno on failure + */ +int xe_gt_sriov_vf_init(struct xe_gt *gt) +{ + if (!xe_sriov_vf_migration_supported(gt_to_xe(gt))) + return 0; + + /* + * We want to tear down the VF post-migration early during driver + * unload; therefore, we add this finalization action later during + * driver load. + */ + return devm_add_action_or_reset(gt_to_xe(gt)->drm.dev, + vf_migration_fini, gt); +} + +/** + * xe_gt_sriov_vf_recovery_pending() - VF post migration recovery pending + * @gt: the &xe_gt + * + * The return value of this function must be immediately visible upon vCPU + * unhalt and must persist until RESFIX_DONE is issued. This guarantee is + * currently implemented only for platforms that support memirq. If non-memirq + * platforms begin to support VF migration, this function will need to be + * updated accordingly. + * + * Return: True if VF post migration recovery is pending, False otherwise + */ +bool xe_gt_sriov_vf_recovery_pending(struct xe_gt *gt) +{ + struct xe_memirq *memirq = >_to_tile(gt)->memirq; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + /* early detection until recovery starts */ + if (xe_device_uses_memirq(gt_to_xe(gt)) && + xe_memirq_guc_sw_int_0_irq_pending(memirq, >->uc.guc)) + return true; + + return READ_ONCE(gt->sriov.vf.migration.recovery_inprogress); +} + +static bool vf_valid_ggtt(struct xe_gt *gt) +{ + struct xe_memirq *memirq = >_to_tile(gt)->memirq; + bool irq_pending = xe_device_uses_memirq(gt_to_xe(gt)) && + xe_memirq_guc_sw_int_0_irq_pending(memirq, >->uc.guc); + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + if (irq_pending || READ_ONCE(gt->sriov.vf.migration.ggtt_need_fixes)) + return false; + + return true; +} + +/** + * xe_vf_migration_fixups_complete_count() - Get count of VF fixups completions. + * @gt: the &xe_gt instance which contains affected Global GTT + * + * Return: number of times VF fixups were completed since driver + * probe, or 0 if migration is not available, or -1 if fixups are + * pending or being applied right now. + */ +int xe_vf_migration_fixups_complete_count(struct xe_gt *gt) +{ + if (!IS_SRIOV_VF(gt_to_xe(gt)) || + !xe_sriov_vf_migration_supported(gt_to_xe(gt))) + return 0; + + /* should never match fixups_complete_count value */ + if (!vf_valid_ggtt(gt)) + return -1; + + return atomic_read(>->sriov.vf.migration.fixups_complete_count); +} + +/** + * xe_gt_sriov_vf_wait_valid_ggtt() - wait for valid GGTT nodes and address refs + * @gt: the &xe_gt instance which contains affected Global GTT + * + * Return: number of times VF fixups were completed since driver + * probe, or 0 if migration is not available. + */ +int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt) +{ + int ret; + + /* + * this condition needs to be identical to one in + * xe_vf_migration_fixups_complete_count() + */ + if (!IS_SRIOV_VF(gt_to_xe(gt)) || + !xe_sriov_vf_migration_supported(gt_to_xe(gt))) + return 0; + + ret = wait_event_interruptible_timeout(gt->sriov.vf.migration.wq, + vf_valid_ggtt(gt), + HZ * 5); + xe_gt_WARN_ON(gt, !ret); + + return atomic_read(>->sriov.vf.migration.fixups_complete_count); +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h new file mode 100644 index 000000000..d171a8242 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_VF_H_ +#define _XE_GT_SRIOV_VF_H_ + +#include + +struct drm_printer; +struct xe_gt; +struct xe_reg; +struct xe_uc_fw_version; + +int xe_gt_sriov_vf_reset(struct xe_gt *gt); +int xe_gt_sriov_vf_bootstrap(struct xe_gt *gt); +void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt, + struct xe_uc_fw_version *wanted, + struct xe_uc_fw_version *found); +int xe_gt_sriov_vf_query_config(struct xe_gt *gt); +int xe_gt_sriov_vf_connect(struct xe_gt *gt); +int xe_gt_sriov_vf_query_runtime(struct xe_gt *gt); +void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt); + +int xe_gt_sriov_vf_init_early(struct xe_gt *gt); +int xe_gt_sriov_vf_init(struct xe_gt *gt); +bool xe_gt_sriov_vf_recovery_pending(struct xe_gt *gt); + +u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt); +u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt); +u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt); +bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt); +u32 xe_gt_sriov_vf_paging_engines(struct xe_gt *gt); + +u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg); +void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val); + +int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p); + +int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt); +int xe_vf_migration_fixups_complete_count(struct xe_gt *gt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c new file mode 100644 index 000000000..507718326 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include + +#include + +#include "xe_gt_debugfs.h" +#include "xe_gt_sriov_vf.h" +#include "xe_gt_sriov_vf_debugfs.h" +#include "xe_gt_types.h" +#include "xe_sriov.h" + +/* + * /sys/kernel/debug/dri/0/ + * ├── gt0 + * │   ├── vf + * │   │   ├── self_config + * │   │   ├── abi_versions + * │   │   ├── runtime_regs + */ + +static const struct drm_info_list vf_info[] = { + { + "self_config", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_vf_print_config, + }, + { + "abi_versions", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_vf_print_version, + }, +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) || IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) + { + "runtime_regs", + .show = xe_gt_debugfs_simple_show, + .data = xe_gt_sriov_vf_print_runtime, + }, +#endif +}; + +/** + * xe_gt_sriov_vf_debugfs_register - Register SR-IOV VF specific entries in GT debugfs. + * @gt: the &xe_gt to register + * @root: the &dentry that represents the GT directory + * + * Register SR-IOV VF entries that are GT related and must be shown under GT debugfs. + */ +void xe_gt_sriov_vf_debugfs_register(struct xe_gt *gt, struct dentry *root) +{ + struct xe_device *xe = gt_to_xe(gt); + struct drm_minor *minor = xe->drm.primary; + struct dentry *vfdentry; + + xe_assert(xe, IS_SRIOV_VF(xe)); + xe_assert(xe, root->d_inode->i_private == gt); + + /* + * /sys/kernel/debug/dri/0/ + * ├── gt0 + * │   ├── vf + */ + vfdentry = debugfs_create_dir("vf", root); + if (IS_ERR(vfdentry)) + return; + vfdentry->d_inode->i_private = gt; + + drm_debugfs_create_files(vf_info, ARRAY_SIZE(vf_info), vfdentry, minor); + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── tile0 + * ├── gt0 + * ├── vf + * ├── resfix_stoppers + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + debugfs_create_x8("resfix_stoppers", 0600, vfdentry, + >->sriov.vf.migration.debug.resfix_stoppers); + } +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.h new file mode 100644 index 000000000..b2cff7ef5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_VF_DEBUGFS_H_ +#define _XE_GT_SRIOV_VF_DEBUGFS_H_ + +struct xe_gt; +struct dentry; + +void xe_gt_sriov_vf_debugfs_register(struct xe_gt *gt, struct dentry *root); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h new file mode 100644 index 000000000..466f0abd9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_VF_TYPES_H_ +#define _XE_GT_SRIOV_VF_TYPES_H_ + +#include +#include +#include +#include "xe_uc_fw_types.h" + +/** + * struct xe_gt_sriov_vf_selfconfig - VF configuration data. + */ +struct xe_gt_sriov_vf_selfconfig { + /** @num_ctxs: assigned number of GuC submission context IDs. */ + u16 num_ctxs; + /** @num_dbs: assigned number of GuC doorbells IDs. */ + u16 num_dbs; +}; + +/** + * struct xe_gt_sriov_vf_runtime - VF runtime data. + */ +struct xe_gt_sriov_vf_runtime { + /** @gmdid: cached value of the GDMID register. */ + u32 gmdid; + /** @uses_sched_groups: whether PF enabled sched groups or not. */ + bool uses_sched_groups; + /** + * @num_paging_engine_instances: number of configured paging engines. + */ + u32 num_paging_engine_instances; + /** @regs_size: size of runtime register array. */ + u32 regs_size; + /** @num_regs: number of runtime registers in the array. */ + u32 num_regs; + /** @regs: pointer to array of register offset/value pairs. */ + struct vf_runtime_reg { + /** @regs.offset: register offset. */ + u32 offset; + /** @regs.value: register value. */ + u32 value; + } *regs; +}; + +/** + * struct xe_gt_sriov_vf_migration - VF migration data. + */ +struct xe_gt_sriov_vf_migration { + /** @worker: VF migration recovery worker */ + struct work_struct worker; + /** @lock: Protects recovery_queued, teardown */ + spinlock_t lock; + /** @wq: wait queue for migration fixes */ + wait_queue_head_t wq; + /** @scratch: Scratch memory for VF recovery */ + void *scratch; + /** @fixups_complete_count: Counts completed fixups stages */ + atomic_t fixups_complete_count; + /** @debug: Debug hooks for delaying migration */ + struct { + /** + * @debug.resfix_stoppers: Stop and wait at different stages + * during post migration recovery + */ + u8 resfix_stoppers; + } debug; + /** + * @resfix_marker: Marker sent on start and on end of post-migration + * steps. + */ + u8 resfix_marker; + /** @recovery_teardown: VF post migration recovery is being torn down */ + bool recovery_teardown; + /** @recovery_queued: VF post migration recovery in queued */ + bool recovery_queued; + /** @recovery_inprogress: VF post migration recovery in progress */ + bool recovery_inprogress; + /** @ggtt_need_fixes: VF GGTT and references to it need fixes */ + bool ggtt_need_fixes; +}; + +/** + * struct xe_gt_sriov_vf - GT level VF virtualization data. + */ +struct xe_gt_sriov_vf { + /** @wanted_guc_version: minimum wanted GuC ABI version. */ + struct xe_uc_fw_version wanted_guc_version; + /** @guc_version: negotiated GuC ABI version. */ + struct xe_uc_fw_version guc_version; + /** @self_config: resource configurations. */ + struct xe_gt_sriov_vf_selfconfig self_config; + /** @runtime: runtime data retrieved from the PF. */ + struct xe_gt_sriov_vf_runtime runtime; + /** @migration: migration data for the VF. */ + struct xe_gt_sriov_vf_migration migration; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c new file mode 100644 index 000000000..789397514 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_stats.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_gt_stats.h" + +/** + * DOC: Xe GT Statistics + * + * Overview + * ======== + * + * The Xe driver exposes per-GT statistics through the debugfs filesystem at:: + * + * /sys/kernel/debug/dri//gt/stats + * + * This interface requires the kernel to be built with ``CONFIG_DEBUG_FS=y``. + * + * Reading statistics + * ================== + * + * Reading the file prints all available statistics, one per line, in + * ``name: value`` format:: + * + * $ cat /sys/kernel/debug/dri/0/gt0/stats + * svm_pagefault_count: 0 + * tlb_inval_count: 1234 + * ... + * + * All values are 64-bit unsigned integers aggregated across all CPUs. + * Counters accumulate since the driver was loaded or since the last explicit + * reset. Timing counters use microseconds as their unit; data volume counters + * use KiB. + * + * Resetting statistics + * ==================== + * + * Writing a boolean true value to the file resets all counters to zero:: + * + * echo 1 > /sys/kernel/debug/dri/0/gt0/stats + * + * Any value accepted by ``kstrtobool()`` (e.g. ``1``, ``y``, ``yes``, + * ``on``) triggers the reset. Resetting while the GPU is active may yield + * unpredictable intermediate values; it is recommended to reset only when + * the GPU is idle. + */ + +static void xe_gt_stats_fini(struct drm_device *drm, void *arg) +{ + struct xe_gt *gt = arg; + + free_percpu(gt->stats); +} + +/** + * xe_gt_stats_init() - Initialize GT statistics + * @gt: GT structure + * + * Allocate per-CPU GT statistics. Using per-CPU stats allows increments + * to occur without cross-CPU atomics. + * + * Return: 0 on success, -ENOMEM on failure. + */ +int xe_gt_stats_init(struct xe_gt *gt) +{ + gt->stats = alloc_percpu(struct xe_gt_stats); + if (!gt->stats) + return -ENOMEM; + + return drmm_add_action_or_reset(>_to_xe(gt)->drm, xe_gt_stats_fini, + gt); +} + +/** + * xe_gt_stats_incr - Increments the specified stats counter + * @gt: GT structure + * @id: xe_gt_stats_id type id that needs to be incremented + * @incr: value to be incremented with + * + * Increments the specified stats counter. + */ +void xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, int incr) +{ + if (id >= __XE_GT_STATS_NUM_IDS) + return; + + this_cpu_add(gt->stats->counters[id], incr); +} + +#define DEF_STAT_STR(ID, name) [XE_GT_STATS_ID_##ID] = name + +static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = { + DEF_STAT_STR(SVM_PAGEFAULT_COUNT, "svm_pagefault_count"), + DEF_STAT_STR(TLB_INVAL, "tlb_inval_count"), + DEF_STAT_STR(SVM_TLB_INVAL_COUNT, "svm_tlb_inval_count"), + DEF_STAT_STR(SVM_TLB_INVAL_US, "svm_tlb_inval_us"), + DEF_STAT_STR(VMA_PAGEFAULT_COUNT, "vma_pagefault_count"), + DEF_STAT_STR(VMA_PAGEFAULT_KB, "vma_pagefault_kb"), + DEF_STAT_STR(INVALID_PREFETCH_PAGEFAULT_COUNT, "invalid_prefetch_pagefault_count"), + DEF_STAT_STR(SVM_4K_PAGEFAULT_COUNT, "svm_4K_pagefault_count"), + DEF_STAT_STR(SVM_64K_PAGEFAULT_COUNT, "svm_64K_pagefault_count"), + DEF_STAT_STR(SVM_2M_PAGEFAULT_COUNT, "svm_2M_pagefault_count"), + DEF_STAT_STR(SVM_4K_VALID_PAGEFAULT_COUNT, "svm_4K_valid_pagefault_count"), + DEF_STAT_STR(SVM_64K_VALID_PAGEFAULT_COUNT, "svm_64K_valid_pagefault_count"), + DEF_STAT_STR(SVM_2M_VALID_PAGEFAULT_COUNT, "svm_2M_valid_pagefault_count"), + DEF_STAT_STR(SVM_4K_PAGEFAULT_US, "svm_4K_pagefault_us"), + DEF_STAT_STR(SVM_64K_PAGEFAULT_US, "svm_64K_pagefault_us"), + DEF_STAT_STR(SVM_2M_PAGEFAULT_US, "svm_2M_pagefault_us"), + DEF_STAT_STR(SVM_4K_MIGRATE_COUNT, "svm_4K_migrate_count"), + DEF_STAT_STR(SVM_64K_MIGRATE_COUNT, "svm_64K_migrate_count"), + DEF_STAT_STR(SVM_2M_MIGRATE_COUNT, "svm_2M_migrate_count"), + DEF_STAT_STR(SVM_4K_MIGRATE_US, "svm_4K_migrate_us"), + DEF_STAT_STR(SVM_64K_MIGRATE_US, "svm_64K_migrate_us"), + DEF_STAT_STR(SVM_2M_MIGRATE_US, "svm_2M_migrate_us"), + DEF_STAT_STR(SVM_DEVICE_COPY_US, "svm_device_copy_us"), + DEF_STAT_STR(SVM_4K_DEVICE_COPY_US, "svm_4K_device_copy_us"), + DEF_STAT_STR(SVM_64K_DEVICE_COPY_US, "svm_64K_device_copy_us"), + DEF_STAT_STR(SVM_2M_DEVICE_COPY_US, "svm_2M_device_copy_us"), + DEF_STAT_STR(SVM_CPU_COPY_US, "svm_cpu_copy_us"), + DEF_STAT_STR(SVM_4K_CPU_COPY_US, "svm_4K_cpu_copy_us"), + DEF_STAT_STR(SVM_64K_CPU_COPY_US, "svm_64K_cpu_copy_us"), + DEF_STAT_STR(SVM_2M_CPU_COPY_US, "svm_2M_cpu_copy_us"), + DEF_STAT_STR(SVM_DEVICE_COPY_KB, "svm_device_copy_kb"), + DEF_STAT_STR(SVM_4K_DEVICE_COPY_KB, "svm_4K_device_copy_kb"), + DEF_STAT_STR(SVM_64K_DEVICE_COPY_KB, "svm_64K_device_copy_kb"), + DEF_STAT_STR(SVM_2M_DEVICE_COPY_KB, "svm_2M_device_copy_kb"), + DEF_STAT_STR(SVM_CPU_COPY_KB, "svm_cpu_copy_kb"), + DEF_STAT_STR(SVM_4K_CPU_COPY_KB, "svm_4K_cpu_copy_kb"), + DEF_STAT_STR(SVM_64K_CPU_COPY_KB, "svm_64K_cpu_copy_kb"), + DEF_STAT_STR(SVM_2M_CPU_COPY_KB, "svm_2M_cpu_copy_kb"), + DEF_STAT_STR(SVM_4K_GET_PAGES_US, "svm_4K_get_pages_us"), + DEF_STAT_STR(SVM_64K_GET_PAGES_US, "svm_64K_get_pages_us"), + DEF_STAT_STR(SVM_2M_GET_PAGES_US, "svm_2M_get_pages_us"), + DEF_STAT_STR(SVM_4K_BIND_US, "svm_4K_bind_us"), + DEF_STAT_STR(SVM_64K_BIND_US, "svm_64K_bind_us"), + DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"), + DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, + "hw_engine_group_suspend_lr_queue_count"), + DEF_STAT_STR(HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, + "hw_engine_group_skip_lr_queue_count"), + DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, + "hw_engine_group_wait_dma_queue_count"), + DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, + "hw_engine_group_suspend_lr_queue_us"), + DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, + "hw_engine_group_wait_dma_queue_us"), + DEF_STAT_STR(PRL_4K_ENTRY_COUNT, "prl_4k_entry_count"), + DEF_STAT_STR(PRL_64K_ENTRY_COUNT, "prl_64k_entry_count"), + DEF_STAT_STR(PRL_2M_ENTRY_COUNT, "prl_2m_entry_count"), + DEF_STAT_STR(PRL_ISSUED_COUNT, "prl_issued_count"), + DEF_STAT_STR(PRL_ABORTED_COUNT, "prl_aborted_count"), +}; + +/** + * xe_gt_stats_print_info - Print the GT stats + * @gt: GT structure + * @p: drm_printer where it will be printed out. + * + * This prints out all the available GT stats. + */ +int xe_gt_stats_print_info(struct xe_gt *gt, struct drm_printer *p) +{ + enum xe_gt_stats_id id; + + for (id = 0; id < __XE_GT_STATS_NUM_IDS; ++id) { + u64 total = 0; + int cpu; + + for_each_possible_cpu(cpu) { + struct xe_gt_stats *s = per_cpu_ptr(gt->stats, cpu); + + total += s->counters[id]; + } + + drm_printf(p, "%s: %lld\n", stat_description[id], total); + } + + return 0; +} + +/** + * xe_gt_stats_clear() - Clear the GT stats + * @gt: GT structure + * + * Clear (zero) all available GT stats. Note that if the stats are being + * updated while this function is running, the results may be unpredictable. + * Intended to be called on an idle GPU. + */ +void xe_gt_stats_clear(struct xe_gt *gt) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct xe_gt_stats *s = per_cpu_ptr(gt->stats, cpu); + + memset(s, 0, sizeof(*s)); + } +} diff --git a/drivers/gpu/drm/xe/xe_gt_stats.h b/drivers/gpu/drm/xe/xe_gt_stats.h new file mode 100644 index 000000000..3d0defab9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_stats.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GT_STATS_H_ +#define _XE_GT_STATS_H_ + +#include + +#include "xe_gt_stats_types.h" + +struct xe_gt; +struct drm_printer; + +#ifdef CONFIG_DEBUG_FS +int xe_gt_stats_init(struct xe_gt *gt); +int xe_gt_stats_print_info(struct xe_gt *gt, struct drm_printer *p); +void xe_gt_stats_clear(struct xe_gt *gt); +void xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, int incr); +#else +static inline int xe_gt_stats_init(struct xe_gt *gt) +{ + return 0; +} + +static inline void +xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, + int incr) +{ +} + +#endif + +/** + * xe_gt_stats_ktime_us_delta() - Get delta in microseconds between now and a + * start time + * @start: Start time + * + * Helper for GT stats to get delta in microseconds between now and a start + * time, compiles out if GT stats are disabled. + * + * Return: Delta in microseconds between now and a start time + */ +static inline s64 xe_gt_stats_ktime_us_delta(ktime_t start) +{ + return IS_ENABLED(CONFIG_DEBUG_FS) ? + ktime_us_delta(ktime_get(), start) : 0; +} + +/** + * xe_gt_stats_ktime_get() - Get current ktime + * + * Helper for GT stats to get current ktime, compiles out if GT stats are + * disabled. + * + * Return: Get current ktime + */ +static inline ktime_t xe_gt_stats_ktime_get(void) +{ + return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h new file mode 100644 index 000000000..425491bed --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GT_STATS_TYPES_H_ +#define _XE_GT_STATS_TYPES_H_ + +#include + +/** + * enum xe_gt_stats_id - GT statistics identifiers + * @XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT: Total SVM page faults handled. + * @XE_GT_STATS_ID_TLB_INVAL: Total GPU Translation Lookaside Buffer (TLB) + * invalidations issued. + * @XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT: TLB invalidations issued during SVM + * page-fault handling. + * @XE_GT_STATS_ID_SVM_TLB_INVAL_US: Cumulative time (µs) waiting for TLB + * invalidations during SVM page-fault handling. + * + * @XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT: Buffer-object (non-SVM) page faults + * handled. + * @XE_GT_STATS_ID_VMA_PAGEFAULT_KB: Size (KiB) of VMAs involved in + * buffer-object page fault handling. + * @XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT: GPU prefetch faults for + * addresses with no valid backing. + * + * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 4K pages. + * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 64K pages. + * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 2M pages. + * @XE_GT_STATS_ID_SVM_4K_VALID_PAGEFAULT_COUNT: Valid SVM page faults + * at 4K page size, where the GPU mapping was already valid — resolved without + * creating new mappings. + * @XE_GT_STATS_ID_SVM_64K_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 64K + * page size. + * @XE_GT_STATS_ID_SVM_2M_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 2M + * page size. + * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_US: Cumulative time (µs) handling 4K SVM + * page faults. + * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_US: Cumulative time (µs) handling 64K + * SVM page faults. + * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_US: Cumulative time (µs) handling 2M SVM + * page faults. + * + * @XE_GT_STATS_ID_SVM_4K_MIGRATE_COUNT: 4K pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_64K_MIGRATE_COUNT: 64K pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_2M_MIGRATE_COUNT: 2M pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_4K_MIGRATE_US: Cumulative time (µs) moving 4K pages + * from CPU to device memory. + * @XE_GT_STATS_ID_SVM_64K_MIGRATE_US: Cumulative time (µs) moving 64K pages + * from CPU to device memory. + * @XE_GT_STATS_ID_SVM_2M_MIGRATE_US: Cumulative time (µs) moving 2M pages + * from CPU to device memory. + * + * @XE_GT_STATS_ID_SVM_DEVICE_COPY_US: Cumulative time (µs) for memory copies to + * device, across all page sizes. + * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US: Cumulative time (µs) for memory copies + * of 4K pages to device. + * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US: Cumulative time (µs) for memory + * copies of 64K pages to device. + * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US: Cumulative time (µs) for memory copies + * of 2M pages to device. + * @XE_GT_STATS_ID_SVM_CPU_COPY_US: Cumulative time (µs) for memory copies to + * CPU, across all page sizes. + * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_US: Cumulative time (µs) for memory copies of + * 4K pages to CPU. + * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_US: Cumulative time (µs) for memory copies + * of 64K pages to CPU. + * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_US: Cumulative time (µs) for memory copies of + * 2M pages to CPU. + * @XE_GT_STATS_ID_SVM_DEVICE_COPY_KB: Data (KiB) copied to device across all + * page sizes. + * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB: Data (KiB) copied to device for 4K + * pages. + * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB: Data (KiB) copied to device for + * 64K pages. + * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB: Data (KiB) copied to device for 2M + * pages. + * @XE_GT_STATS_ID_SVM_CPU_COPY_KB: Data (KiB) copied to CPU across all page + * sizes. + * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB: Data (KiB) copied to CPU for 4K pages. + * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB: Data (KiB) copied to CPU for 64K pages. + * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB: Data (KiB) copied to CPU for 2M pages. + * + * @XE_GT_STATS_ID_SVM_4K_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 4K page size. + * @XE_GT_STATS_ID_SVM_64K_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 64K page size. + * @XE_GT_STATS_ID_SVM_2M_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 2M page size. + * @XE_GT_STATS_ID_SVM_4K_BIND_US: Cumulative time (µs) binding 4K pages + * into the GPU page table. + * @XE_GT_STATS_ID_SVM_64K_BIND_US: Cumulative time (µs) binding 64K pages + * into the GPU page table. + * @XE_GT_STATS_ID_SVM_2M_BIND_US: Cumulative time (µs) binding 2M pages + * into the GPU page table. + * + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT: Times the + * scheduler preempted a long-running (LR) GPU exec queue. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT: Times the scheduler + * skipped suspend because the system was idle. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT: Times the driver + * stalled waiting for prior GPU work to complete before scheduling more. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US: Cumulative time + * (µs) spent preempting long-running (LR) GPU exec queues. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US: Cumulative time (µs) + * stalled waiting for prior GPU work to complete. + * + * @XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT: 4K-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT: 64K-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT: 2M-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_ISSUED_COUNT: Times a page reclamation was issued. + * @XE_GT_STATS_ID_PRL_ABORTED_COUNT: Times the page reclaim process was + * aborted. + * + * @__XE_GT_STATS_NUM_IDS: Number of valid IDs; not a real counter. + * + * See Documentation/gpu/xe/xe_gt_stats.rst. + */ +enum xe_gt_stats_id { + XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT, + XE_GT_STATS_ID_TLB_INVAL, + XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT, + XE_GT_STATS_ID_SVM_TLB_INVAL_US, + XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, + XE_GT_STATS_ID_VMA_PAGEFAULT_KB, + XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_2M_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_4K_VALID_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_64K_VALID_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_2M_VALID_PAGEFAULT_COUNT, + XE_GT_STATS_ID_SVM_4K_PAGEFAULT_US, + XE_GT_STATS_ID_SVM_64K_PAGEFAULT_US, + XE_GT_STATS_ID_SVM_2M_PAGEFAULT_US, + XE_GT_STATS_ID_SVM_4K_MIGRATE_COUNT, + XE_GT_STATS_ID_SVM_64K_MIGRATE_COUNT, + XE_GT_STATS_ID_SVM_2M_MIGRATE_COUNT, + XE_GT_STATS_ID_SVM_4K_MIGRATE_US, + XE_GT_STATS_ID_SVM_64K_MIGRATE_US, + XE_GT_STATS_ID_SVM_2M_MIGRATE_US, + XE_GT_STATS_ID_SVM_DEVICE_COPY_US, + XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US, + XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US, + XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US, + XE_GT_STATS_ID_SVM_CPU_COPY_US, + XE_GT_STATS_ID_SVM_4K_CPU_COPY_US, + XE_GT_STATS_ID_SVM_64K_CPU_COPY_US, + XE_GT_STATS_ID_SVM_2M_CPU_COPY_US, + XE_GT_STATS_ID_SVM_DEVICE_COPY_KB, + XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB, + XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB, + XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB, + XE_GT_STATS_ID_SVM_CPU_COPY_KB, + XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB, + XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB, + XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB, + XE_GT_STATS_ID_SVM_4K_GET_PAGES_US, + XE_GT_STATS_ID_SVM_64K_GET_PAGES_US, + XE_GT_STATS_ID_SVM_2M_GET_PAGES_US, + XE_GT_STATS_ID_SVM_4K_BIND_US, + XE_GT_STATS_ID_SVM_64K_BIND_US, + XE_GT_STATS_ID_SVM_2M_BIND_US, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, + XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, + XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, + XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, + XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, + XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT, + XE_GT_STATS_ID_PRL_ISSUED_COUNT, + XE_GT_STATS_ID_PRL_ABORTED_COUNT, + /* must be the last entry */ + __XE_GT_STATS_NUM_IDS, +}; + +/** + * struct xe_gt_stats - Per-CPU GT statistics counters + * @counters: Array of 64-bit counters indexed by &enum xe_gt_stats_id + * + * This structure is used for high-frequency, per-CPU statistics collection + * in the Xe driver. By using a per-CPU allocation and ensuring the structure + * is cache-line aligned, we avoid the performance-heavy atomics and cache + * coherency traffic. + * + * Updates to these counters should be performed using the this_cpu_add() + * macro to ensure they are atomic with respect to local interrupts and + * preemption-safe without the overhead of explicit locking. + */ +struct xe_gt_stats { + u64 counters[__XE_GT_STATS_NUM_IDS]; +} ____cacheline_aligned; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs.c b/drivers/gpu/drm/xe/xe_gt_sysfs.c new file mode 100644 index 000000000..7fc0bed6e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sysfs.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gt_sysfs.h" + +#include +#include + +#include + +#include "xe_gt_types.h" + +static void xe_gt_sysfs_kobj_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static const struct kobj_type xe_gt_sysfs_kobj_type = { + .release = xe_gt_sysfs_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, +}; + +static void gt_sysfs_fini(void *arg) +{ + struct xe_gt *gt = arg; + + kobject_put(gt->sysfs); +} + +int xe_gt_sysfs_init(struct xe_gt *gt) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct kobj_gt *kg; + int err; + + kg = kzalloc_obj(*kg); + if (!kg) + return -ENOMEM; + + kobject_init(&kg->base, &xe_gt_sysfs_kobj_type); + kg->gt = gt; + + err = kobject_add(&kg->base, tile->sysfs, "gt%d", gt->info.id); + if (err) { + kobject_put(&kg->base); + return err; + } + + gt->sysfs = &kg->base; + + return devm_add_action_or_reset(xe->drm.dev, gt_sysfs_fini, gt); +} diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs.h b/drivers/gpu/drm/xe/xe_gt_sysfs.h new file mode 100644 index 000000000..ecbfcc5c7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sysfs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_SYSFS_H_ +#define _XE_GT_SYSFS_H_ + +#include "xe_gt_sysfs_types.h" + +int xe_gt_sysfs_init(struct xe_gt *gt); + +static inline struct xe_gt * +kobj_to_gt(struct kobject *kobj) +{ + return container_of(kobj, struct kobj_gt, base)->gt; +} + +#endif /* _XE_GT_SYSFS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs_types.h b/drivers/gpu/drm/xe/xe_gt_sysfs_types.h new file mode 100644 index 000000000..d3bc6b833 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sysfs_types.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_SYSFS_TYPES_H_ +#define _XE_GT_SYSFS_TYPES_H_ + +#include + +struct xe_gt; + +/** + * struct kobj_gt - A GT's kobject struct that connects the kobject and the GT + * + * When dealing with multiple GTs, this struct helps to understand which GT + * needs to be addressed on a given sysfs call. + */ +struct kobj_gt { + /** @base: The actual kobject */ + struct kobject base; + /** @gt: A pointer to the GT itself */ + struct xe_gt *gt; +}; + +#endif /* _XE_GT_SYSFS_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c new file mode 100644 index 000000000..1e7e3a31a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_throttle.c @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include +#include "xe_device_types.h" +#include "xe_gt.h" +#include "xe_gt_sysfs.h" +#include "xe_gt_throttle.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "xe_pm.h" + +/** + * DOC: Xe GT Throttle + * + * The GT frequency may be throttled by hardware/firmware for various reasons + * that are provided through attributes under the ``freq0/throttle/`` directory. + * Their availability depend on the platform and some may not be visible if that + * reason is not available. + * + * The ``reasons`` attribute can be used by sysadmin to monitor all possible + * reasons for throttling and report them. It's preferred over monitoring + * ``status`` and then reading the reason from individual attributes since that + * is racy. If there's no throttling happening, "none" is returned. + * + * The following attributes are available on Crescent Island platform: + * + * - ``status``: Overall throttle status (0: no throttling, 1: throttling) + * - ``reasons``: Array of reasons causing throttling separated by space + * - ``reason_pl1``: package PL1 + * - ``reason_pl2``: package PL2 + * - ``reason_pl4``: package PL4 + * - ``reason_prochot``: prochot + * - ``reason_soc_thermal``: SoC thermal + * - ``reason_mem_thermal``: Memory thermal + * - ``reason_vr_thermal``: VR thermal + * - ``reason_iccmax``: ICCMAX + * - ``reason_ratl``: RATL thermal algorithm + * - ``reason_soc_avg_thermal``: SoC average temp + * - ``reason_fastvmode``: VR is hitting FastVMode + * - ``reason_psys_pl1``: PSYS PL1 + * - ``reason_psys_pl2``: PSYS PL2 + * - ``reason_p0_freq``: P0 frequency + * - ``reason_psys_crit``: PSYS critical + * + * Other platforms support the following reasons: + * + * - ``status``: Overall throttle status (0: no throttling, 1: throttling) + * - ``reasons``: Array of reasons causing throttling separated by space + * - ``reason_pl1``: package PL1 + * - ``reason_pl2``: package PL2 + * - ``reason_pl4``: package PL4, Iccmax etc. + * - ``reason_thermal``: thermal + * - ``reason_prochot``: prochot + * - ``reason_ratl``: RATL hermal algorithm + * - ``reason_vr_thermalert``: VR THERMALERT + * - ``reason_vr_tdc``: VR TDC + */ + +struct throttle_attribute { + struct kobj_attribute attr; + u32 mask; +}; + +static struct xe_gt *dev_to_gt(struct device *dev) +{ + return kobj_to_gt(dev->kobj.parent); +} + +static struct xe_gt *throttle_to_gt(struct kobject *kobj) +{ + return dev_to_gt(kobj_to_dev(kobj)); +} + +static struct throttle_attribute *kobj_attribute_to_throttle(struct kobj_attribute *attr) +{ + return container_of(attr, struct throttle_attribute, attr); +} + +u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_reg reg; + u32 mask; + + if (xe_gt_is_media_type(gt)) + reg = MTL_MEDIA_PERF_LIMIT_REASONS; + else + reg = GT0_PERF_LIMIT_REASONS; + + if (xe->info.platform == XE_CRESCENTISLAND) + mask = CRI_PERF_LIMIT_REASONS_MASK; + else + mask = GT0_PERF_LIMIT_REASONS_MASK; + + guard(xe_pm_runtime)(xe); + return xe_mmio_read32(>->mmio, reg) & mask; +} + +static bool is_throttled_by(struct xe_gt *gt, u32 mask) +{ + return xe_gt_throttle_get_limit_reasons(gt) & mask; +} + +static ssize_t reason_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct throttle_attribute *ta = kobj_attribute_to_throttle(attr); + struct xe_gt *gt = throttle_to_gt(kobj); + + return sysfs_emit(buff, "%u\n", is_throttled_by(gt, ta->mask)); +} + +static const struct attribute_group *get_platform_throttle_group(struct xe_device *xe); + +static ssize_t reasons_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct xe_gt *gt = throttle_to_gt(kobj); + struct xe_device *xe = gt_to_xe(gt); + const struct attribute_group *group; + struct attribute **pother; + ssize_t ret = 0; + u32 reasons; + + reasons = xe_gt_throttle_get_limit_reasons(gt); + if (!reasons) + goto ret_none; + + group = get_platform_throttle_group(xe); + for (pother = group->attrs; *pother; pother++) { + struct kobj_attribute *kattr = container_of(*pother, struct kobj_attribute, attr); + struct throttle_attribute *other_ta = kobj_attribute_to_throttle(kattr); + + if (other_ta->mask != U32_MAX && reasons & other_ta->mask) + ret += sysfs_emit_at(buff, ret, "%s ", (*pother)->name + strlen("reason_")); + } + + if (drm_WARN_ONCE(&xe->drm, !ret, "Unknown reason: %#x\n", reasons)) + goto ret_none; + + /* Drop extra space from last iteration above */ + ret--; + ret += sysfs_emit_at(buff, ret, "\n"); + + return ret; + +ret_none: + return sysfs_emit(buff, "none\n"); +} + +#define THROTTLE_ATTR_RO(name, _mask) \ + struct throttle_attribute attr_##name = { \ + .attr = __ATTR(name, 0444, reason_show, NULL), \ + .mask = _mask, \ + } + +#define THROTTLE_ATTR_RO_FUNC(name, _mask, _show) \ + struct throttle_attribute attr_##name = { \ + .attr = __ATTR(name, 0444, _show, NULL), \ + .mask = _mask, \ + } + +static THROTTLE_ATTR_RO_FUNC(reasons, 0, reasons_show); +static THROTTLE_ATTR_RO(status, U32_MAX); +static THROTTLE_ATTR_RO(reason_pl1, POWER_LIMIT_1_MASK); +static THROTTLE_ATTR_RO(reason_pl2, POWER_LIMIT_2_MASK); +static THROTTLE_ATTR_RO(reason_pl4, POWER_LIMIT_4_MASK); +static THROTTLE_ATTR_RO(reason_thermal, THERMAL_LIMIT_MASK); +static THROTTLE_ATTR_RO(reason_prochot, PROCHOT_MASK); +static THROTTLE_ATTR_RO(reason_ratl, RATL_MASK); +static THROTTLE_ATTR_RO(reason_vr_thermalert, VR_THERMALERT_MASK); +static THROTTLE_ATTR_RO(reason_vr_tdc, VR_TDC_MASK); + +static struct attribute *throttle_attrs[] = { + &attr_reasons.attr.attr, + &attr_status.attr.attr, + &attr_reason_pl1.attr.attr, + &attr_reason_pl2.attr.attr, + &attr_reason_pl4.attr.attr, + &attr_reason_thermal.attr.attr, + &attr_reason_prochot.attr.attr, + &attr_reason_ratl.attr.attr, + &attr_reason_vr_thermalert.attr.attr, + &attr_reason_vr_tdc.attr.attr, + NULL +}; + +static THROTTLE_ATTR_RO(reason_vr_thermal, VR_THERMAL_MASK); +static THROTTLE_ATTR_RO(reason_soc_thermal, SOC_THERMAL_LIMIT_MASK); +static THROTTLE_ATTR_RO(reason_mem_thermal, MEM_THERMAL_MASK); +static THROTTLE_ATTR_RO(reason_iccmax, ICCMAX_MASK); +static THROTTLE_ATTR_RO(reason_soc_avg_thermal, SOC_AVG_THERMAL_MASK); +static THROTTLE_ATTR_RO(reason_fastvmode, FASTVMODE_MASK); +static THROTTLE_ATTR_RO(reason_psys_pl1, PSYS_PL1_MASK); +static THROTTLE_ATTR_RO(reason_psys_pl2, PSYS_PL2_MASK); +static THROTTLE_ATTR_RO(reason_p0_freq, P0_FREQ_MASK); +static THROTTLE_ATTR_RO(reason_psys_crit, PSYS_CRIT_MASK); + +static struct attribute *cri_throttle_attrs[] = { + /* Common */ + &attr_reasons.attr.attr, + &attr_status.attr.attr, + &attr_reason_pl1.attr.attr, + &attr_reason_pl2.attr.attr, + &attr_reason_pl4.attr.attr, + &attr_reason_prochot.attr.attr, + &attr_reason_ratl.attr.attr, + /* CRI */ + &attr_reason_vr_thermal.attr.attr, + &attr_reason_soc_thermal.attr.attr, + &attr_reason_mem_thermal.attr.attr, + &attr_reason_iccmax.attr.attr, + &attr_reason_soc_avg_thermal.attr.attr, + &attr_reason_fastvmode.attr.attr, + &attr_reason_psys_pl1.attr.attr, + &attr_reason_psys_pl2.attr.attr, + &attr_reason_p0_freq.attr.attr, + &attr_reason_psys_crit.attr.attr, + NULL +}; + +static const struct attribute_group throttle_group_attrs = { + .name = "throttle", + .attrs = throttle_attrs, +}; + +static const struct attribute_group cri_throttle_group_attrs = { + .name = "throttle", + .attrs = cri_throttle_attrs, +}; + +static const struct attribute_group *get_platform_throttle_group(struct xe_device *xe) +{ + switch (xe->info.platform) { + case XE_CRESCENTISLAND: + return &cri_throttle_group_attrs; + default: + return &throttle_group_attrs; + } +} + +static void gt_throttle_sysfs_fini(void *arg) +{ + struct xe_gt *gt = arg; + struct xe_device *xe = gt_to_xe(gt); + const struct attribute_group *group = get_platform_throttle_group(xe); + + sysfs_remove_group(gt->freq, group); +} + +int xe_gt_throttle_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + const struct attribute_group *group = get_platform_throttle_group(xe); + int err; + + err = sysfs_create_group(gt->freq, group); + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, gt_throttle_sysfs_fini, gt); +} diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.h b/drivers/gpu/drm/xe/xe_gt_throttle.h new file mode 100644 index 000000000..022774947 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_throttle.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_THROTTLE_H_ +#define _XE_GT_THROTTLE_H_ + +#include + +struct xe_gt; + +int xe_gt_throttle_init(struct xe_gt *gt); + +u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt); + +#endif /* _XE_GT_THROTTLE_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_topology.c b/drivers/gpu/drm/xe/xe_gt_topology.c new file mode 100644 index 000000000..bfe87e682 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_topology.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gt_topology.h" + +#include +#include +#include + +#include "regs/xe_gt_regs.h" +#include "xe_assert.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_mmio.h" +#include "xe_wa.h" + +static void load_dss_mask(struct xe_gt *gt, xe_dss_mask_t mask, int numregs, + const struct xe_reg regs[]) +{ + u32 fuse_val[XE_MAX_DSS_FUSE_REGS] = {}; + int i; + + xe_gt_assert(gt, numregs <= ARRAY_SIZE(fuse_val)); + + for (i = 0; i < numregs; i++) + fuse_val[i] = xe_mmio_read32(>->mmio, regs[i]); + + bitmap_from_arr32(mask, fuse_val, numregs * 32); +} + +static void +load_eu_mask(struct xe_gt *gt, xe_eu_mask_t mask, enum xe_gt_eu_type *eu_type) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 reg_val = xe_mmio_read32(>->mmio, XELP_EU_ENABLE); + u32 val = 0; + int i; + + BUILD_BUG_ON(XE_MAX_EU_FUSE_REGS > 1); + + /* + * Pre-Xe_HP platforms inverted the bit meaning (disable instead + * of enable). + */ + if (GRAPHICS_VERx100(xe) < 1250) + reg_val = ~reg_val & XELP_EU_MASK; + + if (GRAPHICS_VERx100(xe) == 1260 || GRAPHICS_VER(xe) >= 20) { + /* SIMD16 EUs, one bit == one EU */ + *eu_type = XE_GT_EU_TYPE_SIMD16; + val = reg_val; + } else { + /* SIMD8 EUs, one bit == 2 EU */ + *eu_type = XE_GT_EU_TYPE_SIMD8; + for (i = 0; i < fls(reg_val); i++) + if (reg_val & BIT(i)) + val |= 0x3 << 2 * i; + } + + bitmap_from_arr32(mask, &val, XE_MAX_EU_FUSE_BITS); +} + +/** + * gen_l3_mask_from_pattern - Replicate a bit pattern according to a mask + * + * It is used to compute the L3 bank masks in a generic format on + * various platforms where the internal representation of L3 node + * and masks from registers are different. + * + * @xe: device + * @dst: destination + * @pattern: pattern to replicate + * @patternbits: size of the pattern, in bits + * @mask: mask describing where to replicate the pattern + * + * Example 1: + * ---------- + * @pattern = 0b1111 + * └┬─┘ + * @patternbits = 4 (bits) + * @mask = 0b0101 + * ││││ + * │││└────────────────── 0b1111 (=1×0b1111) + * ││└──────────── 0b0000 │ (=0×0b1111) + * │└────── 0b1111 │ │ (=1×0b1111) + * └ 0b0000 │ │ │ (=0×0b1111) + * │ │ │ │ + * @dst = 0b0000 0b1111 0b0000 0b1111 + * + * Example 2: + * ---------- + * @pattern = 0b11111111 + * └┬─────┘ + * @patternbits = 8 (bits) + * @mask = 0b10 + * ││ + * ││ + * ││ + * │└────────── 0b00000000 (=0×0b11111111) + * └ 0b11111111 │ (=1×0b11111111) + * │ │ + * @dst = 0b11111111 0b00000000 + */ +static void +gen_l3_mask_from_pattern(struct xe_device *xe, xe_l3_bank_mask_t dst, + xe_l3_bank_mask_t pattern, int patternbits, + unsigned long mask) +{ + unsigned long bit; + + xe_assert(xe, find_last_bit(pattern, XE_MAX_L3_BANK_MASK_BITS) < patternbits || + bitmap_empty(pattern, XE_MAX_L3_BANK_MASK_BITS)); + xe_assert(xe, !mask || patternbits * (__fls(mask) + 1) <= XE_MAX_L3_BANK_MASK_BITS); + for_each_set_bit(bit, &mask, 32) { + xe_l3_bank_mask_t shifted_pattern = {}; + + bitmap_shift_left(shifted_pattern, pattern, bit * patternbits, + XE_MAX_L3_BANK_MASK_BITS); + bitmap_or(dst, dst, shifted_pattern, XE_MAX_L3_BANK_MASK_BITS); + } +} + +bool xe_gt_topology_report_l3(struct xe_gt *gt) +{ + /* + * No known userspace needs/uses the L3 bank mask reported by + * the media GT, and the hardware itself is known to report bogus + * values on several platforms. Only report L3 bank mask as part + * of the media GT's topology on pre-Xe3 platforms since that's + * already part of our ABI. + */ + if (xe_gt_is_media_type(gt) && MEDIA_VER(gt_to_xe(gt)) >= 30) + return false; + + return true; +} + +static void +load_l3_bank_mask(struct xe_gt *gt, xe_l3_bank_mask_t l3_bank_mask) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_mmio *mmio = >->mmio; + u32 fuse3 = xe_mmio_read32(mmio, MIRROR_FUSE3); + + if (!xe_gt_topology_report_l3(gt)) + return; + + if (GRAPHICS_VER(xe) >= 35) { + u32 fuse_val = xe_mmio_read32(mmio, MIRROR_L3BANK_ENABLE); + + bitmap_from_arr32(l3_bank_mask, &fuse_val, 32); + } else if (GRAPHICS_VER(xe) >= 30) { + xe_l3_bank_mask_t per_node = {}; + u32 meml3_en = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, fuse3); + u32 mirror_l3bank_enable = xe_mmio_read32(mmio, MIRROR_L3BANK_ENABLE); + u32 bank_val = REG_FIELD_GET(XE3_L3BANK_ENABLE, mirror_l3bank_enable); + + bitmap_from_arr32(per_node, &bank_val, 32); + gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 32, + meml3_en); + } else if (GRAPHICS_VER(xe) >= 20) { + xe_l3_bank_mask_t per_node = {}; + u32 meml3_en = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, fuse3); + u32 bank_val = REG_FIELD_GET(XE2_GT_L3_MODE_MASK, fuse3); + + bitmap_from_arr32(per_node, &bank_val, 32); + gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 4, + meml3_en); + } else if (GRAPHICS_VERx100(xe) >= 1270) { + xe_l3_bank_mask_t per_node = {}; + xe_l3_bank_mask_t per_mask_bit = {}; + u32 meml3_en = REG_FIELD_GET(MEML3_EN_MASK, fuse3); + u32 fuse4 = xe_mmio_read32(mmio, XEHP_FUSE4); + u32 bank_val = REG_FIELD_GET(GT_L3_EXC_MASK, fuse4); + + bitmap_set_value8(per_mask_bit, 0x3, 0); + gen_l3_mask_from_pattern(xe, per_node, per_mask_bit, 2, bank_val); + gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 4, + meml3_en); + } else if (xe->info.platform == XE_PVC) { + xe_l3_bank_mask_t per_node = {}; + xe_l3_bank_mask_t per_mask_bit = {}; + u32 meml3_en = REG_FIELD_GET(MEML3_EN_MASK, fuse3); + u32 bank_val = REG_FIELD_GET(XEHPC_GT_L3_MODE_MASK, fuse3); + + bitmap_set_value8(per_mask_bit, 0xf, 0); + gen_l3_mask_from_pattern(xe, per_node, per_mask_bit, 4, + bank_val); + gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 16, + meml3_en); + } else if (xe->info.platform == XE_DG2) { + xe_l3_bank_mask_t per_node = {}; + u32 mask = REG_FIELD_GET(MEML3_EN_MASK, fuse3); + + bitmap_set_value8(per_node, 0xff, 0); + gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 8, mask); + } else { + /* 1:1 register bit to mask bit (inverted register bits) */ + u32 mask = REG_FIELD_GET(XELP_GT_L3_MODE_MASK, ~fuse3); + + bitmap_from_arr32(l3_bank_mask, &mask, 32); + } +} + +void +xe_gt_topology_init(struct xe_gt *gt) +{ + static const struct xe_reg geometry_regs[] = { + XELP_GT_GEOMETRY_DSS_ENABLE, + XE2_GT_GEOMETRY_DSS_1, + XE2_GT_GEOMETRY_DSS_2, + XE3P_XPC_GT_GEOMETRY_DSS_3, + }; + static const struct xe_reg compute_regs[] = { + XEHP_GT_COMPUTE_DSS_ENABLE, + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT, + XE2_GT_COMPUTE_DSS_2, + XE3P_XPC_GT_COMPUTE_DSS_3, + }; + struct drm_printer p; + + /* + * Register counts returned shouldn't exceed the number of registers + * passed as parameters below. + */ + xe_gt_assert(gt, gt->info.num_geometry_xecore_fuse_regs <= ARRAY_SIZE(geometry_regs)); + xe_gt_assert(gt, gt->info.num_compute_xecore_fuse_regs <= ARRAY_SIZE(compute_regs)); + + load_dss_mask(gt, gt->fuse_topo.g_dss_mask, + gt->info.num_geometry_xecore_fuse_regs, geometry_regs); + load_dss_mask(gt, gt->fuse_topo.c_dss_mask, + gt->info.num_compute_xecore_fuse_regs, compute_regs); + + load_eu_mask(gt, gt->fuse_topo.eu_mask_per_dss, >->fuse_topo.eu_type); + load_l3_bank_mask(gt, gt->fuse_topo.l3_bank_mask); + + p = xe_gt_dbg_printer(gt); + xe_gt_topology_dump(gt, &p); +} + +static const char *eu_type_to_str(enum xe_gt_eu_type eu_type) +{ + switch (eu_type) { + case XE_GT_EU_TYPE_SIMD16: + return "simd16"; + case XE_GT_EU_TYPE_SIMD8: + return "simd8"; + } + + return NULL; +} + +/** + * xe_gt_topology_dump() - Dump GT topology into a drm printer. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Return: always 0. + */ +int xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p) +{ + drm_printf(p, "dss mask (geometry): %*pb\n", XE_MAX_DSS_FUSE_BITS, + gt->fuse_topo.g_dss_mask); + drm_printf(p, "dss mask (compute): %*pb\n", XE_MAX_DSS_FUSE_BITS, + gt->fuse_topo.c_dss_mask); + + drm_printf(p, "EU mask per DSS: %*pb\n", XE_MAX_EU_FUSE_BITS, + gt->fuse_topo.eu_mask_per_dss); + drm_printf(p, "EU type: %s\n", + eu_type_to_str(gt->fuse_topo.eu_type)); + + if (xe_gt_topology_report_l3(gt)) + drm_printf(p, "L3 bank mask: %*pb\n", XE_MAX_L3_BANK_MASK_BITS, + gt->fuse_topo.l3_bank_mask); + return 0; +} + +/* + * Used to obtain the index of the first DSS. Can start searching from the + * beginning of a specific dss group (e.g., gslice, cslice, etc.) if + * groupsize and groupnum are non-zero. + */ +unsigned int +xe_dss_mask_group_ffs(const xe_dss_mask_t mask, int groupsize, int groupnum) +{ + return find_next_bit(mask, XE_MAX_DSS_FUSE_BITS, groupnum * groupsize); +} + +/* Used to obtain the index of the first L3 bank. */ +unsigned int +xe_l3_bank_mask_ffs(const xe_l3_bank_mask_t mask) +{ + return find_first_bit(mask, XE_MAX_L3_BANK_MASK_BITS); +} + +/** + * xe_gt_topology_has_dss_in_quadrant - check fusing of DSS in GT quadrant + * @gt: GT to check + * @quad: Which quadrant of the DSS space to check + * + * Since Xe_HP platforms can have up to four CCS engines, those engines + * are each logically associated with a quarter of the possible DSS. If there + * are no DSS present in one of the four quadrants of the DSS space, the + * corresponding CCS engine is also not available for use. + * + * Returns false if all DSS in a quadrant of the GT are fused off, else true. + */ +bool xe_gt_topology_has_dss_in_quadrant(struct xe_gt *gt, int quad) +{ + xe_dss_mask_t all_dss; + int dss_per_quad, quad_first; + + bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS); + + dss_per_quad = 32 * max(gt->info.num_geometry_xecore_fuse_regs, + gt->info.num_compute_xecore_fuse_regs) / 4; + + quad_first = xe_dss_mask_group_ffs(all_dss, dss_per_quad, quad); + + return quad_first < (quad + 1) * dss_per_quad; +} + +bool xe_gt_has_geometry_dss(struct xe_gt *gt, unsigned int dss) +{ + return test_bit(dss, gt->fuse_topo.g_dss_mask); +} + +bool xe_gt_has_compute_dss(struct xe_gt *gt, unsigned int dss) +{ + return test_bit(dss, gt->fuse_topo.c_dss_mask); +} + +bool xe_gt_has_discontiguous_dss_groups(const struct xe_gt *gt) +{ + unsigned int xecore; + int last_group = -1; + u16 group, instance; + + for_each_dss_steering(xecore, gt, group, instance) { + if (last_group != group) { + if (group - last_group > 1) + return true; + last_group = group; + } + } + return false; +} diff --git a/drivers/gpu/drm/xe/xe_gt_topology.h b/drivers/gpu/drm/xe/xe_gt_topology.h new file mode 100644 index 000000000..162d603c9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_topology.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_TOPOLOGY_H_ +#define _XE_GT_TOPOLOGY_H_ + +#include "xe_gt_types.h" + +/* + * Loop over each DSS with the bit is 1 in geometry or compute mask + * @dss: iterated DSS bit from the DSS mask + * @gt: GT structure + */ +#define for_each_dss(dss, gt) \ + for_each_or_bit((dss), \ + (gt)->fuse_topo.g_dss_mask, \ + (gt)->fuse_topo.c_dss_mask, \ + XE_MAX_DSS_FUSE_BITS) + +struct drm_printer; + +void xe_gt_topology_init(struct xe_gt *gt); + +int xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p); + +/** + * xe_gt_topology_mask_last_dss() - Returns the index of the last DSS in a mask. + * @mask: Input DSS mask + * + * Return: Index of the last DSS in the input DSS mask, + * XE_MAX_DSS_FUSE_BITS if DSS mask is empty. + */ +static inline unsigned int +xe_gt_topology_mask_last_dss(const xe_dss_mask_t mask) +{ + return find_last_bit(mask, XE_MAX_DSS_FUSE_BITS); +} + +unsigned int +xe_dss_mask_group_ffs(const xe_dss_mask_t mask, int groupsize, int groupnum); +unsigned int +xe_l3_bank_mask_ffs(const xe_l3_bank_mask_t mask); + +bool +xe_gt_topology_has_dss_in_quadrant(struct xe_gt *gt, int quad); + +bool xe_gt_has_geometry_dss(struct xe_gt *gt, unsigned int dss); +bool xe_gt_has_compute_dss(struct xe_gt *gt, unsigned int dss); + +bool xe_gt_has_discontiguous_dss_groups(const struct xe_gt *gt); + +bool xe_gt_topology_report_l3(struct xe_gt *gt); + +#endif /* _XE_GT_TOPOLOGY_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h new file mode 100644 index 000000000..a8bbfbdf3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -0,0 +1,395 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022-2023 Intel Corporation + */ + +#ifndef _XE_GT_TYPES_H_ +#define _XE_GT_TYPES_H_ + +#include "xe_device_types.h" +#include "xe_force_wake_types.h" +#include "xe_gt_idle_types.h" +#include "xe_gt_sriov_pf_types.h" +#include "xe_gt_sriov_vf_types.h" +#include "xe_gt_stats_types.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence_types.h" +#include "xe_oa_types.h" +#include "xe_reg_sr_types.h" +#include "xe_sa_types.h" +#include "xe_tlb_inval_types.h" +#include "xe_uc_types.h" + +struct xe_exec_queue_ops; +struct xe_migrate; +struct xe_ring_ops; + +enum xe_gt_type { + XE_GT_TYPE_UNINITIALIZED, + XE_GT_TYPE_MAIN, + XE_GT_TYPE_MEDIA, +}; + +enum xe_gt_eu_type { + XE_GT_EU_TYPE_SIMD8, + XE_GT_EU_TYPE_SIMD16, +}; + +#define XE_MAX_DSS_FUSE_REGS 4 +#define XE_MAX_DSS_FUSE_BITS (32 * XE_MAX_DSS_FUSE_REGS) +#define XE_MAX_EU_FUSE_REGS 1 +#define XE_MAX_EU_FUSE_BITS (32 * XE_MAX_EU_FUSE_REGS) +#define XE_MAX_L3_BANK_MASK_BITS 64 + +typedef unsigned long xe_dss_mask_t[BITS_TO_LONGS(XE_MAX_DSS_FUSE_BITS)]; +typedef unsigned long xe_eu_mask_t[BITS_TO_LONGS(XE_MAX_EU_FUSE_BITS)]; +typedef unsigned long xe_l3_bank_mask_t[BITS_TO_LONGS(XE_MAX_L3_BANK_MASK_BITS)]; + +/* + * The hardware has multiple kinds of multicast register ranges that need + * special register steering (and future platforms are expected to add + * additional types). + * + * During driver startup, we initialize the steering control register to + * direct reads to a slice/subslice that are valid for the 'subslice' class + * of multicast registers. If another type of steering does not have any + * overlap in valid steering targets with 'subslice' style registers, we will + * need to explicitly re-steer reads of registers of the other type. + * + * Only the replication types that may need additional non-default steering + * are listed here. + */ +enum xe_steering_type { + L3BANK, + NODE, + MSLICE, + LNCF, + DSS, + OADDRM, + SQIDI_PSMI, + + /* + * Although most GAM ranges must be steered to (0,0) and thus use the + * INSTANCE0 type farther down, some platforms have special rules + * for specific subtypes that require steering to (1,0) instead. + */ + GAM1, + + /* + * On some platforms there are multiple types of MCR registers that + * will always return a non-terminated value at instance (0, 0). We'll + * lump those all into a single category to keep things simple. + */ + INSTANCE0, + + /* + * Register ranges that don't need special steering for each register: + * it's sufficient to keep the HW-default for the selector, or only + * change it once, on GT initialization. This needs to be the last + * steering type. + */ + IMPLICIT_STEERING, + NUM_STEERING_TYPES +}; + +#define gt_to_tile(gt__) \ + _Generic(gt__, \ + const struct xe_gt * : (const struct xe_tile *)((gt__)->tile), \ + struct xe_gt * : (gt__)->tile) + +#define gt_to_xe(gt__) \ + _Generic(gt__, \ + const struct xe_gt * : (const struct xe_device *)(gt_to_tile(gt__)->xe), \ + struct xe_gt * : gt_to_tile(gt__)->xe) + +/** + * struct xe_gt - A "Graphics Technology" unit of the GPU + * + * A GT ("Graphics Technology") is the subset of a GPU primarily responsible + * for implementing the graphics, compute, and/or media IP. It encapsulates + * the hardware engines, programmable execution units, and GuC. Each GT has + * its own handling of power management (RC6+forcewake) and multicast register + * steering. + * + * A GPU/tile may have a single GT that supplies all graphics, compute, and + * media functionality, or the graphics/compute and media may be split into + * separate GTs within a tile. + */ +struct xe_gt { + /** @tile: Backpointer to GT's tile */ + struct xe_tile *tile; + + /** @info: GT info */ + struct { + /** @info.type: type of GT */ + enum xe_gt_type type; + /** @info.reference_clock: clock frequency */ + u32 reference_clock; + /** @info.timestamp_base: GT timestamp base */ + u32 timestamp_base; + /** + * @info.engine_mask: mask of engines present on GT. Some of + * them may be reserved in runtime and not available for user. + * See @user_engines.mask + */ + u64 engine_mask; + /** @info.gmdid: raw GMD_ID value from hardware */ + u32 gmdid; + /** + * @info.multi_queue_engine_class_mask: Bitmask of engine classes with + * multi queue support enabled. + */ + u16 multi_queue_engine_class_mask; + /** @info.id: Unique ID of this GT within the PCI Device */ + u8 id; + /** @info.has_indirect_ring_state: GT has indirect ring state support */ + u8 has_indirect_ring_state:1; + /** + * @info.has_uncorrectable_error_reporting: GT has uncorrectable + * error reporting support + */ + u8 has_uncorrectable_error_reporting:1; + /** + * @info.has_xe2_blt_instructions: GT supports Xe2-style MEM_SET + * and MEM_COPY blitter functionality. Note that despite the + * name, some Xe1 platforms may also support this "Xe2-style" + * feature. + */ + u8 has_xe2_blt_instructions:1; + /** + * @info.num_geometry_xecore_fuse_regs: Number of 32b-bit fuse + * registers the geometry XeCore mask spans. + */ + u8 num_geometry_xecore_fuse_regs; + /** + * @info.num_compute_xecore_fuse_regs: Number of 32b-bit fuse + * registers the compute XeCore mask spans. + */ + u8 num_compute_xecore_fuse_regs; + } info; + +#if IS_ENABLED(CONFIG_DEBUG_FS) + /** @stats: GT stats */ + struct xe_gt_stats __percpu *stats; +#endif + + /** + * @mmio: mmio info for GT. All GTs within a tile share the same + * register space, but have their own copy of GSI registers at a + * specific offset. + */ + struct xe_mmio mmio; + + /** + * @pm: power management info for GT. The driver uses the GT's + * "force wake" interface to wake up specific parts of the GT hardware + * from C6 sleep states and ensure the hardware remains awake while it + * is being actively used. + */ + struct { + /** @pm.fw: force wake for GT */ + struct xe_force_wake fw; + } pm; + + /** @sriov: virtualization data related to GT */ + union { + /** @sriov.pf: PF data. Valid only if driver is running as PF */ + struct xe_gt_sriov_pf pf; + /** @sriov.vf: VF data. Valid only if driver is running as VF */ + struct xe_gt_sriov_vf vf; + } sriov; + + /** + * @reg_sr: table with registers to be restored on GT init/resume/reset + */ + struct xe_reg_sr reg_sr; + + /** @reset: state for GT resets */ + struct { + /** + * @reset.worker: work so GT resets can done async allowing to reset + * code to safely flush all code paths + */ + struct work_struct worker; + } reset; + + /** @tlb_inval: TLB invalidation state */ + struct xe_tlb_inval tlb_inval; + + /** + * @ccs_mode: Number of compute engines enabled. + * Allows fixed mapping of available compute slices to compute engines. + * By default only the first available compute engine is enabled and all + * available compute slices are allocated to it. + */ + u32 ccs_mode; + + /** @usm: unified shared memory state */ + struct { + /** + * @usm.bb_pool: Pool from which batchbuffers, for USM operations + * (e.g. migrations, fixing page tables), are allocated. + * Dedicated pool needed so USM operations do not get blocked + * behind any user operations which may have resulted in a + * fault. + */ + struct xe_sa_manager *bb_pool; + /** + * @usm.paging_hwe0: The first designated paging engine. + * This is some reserved BCS instance used for USM operations + * (e.g. migrations, fixing page tables) + */ + struct xe_hw_engine *paging_hwe0; + /** + * @usm.paging_logical_mask: logical mask of paging engines. + * Should be densely populated. + */ + u32 paging_logical_mask; + } usm; + + /** @ordered_wq: used to serialize GT resets and TDRs */ + struct workqueue_struct *ordered_wq; + + /** @uc: micro controllers on the GT */ + struct xe_uc uc; + + /** @gtidle: idle properties of GT */ + struct xe_gt_idle gtidle; + + /** @exec_queue_ops: submission backend exec queue operations */ + const struct xe_exec_queue_ops *exec_queue_ops; + + /** + * @ring_ops: ring operations for this hw engine (1 per engine class) + */ + const struct xe_ring_ops *ring_ops[XE_ENGINE_CLASS_MAX]; + + /** @fence_irq: fence IRQs (1 per engine class) */ + struct xe_hw_fence_irq fence_irq[XE_ENGINE_CLASS_MAX]; + + /** @default_lrc: default LRC state */ + void *default_lrc[XE_ENGINE_CLASS_MAX]; + + /** @hw_engines: hardware engines on the GT */ + struct xe_hw_engine hw_engines[XE_NUM_HW_ENGINES]; + + /** @eclass: per hardware engine class interface on the GT */ + struct xe_hw_engine_class_intf eclass[XE_ENGINE_CLASS_MAX]; + + /** @sysfs: sysfs' kobj used by xe_gt_sysfs */ + struct kobject *sysfs; + + /** @freq: Main GT freq sysfs control */ + struct kobject *freq; + + /** @mocs: info */ + struct { + /** @mocs.uc_index: UC index */ + u8 uc_index; + /** @mocs.wb_index: WB index, only used on L3_CCS platforms */ + u8 wb_index; + } mocs; + + /** @fuse_topo: GT topology reported by fuse registers */ + struct { + /** @fuse_topo.g_dss_mask: dual-subslices usable by geometry */ + xe_dss_mask_t g_dss_mask; + + /** @fuse_topo.c_dss_mask: dual-subslices usable by compute */ + xe_dss_mask_t c_dss_mask; + + /** @fuse_topo.eu_mask_per_dss: EU mask per DSS*/ + xe_eu_mask_t eu_mask_per_dss; + + /** @fuse_topo.l3_bank_mask: L3 bank mask */ + xe_l3_bank_mask_t l3_bank_mask; + + /** + * @fuse_topo.eu_type: type/width of EU stored in + * fuse_topo.eu_mask_per_dss + */ + enum xe_gt_eu_type eu_type; + } fuse_topo; + + /** @steering: register steering for individual HW units */ + struct { + /** @steering.ranges: register ranges used for this steering type */ + const struct xe_mmio_range *ranges; + + /** @steering.group_target: target to steer accesses to */ + u16 group_target; + /** @steering.instance_target: instance to steer accesses to */ + u16 instance_target; + /** @steering.initialized: Whether this steering range is initialized */ + bool initialized; + } steering[NUM_STEERING_TYPES]; + + /** + * @steering_dss_per_grp: number of DSS per steering group (gslice, + * cslice, etc.). + */ + unsigned int steering_dss_per_grp; + + /** + * @mcr_lock: protects the MCR_SELECTOR register for the duration + * of a steered operation + */ + spinlock_t mcr_lock; + + /** + * @global_invl_lock: protects the register for the duration + * of a global invalidation of l2 cache + */ + spinlock_t global_invl_lock; + + /** @wa_active: keep track of active workarounds */ + struct { + /** @wa_active.gt: bitmap with active GT workarounds */ + unsigned long *gt; + /** @wa_active.engine: bitmap with active engine workarounds */ + unsigned long *engine; + /** @wa_active.lrc: bitmap with active LRC workarounds */ + unsigned long *lrc; + /** @wa_active.oob: bitmap with active OOB workarounds */ + unsigned long *oob; + /** + * @wa_active.oob_initialized: mark oob as initialized to help + * detecting misuse of XE_GT_WA() - it can only be called on + * initialization after OOB WAs have been processed + */ + bool oob_initialized; + } wa_active; + + /** @tuning_active: keep track of active tunings */ + struct { + /** @tuning_active.gt: bitmap with active GT tunings */ + unsigned long *gt; + /** @tuning_active.engine: bitmap with active engine tunings */ + unsigned long *engine; + /** @tuning_active.lrc: bitmap with active LRC tunings */ + unsigned long *lrc; + } tuning_active; + + /** @user_engines: engines present in GT and available to userspace */ + struct { + /** + * @user_engines.mask: like @info.engine_mask, but take in + * consideration only engines available to userspace + */ + u64 mask; + + /** + * @user_engines.instances_per_class: aggregate per class the + * number of engines available to userspace + */ + u8 instances_per_class[XE_ENGINE_CLASS_MAX]; + } user_engines; + + /** @oa: oa observation subsystem per gt info */ + struct xe_oa_gt oa; + + /** @eu_stall: EU stall counters subsystem per gt info */ + struct xe_eu_stall_gt *eu_stall; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guard.h b/drivers/gpu/drm/xe/xe_guard.h new file mode 100644 index 000000000..333f8e13b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guard.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GUARD_H_ +#define _XE_GUARD_H_ + +#include + +/** + * struct xe_guard - Simple logic to protect a feature. + * + * Implements simple semaphore-like logic that can be used to lockdown the + * feature unless it is already in use. Allows enabling of the otherwise + * incompatible features, where we can't follow the strict owner semantics + * required by the &rw_semaphore. + * + * NOTE! It shouldn't be used to protect a data, use &rw_semaphore instead. + */ +struct xe_guard { + /** + * @counter: implements simple exclusive/lockdown logic: + * if == 0 then guard/feature is idle/not in use, + * if < 0 then feature is active and can't be locked-down, + * if > 0 then feature is lockded-down and can't be activated. + */ + int counter; + + /** @name: the name of the guard (useful for debug) */ + const char *name; + + /** @owner: the info about the last owner of the guard (for debug) */ + void *owner; + + /** @lock: protects guard's data */ + spinlock_t lock; +}; + +/** + * xe_guard_init() - Initialize the guard. + * @guard: the &xe_guard to init + * @name: name of the guard + */ +static inline void xe_guard_init(struct xe_guard *guard, const char *name) +{ + spin_lock_init(&guard->lock); + guard->counter = 0; + guard->name = name; +} + +/** + * xe_guard_arm() - Arm the guard for the exclusive/lockdown mode. + * @guard: the &xe_guard to arm + * @lockdown: arm for lockdown(true) or exclusive(false) mode + * @who: optional owner info (for debug only) + * + * Multiple lockdown requests are allowed. + * Only single exclusive access can be granted. + * Will fail if the guard is already in exclusive mode. + * On success, must call the xe_guard_disarm() to release. + * + * Return: 0 on success or a negative error code on failure. + */ +static inline int xe_guard_arm(struct xe_guard *guard, bool lockdown, void *who) +{ + guard(spinlock)(&guard->lock); + + if (lockdown) { + if (guard->counter < 0) + return -EBUSY; + guard->counter++; + } else { + if (guard->counter > 0) + return -EPERM; + if (guard->counter < 0) + return -EUSERS; + guard->counter--; + } + + guard->owner = who; + return 0; +} + +/** + * xe_guard_disarm() - Disarm the guard from exclusive/lockdown mode. + * @guard: the &xe_guard to disarm + * @lockdown: disarm from lockdown(true) or exclusive(false) mode + * + * Return: true if successfully disarmed or false in case of mismatch. + */ +static inline bool xe_guard_disarm(struct xe_guard *guard, bool lockdown) +{ + guard(spinlock)(&guard->lock); + + if (lockdown) { + if (guard->counter <= 0) + return false; + guard->counter--; + } else { + if (guard->counter != -1) + return false; + guard->counter++; + } + return true; +} + +/** + * xe_guard_mode_str() - Convert guard mode into a string. + * @lockdown: flag used to select lockdown or exclusive mode + * + * Return: "lockdown" or "exclusive" string. + */ +static inline const char *xe_guard_mode_str(bool lockdown) +{ + return lockdown ? "lockdown" : "exclusive"; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c new file mode 100644 index 000000000..4286bd05c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -0,0 +1,1921 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc.h" + +#include +#include + +#include + +#include "abi/guc_actions_abi.h" +#include "abi/guc_errors_abi.h" +#include "abi/guc_klvs_abi.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_gtt_defs.h" +#include "regs/xe_guc_regs.h" +#include "regs/xe_irq_regs.h" +#include "xe_bo.h" +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_vf.h" +#include "xe_gt_throttle.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_guc_ads.h" +#include "xe_guc_buf.h" +#include "xe_guc_capture.h" +#include "xe_guc_ct.h" +#include "xe_guc_db_mgr.h" +#include "xe_guc_engine_activity.h" +#include "xe_guc_hwconfig.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_log.h" +#include "xe_guc_pc.h" +#include "xe_guc_rc.h" +#include "xe_guc_relay.h" +#include "xe_guc_submit.h" +#include "xe_memirq.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "xe_sleep.h" +#include "xe_sriov.h" +#include "xe_sriov_pf_migration.h" +#include "xe_uc.h" +#include "xe_uc_fw.h" +#include "xe_wa.h" +#include "xe_wopcm.h" + +static u32 guc_bo_ggtt_addr(struct xe_guc *guc, + struct xe_bo *bo) +{ + struct xe_device *xe = guc_to_xe(guc); + u32 addr; + + /* + * For most BOs, the address on the allocating tile is fine. However for + * some, e.g. G2G CTB, the address on a specific tile is required as it + * might be different for each tile. So, just always ask for the address + * on the target GuC. + */ + addr = __xe_bo_ggtt_addr(bo, gt_to_tile(guc_to_gt(guc))->id); + + /* GuC addresses above GUC_GGTT_TOP don't map through the GTT */ + xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc))); + xe_assert(xe, addr < GUC_GGTT_TOP); + xe_assert(xe, xe_bo_size(bo) <= GUC_GGTT_TOP - addr); + + return addr; +} + +static u32 guc_ctl_debug_flags(struct xe_guc *guc) +{ + u32 level = xe_guc_log_get_level(&guc->log); + u32 flags = 0; + + if (!GUC_LOG_LEVEL_IS_VERBOSE(level)) + flags |= GUC_LOG_DISABLED; + else + flags |= FIELD_PREP(GUC_LOG_VERBOSITY, GUC_LOG_LEVEL_TO_VERBOSITY(level)); + + return flags; +} + +static u32 guc_ctl_feature_flags(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + u32 flags = GUC_CTL_ENABLE_LITE_RESTORE; + + if (!xe->info.skip_guc_pc) + flags |= GUC_CTL_ENABLE_SLPC; + + if (xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev))) + flags |= GUC_CTL_ENABLE_PSMI_LOGGING; + + if (xe_guc_using_main_gamctrl_queues(guc)) + flags |= GUC_CTL_MAIN_GAMCTRL_QUEUES; + + if (xe_device_is_l2_flush_optimized(xe) && xe_gt_is_media_type(guc_to_gt(guc))) + flags |= GUC_CTL_ENABLE_L2FLUSH_OPT; + + /* + * On GuC firmware 70.66 and above, the GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE + * Feature KLV is used instead. + */ + if (!xe_configfs_get_enable_multi_queue(to_pci_dev(xe->drm.dev)) && + !GUC_FIRMWARE_VER_AT_LEAST(guc, 70, 66)) + flags |= GUC_CTL_DISABLE_MULTI_QUEUE; + + return flags; +} + +static u32 guc_ctl_log_params_flags(struct xe_guc *guc) +{ + u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT; + u32 flags; + + #if (((XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) % SZ_1M) == 0) + #define LOG_UNIT SZ_1M + #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS + #else + #define LOG_UNIT SZ_4K + #define LOG_FLAG 0 + #endif + + #if (((XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) % SZ_1M) == 0) + #define CAPTURE_UNIT SZ_1M + #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS + #else + #define CAPTURE_UNIT SZ_4K + #define CAPTURE_FLAG 0 + #endif + + BUILD_BUG_ON(!XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE, CAPTURE_UNIT)); + + flags = GUC_LOG_VALID | + GUC_LOG_NOTIFY_ON_HALF_FULL | + CAPTURE_FLAG | + LOG_FLAG | + FIELD_PREP(GUC_LOG_CRASH_DUMP, XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE / LOG_UNIT - 1) | + FIELD_PREP(GUC_LOG_EVENT_DATA, XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE / LOG_UNIT - 1) | + FIELD_PREP(GUC_LOG_STATE_CAPTURE, XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE / + CAPTURE_UNIT - 1) | + FIELD_PREP(GUC_LOG_BUF_ADDR, offset); + + #undef LOG_UNIT + #undef LOG_FLAG + #undef CAPTURE_UNIT + #undef CAPTURE_FLAG + + return flags; +} + +static u32 guc_ctl_ads_flags(struct xe_guc *guc) +{ + u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT; + u32 flags = FIELD_PREP(GUC_ADS_ADDR, ads); + + return flags; +} + +static bool needs_wa_dual_queue(struct xe_gt *gt) +{ + /* + * The DUAL_QUEUE_WA tells the GuC to not allow concurrent submissions + * on RCS and CCSes with different address spaces, which on DG2 is + * required as a WA for an HW bug. + */ + if (XE_GT_WA(gt, 22011391025)) + return true; + + /* + * On newer platforms, the HW has been updated to not allow parallel + * execution of different address spaces, so the RCS/CCS will stall the + * context switch if one of the other RCS/CCSes is busy with a different + * address space. While functionally correct, having a submission + * stalled on the HW limits the GuC ability to shuffle things around and + * can cause complications if the non-stalled submission runs for a long + * time, because the GuC doesn't know that the stalled submission isn't + * actually running and might declare it as hung. Therefore, we enable + * the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines + * to move management back to the GuC. + */ + if (CCS_INSTANCES(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) + return true; + + return false; +} + +static u32 guc_ctl_wa_flags(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + u32 flags = 0; + + if (XE_GT_WA(gt, 22012773006)) + flags |= GUC_WA_POLLCS; + + if (XE_GT_WA(gt, 14014475959)) + flags |= GUC_WA_HOLD_CCS_SWITCHOUT; + + if (needs_wa_dual_queue(gt)) + flags |= GUC_WA_DUAL_QUEUE; + + /* + * Wa_22011802037: FIXME - there's more to be done than simply setting + * this flag: make sure each CS is stopped when preparing for GT reset + * and wait for pending MI_FW. + */ + if (GRAPHICS_VERx100(xe) < 1270) + flags |= GUC_WA_PRE_PARSER; + + if (XE_GT_WA(gt, 22012727170) || XE_GT_WA(gt, 22012727685)) + flags |= GUC_WA_CONTEXT_ISOLATION; + + if (XE_GT_WA(gt, 18020744125) && + !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER)) + flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST; + + if (XE_GT_WA(gt, 14018913170)) + flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6; + + if (XE_GT_WA(gt, 16023683509)) + flags |= GUC_WA_SAVE_RESTORE_MCFG_REG_AT_MC6; + + return flags; +} + +static u32 guc_ctl_devid(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + + return (((u32)xe->info.devid) << 16) | xe->info.revid; +} + +static void guc_print_params(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 *params = guc->params; + int i; + + BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); + BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT); + + for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) + xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]); +} + +static void guc_init_params(struct xe_guc *guc) +{ + u32 *params = guc->params; + + params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); + params[GUC_CTL_FEATURE] = 0; + params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); + params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc); + params[GUC_CTL_WA] = 0; + params[GUC_CTL_DEVID] = guc_ctl_devid(guc); + + guc_print_params(guc); +} + +static void guc_init_params_post_hwconfig(struct xe_guc *guc) +{ + u32 *params = guc->params; + + params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); + params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); + params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); + params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc); + params[GUC_CTL_WA] = guc_ctl_wa_flags(guc); + params[GUC_CTL_DEVID] = guc_ctl_devid(guc); + + guc_print_params(guc); +} + +/* + * Initialize the GuC parameter block before starting the firmware + * transfer. These parameters are read by the firmware on startup + * and cannot be changed thereafter. + */ +static void guc_write_params(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + int i; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + xe_mmio_write32(>->mmio, SOFT_SCRATCH(0), 0); + + for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) + xe_mmio_write32(>->mmio, SOFT_SCRATCH(1 + i), guc->params[i]); +} + +static int guc_action_register_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev, + u32 desc_addr, u32 buff_addr, u32 size) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + u32 action[] = { + XE_GUC_ACTION_REGISTER_G2G, + FIELD_PREP(XE_G2G_REGISTER_SIZE, size / SZ_4K - 1) | + FIELD_PREP(XE_G2G_REGISTER_TYPE, type) | + FIELD_PREP(XE_G2G_REGISTER_TILE, dst_tile) | + FIELD_PREP(XE_G2G_REGISTER_DEVICE, dst_dev), + desc_addr, + buff_addr, + }; + + xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT)); + xe_assert(xe, !(size % SZ_4K)); + + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +static int guc_action_deregister_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + u32 action[] = { + XE_GUC_ACTION_DEREGISTER_G2G, + FIELD_PREP(XE_G2G_DEREGISTER_TYPE, type) | + FIELD_PREP(XE_G2G_DEREGISTER_TILE, dst_tile) | + FIELD_PREP(XE_G2G_DEREGISTER_DEVICE, dst_dev), + }; + + xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT)); + + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +#define G2G_DEV(gt) (((gt)->info.type == XE_GT_TYPE_MAIN) ? 0 : 1) + +#define G2G_BUFFER_SIZE (SZ_4K) +#define G2G_DESC_SIZE (64) +#define G2G_DESC_AREA_SIZE (SZ_4K) + +/* + * Generate a unique id for each bi-directional CTB for each pair of + * near and far tiles/devices. The id can then be used as an index into + * a single allocation that is sub-divided into multiple CTBs. + * + * For example, with two devices per tile and two tiles, the table should + * look like: + * Far . + * 0.0 0.1 1.0 1.1 + * N 0.0 --/-- 00/01 02/03 04/05 + * e 0.1 01/00 --/-- 06/07 08/09 + * a 1.0 03/02 07/06 --/-- 10/11 + * r 1.1 05/04 09/08 11/10 --/-- + * + * Where each entry is Rx/Tx channel id. + * + * So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would + * be reading from channel #11 and writing to channel #10. Whereas, + * GuC #2 talking to GuC #3 would be read on #10 and write to #11. + */ +static unsigned int g2g_slot(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev, + u32 type, u32 max_inst, bool have_dev) +{ + u32 near = near_tile, far = far_tile; + u32 idx = 0, x, y, direction; + int i; + + if (have_dev) { + near = (near << 1) | near_dev; + far = (far << 1) | far_dev; + } + + /* No need to send to one's self */ + if (far == near) + return -1; + + if (far > near) { + /* Top right table half */ + x = far; + y = near; + + /* T/R is 'forwards' direction */ + direction = type; + } else { + /* Bottom left table half */ + x = near; + y = far; + + /* B/L is 'backwards' direction */ + direction = (1 - type); + } + + /* Count the rows prior to the target */ + for (i = y; i > 0; i--) + idx += max_inst - i; + + /* Count this row up to the target */ + idx += (x - 1 - y); + + /* Slots are in Rx/Tx pairs */ + idx *= 2; + + /* Pick Rx/Tx direction */ + idx += direction; + + return idx; +} + +static int guc_g2g_register(struct xe_guc *near_guc, struct xe_gt *far_gt, u32 type, bool have_dev) +{ + struct xe_gt *near_gt = guc_to_gt(near_guc); + struct xe_device *xe = gt_to_xe(near_gt); + struct xe_bo *g2g_bo; + u32 near_tile = gt_to_tile(near_gt)->id; + u32 near_dev = G2G_DEV(near_gt); + u32 far_tile = gt_to_tile(far_gt)->id; + u32 far_dev = G2G_DEV(far_gt); + u32 max = xe->info.gt_count; + u32 base, desc, buf; + int slot; + + /* G2G is not allowed between different cards */ + xe_assert(xe, xe == gt_to_xe(far_gt)); + + g2g_bo = near_guc->g2g.bo; + xe_assert(xe, g2g_bo); + + slot = g2g_slot(near_tile, near_dev, far_tile, far_dev, type, max, have_dev); + xe_assert(xe, slot >= 0); + + base = guc_bo_ggtt_addr(near_guc, g2g_bo); + desc = base + slot * G2G_DESC_SIZE; + buf = base + G2G_DESC_AREA_SIZE + slot * G2G_BUFFER_SIZE; + + xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE); + xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= xe_bo_size(g2g_bo)); + + return guc_action_register_g2g_buffer(near_guc, type, far_tile, far_dev, + desc, buf, G2G_BUFFER_SIZE); +} + +static void guc_g2g_deregister(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type) +{ + guc_action_deregister_g2g_buffer(guc, type, far_tile, far_dev); +} + +static u32 guc_g2g_size(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + unsigned int count = xe->info.gt_count; + u32 num_channels = (count * (count - 1)) / 2; + + xe_assert(xe, num_channels * XE_G2G_TYPE_LIMIT * G2G_DESC_SIZE <= G2G_DESC_AREA_SIZE); + + return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE; +} + +static bool xe_guc_g2g_wanted(struct xe_device *xe) +{ + /* Can't do GuC to GuC communication if there is only one GuC */ + if (xe->info.gt_count <= 1) + return false; + + /* No current user */ + return false; +} + +static int guc_g2g_alloc(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *bo; + u32 g2g_size; + + if (guc->g2g.bo) + return 0; + + if (gt->info.id != 0) { + struct xe_gt *root_gt = xe_device_get_gt(xe, 0); + struct xe_guc *root_guc = &root_gt->uc.guc; + struct xe_bo *bo; + + bo = xe_bo_get(root_guc->g2g.bo); + if (!bo) + return -ENODEV; + + guc->g2g.bo = bo; + guc->g2g.owned = false; + return 0; + } + + g2g_size = guc_g2g_size(guc); + bo = xe_managed_bo_create_pin_map(xe, tile, g2g_size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_ALL | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size); + guc->g2g.bo = bo; + guc->g2g.owned = true; + + return 0; +} + +static void guc_g2g_fini(struct xe_guc *guc) +{ + if (!guc->g2g.bo) + return; + + /* Unpinning the owned object is handled by generic shutdown */ + if (!guc->g2g.owned) + xe_bo_put(guc->g2g.bo); + + guc->g2g.bo = NULL; +} + +static int guc_g2g_start(struct xe_guc *guc) +{ + struct xe_gt *far_gt, *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + unsigned int i, j; + int t, err; + bool have_dev; + + if (!guc->g2g.bo) { + int ret; + + ret = guc_g2g_alloc(guc); + if (ret) + return ret; + } + + /* GuC interface will need extending if more GT device types are ever created. */ + xe_gt_assert(gt, (gt->info.type == XE_GT_TYPE_MAIN) || (gt->info.type == XE_GT_TYPE_MEDIA)); + + /* Channel numbering depends on whether there are multiple GTs per tile */ + have_dev = xe->info.gt_count > xe->info.tile_count; + + for_each_gt(far_gt, xe, i) { + u32 far_tile, far_dev; + + if (far_gt->info.id == gt->info.id) + continue; + + far_tile = gt_to_tile(far_gt)->id; + far_dev = G2G_DEV(far_gt); + + for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) { + err = guc_g2g_register(guc, far_gt, t, have_dev); + if (err) { + while (--t >= 0) + guc_g2g_deregister(guc, far_tile, far_dev, t); + goto err_deregister; + } + } + } + + return 0; + +err_deregister: + for_each_gt(far_gt, xe, j) { + u32 tile, dev; + + if (far_gt->info.id == gt->info.id) + continue; + + if (j >= i) + break; + + tile = gt_to_tile(far_gt)->id; + dev = G2G_DEV(far_gt); + + for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) + guc_g2g_deregister(guc, tile, dev, t); + } + + return err; +} + +static int __guc_opt_in_features_enable(struct xe_guc *guc, u64 addr, u32 num_dwords) +{ + u32 action[] = { + XE_GUC_ACTION_OPT_IN_FEATURE_KLV, + lower_32_bits(addr), + upper_32_bits(addr), + num_dwords + }; + + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +static bool supports_dynamic_ics(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + + /* Dynamic ICS is available for PVC and Xe2 and newer platforms. */ + if (xe->info.platform != XE_PVC && GRAPHICS_VER(xe) < 20) + return false; + + /* + * The feature is currently not compatible with multi-lrc, so the GuC + * does not support it at all on the media engines (which are the main + * users of mlrc). On the primary GT side, to avoid it being used in + * conjunction with mlrc, we only enable it if we are in single CCS + * mode. + */ + if (xe_gt_is_media_type(gt) || gt->ccs_mode > 1) + return false; + + /* + * Dynamic ICS requires GuC v70.40.1, which maps to compatibility + * version v1.18.4. + */ + return GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 18, 4); +} + +#define OPT_IN_MAX_DWORDS 16 +int xe_guc_opt_in_features_enable(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + CLASS(xe_guc_buf, buf)(&guc->buf, OPT_IN_MAX_DWORDS); + u32 count = 0; + u32 *klvs; + int ret; + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + klvs = xe_guc_buf_cpu_ptr(buf); + + /* + * The extra CAT error type opt-in was added in GuC v70.17.0, which maps + * to compatibility version v1.7.0. + * Note that the GuC allows enabling this KLV even on platforms that do + * not support the extra type; in such case the returned type variable + * will be set to a known invalid value which we can check against. + */ + if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 7, 0)) + klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_EXT_CAT_ERR_TYPE); + + /* + * The uncorrectable local error notification opt-in was added in + * GuC v70.38.0, which maps to compatibility version v1.18.0. + */ + if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 18, 0) && + guc_to_gt(guc)->info.has_uncorrectable_error_reporting) + klvs[count++] = + PREP_GUC_KLV_TAG(OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION); + + if (supports_dynamic_ics(guc)) + klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH); + + if (count) { + xe_assert(xe, count <= OPT_IN_MAX_DWORDS); + + ret = __guc_opt_in_features_enable(guc, xe_guc_buf_flush(buf), count); + if (ret < 0) { + xe_gt_err(guc_to_gt(guc), + "failed to enable GuC opt-in features: %pe\n", + ERR_PTR(ret)); + return ret; + } + } + + return 0; +} + +static void guc_fini_hw(void *arg) +{ + struct xe_guc *guc = arg; + struct xe_gt *gt = guc_to_gt(guc); + + xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) + xe_uc_sanitize_reset(&guc_to_gt(guc)->uc); + + guc_g2g_fini(guc); +} + +static void vf_guc_fini_hw(void *arg) +{ + struct xe_guc *guc = arg; + + xe_gt_sriov_vf_reset(guc_to_gt(guc)); +} + +/** + * xe_guc_comm_init_early - early initialization of GuC communication + * @guc: the &xe_guc to initialize + * + * Must be called prior to first MMIO communication with GuC firmware. + */ +void xe_guc_comm_init_early(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + + if (xe_gt_is_media_type(gt)) + guc->notify_reg = MED_GUC_HOST_INTERRUPT; + else + guc->notify_reg = GUC_HOST_INTERRUPT; +} + +static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc) +{ + struct xe_tile *tile = gt_to_tile(guc_to_gt(guc)); + struct xe_device *xe = guc_to_xe(guc); + int ret; + + if (!IS_DGFX(guc_to_xe(guc))) + return 0; + + ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->fw.bo); + if (ret) + return ret; + + ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->log.bo); + if (ret) + return ret; + + ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ads.bo); + if (ret) + return ret; + + return 0; +} + +static int vf_guc_init_noalloc(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + int err; + + err = xe_gt_sriov_vf_bootstrap(gt); + if (err) + return err; + + err = xe_gt_sriov_vf_query_config(gt); + if (err) + return err; + + return 0; +} + +int xe_guc_init_noalloc(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + xe_guc_comm_init_early(guc); + + ret = xe_guc_ct_init_noalloc(&guc->ct); + if (ret) + goto out; + + ret = xe_guc_relay_init(&guc->relay); + if (ret) + goto out; + + if (IS_SRIOV_VF(xe)) { + ret = vf_guc_init_noalloc(guc); + if (ret) + goto out; + } + + return 0; + +out: + xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret)); + return ret; +} + +int xe_guc_init(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + guc->fw.type = XE_UC_FW_TYPE_GUC; + ret = xe_uc_fw_init(&guc->fw); + if (ret) + return ret; + + if (!xe_uc_fw_is_enabled(&guc->fw)) + return 0; + + /* Disable page reclaim if GuC FW does not support */ + if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0)) + xe->info.has_page_reclaim_hw_assist = false; + + /* Disable indirect_ring_state if missing GuC 70.53+ WA 14025515070. */ + if (gt->info.has_indirect_ring_state && + XE_GT_WA(gt, 14025515070) && + GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 26, 0)) { + gt->info.has_indirect_ring_state = 0; + xe_gt_notice(gt, "indirect ring state requires WA in GuC submit ver 1.26+\n"); + } + + if (IS_SRIOV_VF(xe)) { + ret = devm_add_action_or_reset(xe->drm.dev, vf_guc_fini_hw, guc); + if (ret) + goto out; + + ret = xe_guc_ct_init(&guc->ct); + if (ret) + goto out; + return 0; + } + + ret = xe_guc_log_init(&guc->log); + if (ret) + goto out; + + ret = xe_guc_capture_init(guc); + if (ret) + goto out; + + ret = xe_guc_ads_init(&guc->ads); + if (ret) + goto out; + + ret = xe_guc_ct_init(&guc->ct); + if (ret) + goto out; + + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE); + + ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc); + if (ret) + goto out; + + guc_init_params(guc); + + return 0; + +out: + xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret)); + return ret; +} + +static int vf_guc_init_post_hwconfig(struct xe_guc *guc) +{ + int err; + + err = xe_guc_submit_init(guc, xe_gt_sriov_vf_guc_ids(guc_to_gt(guc))); + if (err) + return err; + + err = xe_guc_buf_cache_init(&guc->buf); + if (err) + return err; + + /* XXX xe_guc_db_mgr_init not needed for now */ + + return 0; +} + +static u32 guc_additional_cache_size(struct xe_device *xe) +{ + if (IS_SRIOV_PF(xe) && xe_sriov_pf_migration_supported(xe)) + return XE_GT_SRIOV_PF_MIGRATION_GUC_DATA_MAX_SIZE; + else + return 0; /* Fallback to default size */ +} + +/** + * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load + * @guc: The GuC object + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_init_post_hwconfig(struct xe_guc *guc) +{ + int ret; + + if (IS_SRIOV_VF(guc_to_xe(guc))) + return vf_guc_init_post_hwconfig(guc); + + ret = xe_guc_realloc_post_hwconfig(guc); + if (ret) + return ret; + + ret = xe_guc_ct_init_post_hwconfig(&guc->ct); + if (ret) + return ret; + + guc_init_params_post_hwconfig(guc); + + ret = xe_guc_submit_init(guc, ~0); + if (ret) + return ret; + + ret = xe_guc_db_mgr_init(&guc->dbm, ~0); + if (ret) + return ret; + + ret = xe_guc_pc_init(&guc->pc); + if (ret) + return ret; + + ret = xe_guc_rc_init(guc); + if (ret) + return ret; + + ret = xe_guc_engine_activity_init(guc); + if (ret) + return ret; + + ret = xe_guc_buf_cache_init_with_size(&guc->buf, + guc_additional_cache_size(guc_to_xe(guc))); + if (ret) + return ret; + + return xe_guc_ads_init_post_hwconfig(&guc->ads); +} + +int xe_guc_post_load_init(struct xe_guc *guc) +{ + int ret; + + xe_guc_ads_populate_post_load(&guc->ads); + + ret = xe_guc_opt_in_features_enable(guc); + if (ret) + return ret; + + if (xe_guc_g2g_wanted(guc_to_xe(guc))) { + ret = guc_g2g_start(guc); + if (ret) + return ret; + } + + return xe_guc_submit_enable(guc); +} + +/* + * Wa_14025883347: Prevent GuC firmware DMA failures during GuC-only reset by ensuring + * SRAM save/restore operations are complete before reset. + */ +static void guc_prevent_fw_dma_failure_on_reset(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 boot_hash_chk, guc_status, sram_status; + int ret; + + guc_status = xe_mmio_read32(>->mmio, GUC_STATUS); + if (guc_status & GS_MIA_IN_RESET) + return; + + boot_hash_chk = xe_mmio_read32(>->mmio, BOOT_HASH_CHK); + if (!(boot_hash_chk & GUC_BOOT_UKERNEL_VALID)) + return; + + /* Disable idle flow during reset (GuC reset re-enables it automatically) */ + xe_mmio_rmw32(>->mmio, GUC_MAX_IDLE_COUNT, 0, GUC_IDLE_FLOW_DISABLE); + + ret = xe_mmio_wait32(>->mmio, GUC_STATUS, GS_UKERNEL_MASK, + FIELD_PREP(GS_UKERNEL_MASK, XE_GUC_LOAD_STATUS_READY), + 100000, &guc_status, false); + if (ret) + xe_gt_warn(gt, "GuC not ready after disabling idle flow (GUC_STATUS: 0x%x)\n", + guc_status); + + ret = xe_mmio_wait32(>->mmio, GUC_SRAM_STATUS, GUC_SRAM_HANDLING_MASK, + 0, 5000, &sram_status, false); + if (ret) + xe_gt_warn(gt, "SRAM handling not complete (GUC_SRAM_STATUS: 0x%x)\n", + sram_status); +} + +int xe_guc_reset(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_mmio *mmio = >->mmio; + u32 guc_status, gdrst; + int ret; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return xe_gt_sriov_vf_bootstrap(gt); + + if (XE_GT_WA(gt, 14025883347)) + guc_prevent_fw_dma_failure_on_reset(guc); + + xe_mmio_write32(mmio, GDRST, GRDOM_GUC); + + ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false); + if (ret) { + xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst); + goto err_out; + } + + guc_status = xe_mmio_read32(mmio, GUC_STATUS); + if (!(guc_status & GS_MIA_IN_RESET)) { + xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n", + guc_status); + ret = -EIO; + goto err_out; + } + + return 0; + +err_out: + + return ret; +} + +static void guc_prepare_xfer(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_mmio *mmio = >->mmio; + struct xe_device *xe = guc_to_xe(guc); + u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC | + GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA | + GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA | + GUC_ENABLE_MIA_CLOCK_GATING; + + if (GRAPHICS_VERx100(xe) < 1250) + shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES | + GUC_ENABLE_MIA_CACHING; + + if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) + shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index); + + /* Must program this register before loading the ucode with DMA */ + xe_mmio_write32(mmio, GUC_SHIM_CONTROL, shim_flags); + + xe_mmio_write32(mmio, GT_PM_CONFIG, GT_DOORBELL_ENABLE); + + /* Make sure GuC receives ARAT interrupts */ + xe_mmio_rmw32(mmio, PMINTRMSK, ARAT_EXPIRED_INTRMSK, 0); +} + +/* + * Supporting MMIO & in memory RSA + */ +static int guc_xfer_rsa(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 rsa[UOS_RSA_SCRATCH_COUNT]; + size_t copied; + int i; + + if (guc->fw.rsa_size > 256) { + u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) + + xe_uc_fw_rsa_offset(&guc->fw); + xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(0), rsa_ggtt_addr); + return 0; + } + + copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa)); + if (copied < sizeof(rsa)) + return -ENOMEM; + + for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++) + xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(i), rsa[i]); + + return 0; +} + +/* + * Wait for the GuC to start up. + * + * Measurements indicate this should take no more than 20ms (assuming the GT + * clock is at maximum frequency). However, thermal throttling and other issues + * can prevent the clock hitting max and thus making the load take significantly + * longer. Allow up to 3s as a safety margin in normal builds. For + * CONFIG_DRM_XE_DEBUG allow up to 10s to account for slower execution, issues + * in PCODE, driver, fan, etc. + * + * Keep checking the GUC_STATUS every 10ms with a debug message every 100 + * attempts as a "I'm slow, but alive" message. Regardless, if it takes more + * than 200ms, emit a warning. + */ + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define GUC_LOAD_TIMEOUT_SEC 20 +#else +#define GUC_LOAD_TIMEOUT_SEC 3 +#endif +#define GUC_LOAD_TIME_WARN_MSEC 200 + +static void print_load_status_err(struct xe_gt *gt, u32 status) +{ + struct xe_mmio *mmio = >->mmio; + u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status); + u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status); + + xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n", + REG_FIELD_GET(GS_MIA_IN_RESET, status), + bootrom, ukernel, + REG_FIELD_GET(GS_MIA_MASK, status), + REG_FIELD_GET(GS_AUTH_STATUS_MASK, status)); + + switch (bootrom) { + case XE_BOOTROM_STATUS_NO_KEY_FOUND: + xe_gt_err(gt, "invalid key requested, header = 0x%08X\n", + xe_mmio_read32(mmio, GUC_HEADER_INFO)); + break; + case XE_BOOTROM_STATUS_RSA_FAILED: + xe_gt_err(gt, "firmware signature verification failed\n"); + break; + case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE: + xe_gt_err(gt, "firmware production part check failure\n"); + break; + } + + switch (ukernel) { + case XE_GUC_LOAD_STATUS_HWCONFIG_START: + xe_gt_err(gt, "still extracting hwconfig table.\n"); + break; + case XE_GUC_LOAD_STATUS_EXCEPTION: + xe_gt_err(gt, "firmware exception. EIP: %#x\n", + xe_mmio_read32(mmio, SOFT_SCRATCH(13))); + break; + case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID: + xe_gt_err(gt, "illegal init/ADS data\n"); + break; + case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID: + xe_gt_err(gt, "illegal register in save/restore workaround list\n"); + break; + case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR: + xe_gt_err(gt, "illegal workaround KLV data\n"); + break; + case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG: + xe_gt_err(gt, "illegal feature flag specified\n"); + break; + } +} + +/* + * Check GUC_STATUS looking for known terminal states (either completion or + * failure) of either the microkernel status field or the boot ROM status field. + * + * Returns 1 for successful completion, -1 for failure and 0 for any + * intermediate state. + */ +static int guc_load_done(struct xe_gt *gt, u32 *status, u32 *tries) +{ + u32 ukernel, bootrom; + + *status = xe_mmio_read32(>->mmio, GUC_STATUS); + ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, *status); + bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, *status); + + switch (ukernel) { + case XE_GUC_LOAD_STATUS_READY: + return 1; + case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH: + case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH: + case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE: + case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR: + case XE_GUC_LOAD_STATUS_BOOTROM_VERSION_MISMATCH: + case XE_GUC_LOAD_STATUS_DPC_ERROR: + case XE_GUC_LOAD_STATUS_EXCEPTION: + case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID: + case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID: + case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID: + case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR: + case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG: + return -1; + } + + switch (bootrom) { + case XE_BOOTROM_STATUS_NO_KEY_FOUND: + case XE_BOOTROM_STATUS_RSA_FAILED: + case XE_BOOTROM_STATUS_PAVPC_FAILED: + case XE_BOOTROM_STATUS_WOPCM_FAILED: + case XE_BOOTROM_STATUS_LOADLOC_FAILED: + case XE_BOOTROM_STATUS_JUMP_FAILED: + case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED: + case XE_BOOTROM_STATUS_MPUMAP_INCORRECT: + case XE_BOOTROM_STATUS_EXCEPTION: + case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE: + return -1; + } + + if (++*tries >= 100) { + struct xe_guc_pc *guc_pc = >->uc.guc.pc; + + *tries = 0; + xe_gt_dbg(gt, "GuC load still in progress, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n", + xe_guc_pc_get_act_freq(guc_pc), + xe_guc_pc_get_cur_freq_fw(guc_pc), + *status, ukernel, bootrom); + } + + return 0; +} + +static int guc_wait_ucode(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_guc_pc *guc_pc = >->uc.guc.pc; + u32 before_freq, act_freq, cur_freq; + u32 status = 0, tries = 0; + int load_result, ret; + ktime_t before; + u64 delta_ms; + + before_freq = xe_guc_pc_get_act_freq(guc_pc); + before = ktime_get(); + + ret = poll_timeout_us(load_result = guc_load_done(gt, &status, &tries), load_result, + 10 * USEC_PER_MSEC, + GUC_LOAD_TIMEOUT_SEC * USEC_PER_SEC, false); + + delta_ms = ktime_to_ms(ktime_sub(ktime_get(), before)); + act_freq = xe_guc_pc_get_act_freq(guc_pc); + cur_freq = xe_guc_pc_get_cur_freq_fw(guc_pc); + + if (ret || load_result <= 0) { + xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz)\n", + status, delta_ms, xe_guc_pc_get_act_freq(guc_pc), + xe_guc_pc_get_cur_freq_fw(guc_pc)); + print_load_status_err(gt, status); + + return -EPROTO; + } + + if (delta_ms > GUC_LOAD_TIME_WARN_MSEC) { + xe_gt_warn(gt, "GuC load: excessive init time: %lldms! [status = 0x%08X]\n", + delta_ms, status); + xe_gt_warn(gt, "GuC load: excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n", + act_freq, cur_freq, before_freq, + xe_gt_throttle_get_limit_reasons(gt)); + } else { + xe_gt_dbg(gt, "GuC load: init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X\n", + delta_ms, act_freq, cur_freq, before_freq, status); + } + + return 0; +} +ALLOW_ERROR_INJECTION(guc_wait_ucode, ERRNO); + +static int __xe_guc_upload(struct xe_guc *guc) +{ + int ret; + + /* Raise GT freq to speed up HuC/GuC load */ + xe_guc_pc_raise_unslice(&guc->pc); + + guc_write_params(guc); + guc_prepare_xfer(guc); + + /* + * Note that GuC needs the CSS header plus uKernel code to be copied + * by the DMA engine in one operation, whereas the RSA signature is + * loaded separately, either by copying it to the UOS_RSA_SCRATCH + * register (if key size <= 256) or through a ggtt-pinned vma (if key + * size > 256). The RSA size and therefore the way we provide it to the + * HW is fixed for each platform and hard-coded in the bootrom. + */ + ret = guc_xfer_rsa(guc); + if (ret) + goto out; + /* + * Current uCode expects the code to be loaded at 8k; locations below + * this are used for the stack. + */ + ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE); + if (ret) + goto out; + + /* Wait for authentication */ + ret = guc_wait_ucode(guc); + if (ret) + goto out; + + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING); + return 0; + +out: + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL); + return ret; +} + +static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + ret = xe_guc_hwconfig_init(guc); + if (ret) + return ret; + + ret = xe_guc_enable_communication(guc); + if (ret) + return ret; + + ret = xe_gt_sriov_vf_connect(gt); + if (ret) + goto err_out; + + ret = xe_gt_sriov_vf_query_runtime(gt); + if (ret) + goto err_out; + + return 0; + +err_out: + xe_guc_sanitize(guc); + return ret; +} + +/** + * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table + * @guc: The GuC object + * + * This function uploads a minimal GuC that does not support submissions but + * in a state where the hwconfig table can be read. Next, it reads and parses + * the hwconfig table so it can be used for subsequent steps in the driver load. + * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only). + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_min_load_for_hwconfig(struct xe_guc *guc) +{ + int ret; + + if (IS_SRIOV_VF(guc_to_xe(guc))) + return vf_guc_min_load_for_hwconfig(guc); + + xe_guc_ads_populate_minimal(&guc->ads); + + xe_guc_pc_init_early(&guc->pc); + + ret = __xe_guc_upload(guc); + if (ret) + return ret; + + ret = xe_guc_hwconfig_init(guc); + if (ret) + return ret; + + ret = xe_guc_enable_communication(guc); + if (ret) + return ret; + + return 0; +} + +int xe_guc_upload(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + + xe_guc_ads_populate(&guc->ads); + + if (xe_guc_using_main_gamctrl_queues(guc)) + xe_mmio_write32(>->mmio, MAIN_GAMCTRL_MODE, MAIN_GAMCTRL_QUEUE_SELECT); + + return __xe_guc_upload(guc); +} + +static void guc_handle_mmio_msg(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 msg; + + if (IS_SRIOV_VF(guc_to_xe(guc))) + return; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + msg = xe_mmio_read32(>->mmio, SOFT_SCRATCH(15)); + msg &= XE_GUC_RECV_MSG_EXCEPTION | + XE_GUC_RECV_MSG_CRASH_DUMP_POSTED; + xe_mmio_write32(>->mmio, SOFT_SCRATCH(15), 0); + + if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED) + xe_gt_err(gt, "Received early GuC crash dump notification!\n"); + + if (msg & XE_GUC_RECV_MSG_EXCEPTION) + xe_gt_err(gt, "Received early GuC exception notification!\n"); +} + +static void guc_enable_irq(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 events = xe_gt_is_media_type(gt) ? + REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) : + REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST); + + /* Primary GuC and media GuC share a single enable bit */ + xe_mmio_write32(>->mmio, GUC_SG_INTR_ENABLE, + REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST)); + + /* + * There are separate mask bits for primary and media GuCs, so use + * a RMW operation to avoid clobbering the other GuC's setting. + */ + xe_mmio_rmw32(>->mmio, GUC_SG_INTR_MASK, events, 0); +} + +int xe_guc_enable_communication(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + int err; + + if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) { + struct xe_gt *gt = guc_to_gt(guc); + struct xe_tile *tile = gt_to_tile(gt); + + err = xe_memirq_init_guc(&tile->memirq, guc); + if (err) + return err; + } else { + guc_enable_irq(guc); + } + + err = xe_guc_ct_enable(&guc->ct); + if (err) + return err; + + guc_handle_mmio_msg(guc); + + return 0; +} + +/** + * xe_guc_softreset() - Soft reset GuC + * @guc: The GuC object + * + * Send soft reset command to GuC through mmio send. + * + * Return: 0 if success, otherwise error code + */ +int xe_guc_softreset(struct xe_guc *guc) +{ + u32 action[] = { + XE_GUC_ACTION_CLIENT_SOFT_RESET, + }; + int ret; + + if (!xe_uc_fw_is_running(&guc->fw)) + return 0; + + ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action)); + if (ret) + return ret; + + return 0; +} + +int xe_guc_suspend(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + ret = xe_guc_softreset(guc); + if (ret) { + xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret)); + return ret; + } + + xe_guc_sanitize(guc); + return 0; +} + +void xe_guc_notify(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + const u32 default_notify_data = 0; + + /* + * Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass + * additional payload data to the GuC but this capability is not + * used by the firmware yet. Use default value in the meantime. + */ + xe_mmio_write32(>->mmio, guc->notify_reg, default_notify_data); +} + +int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr) +{ + u32 action[] = { + XE_GUC_ACTION_AUTHENTICATE_HUC, + rsa_addr + }; + + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +#define MAX_RETRIES_ON_FLR 2 +#define MIN_SLEEP_MS_ON_FLR 256 + +int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request, + u32 len, u32 *response_buf) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + struct xe_mmio *mmio = >->mmio; + struct xe_reg reply_reg = xe_gt_is_media_type(gt) ? + MED_VF_SW_FLAG(0) : VF_SW_FLAG(0); + const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1; + unsigned int sleep_period_ms = 1; + unsigned int lost = 0; + u32 header; + int ret; + int i; + + BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT); + + xe_assert(xe, len); + xe_assert(xe, len <= VF_SW_FLAG_COUNT); + xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT); + xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) == + GUC_HXG_ORIGIN_HOST); + xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) == + GUC_HXG_TYPE_REQUEST); + +retry: + /* Not in critical data-path, just do if else for GT type */ + if (xe_gt_is_media_type(gt)) { + for (i = 0; i < len; ++i) + xe_mmio_write32(mmio, MED_VF_SW_FLAG(i), + request[i]); + xe_mmio_read32(mmio, MED_VF_SW_FLAG(LAST_INDEX)); + } else { + for (i = 0; i < len; ++i) + xe_mmio_write32(mmio, VF_SW_FLAG(i), + request[i]); + xe_mmio_read32(mmio, VF_SW_FLAG(LAST_INDEX)); + } + + xe_guc_notify(guc); + + ret = xe_mmio_wait32(mmio, reply_reg, GUC_HXG_MSG_0_ORIGIN, + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC), + 50000, &header, false); + if (ret) { + /* scratch registers might be cleared during FLR, try once more */ + if (!header) { + if (++lost > MAX_RETRIES_ON_FLR) { + xe_gt_err(gt, "GuC mmio request %#x: lost, too many retries %u\n", + request[0], lost); + return -ENOLINK; + } + xe_gt_dbg(gt, "GuC mmio request %#x: lost, trying again\n", request[0]); + xe_sleep_relaxed_ms(MIN_SLEEP_MS_ON_FLR); + goto retry; + } +timeout: + xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n", + request[0], header); + return ret; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == + GUC_HXG_TYPE_NO_RESPONSE_BUSY) { + /* + * Once we got a BUSY reply we must wait again for the final + * response but this time we can't use ORIGIN mask anymore. + * To spot a right change in the reply, we take advantage that + * response SUCCESS and FAILURE differ only by the single bit + * and all other bits are set and can be used as a new mask. + */ + u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE; + u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits); + + BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS); + BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1); + + ret = xe_mmio_wait32(mmio, reply_reg, resp_mask, resp_mask, + 2000000, &header, false); + + if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) != + GUC_HXG_ORIGIN_GUC)) + goto proto; + if (unlikely(ret)) { + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != + GUC_HXG_TYPE_NO_RESPONSE_BUSY) + goto proto; + goto timeout; + } + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == + GUC_HXG_TYPE_NO_RESPONSE_RETRY) { + u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header); + + xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n", + request[0], reason); + + xe_sleep_exponential_ms(&sleep_period_ms, 256); + goto retry; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == + GUC_HXG_TYPE_RESPONSE_FAILURE) { + u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header); + u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header); + + if (unlikely(error == XE_GUC_RESPONSE_VF_MIGRATED)) { + xe_gt_dbg(gt, "GuC mmio request %#x rejected due to MIGRATION (hint %#x)\n", + request[0], hint); + return -EREMCHG; + } + + xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n", + request[0], error, hint); + return -ENXIO; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != + GUC_HXG_TYPE_RESPONSE_SUCCESS) { +proto: + xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n", + request[0], header); + return -EPROTO; + } + + /* Just copy entire possible message response */ + if (response_buf) { + response_buf[0] = header; + + for (i = 1; i < VF_SW_FLAG_COUNT; i++) { + reply_reg.addr += sizeof(u32); + response_buf[i] = xe_mmio_read32(mmio, reply_reg); + } + } + + /* Use data from the GuC response as our return value */ + return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header); +} +ALLOW_ERROR_INJECTION(xe_guc_mmio_send_recv, ERRNO); + +int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len) +{ + return xe_guc_mmio_send_recv(guc, request, len, NULL); +} + +static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val) +{ + struct xe_device *xe = guc_to_xe(guc); + u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_ACTION_HOST2GUC_SELF_CFG), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) | + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32, + lower_32_bits(val)), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64, + upper_32_bits(val)), + }; + int ret; + + xe_assert(xe, len <= 2); + xe_assert(xe, len != 1 || !upper_32_bits(val)); + + /* Self config must go over MMIO */ + ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request)); + + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret > 1)) + return -EPROTO; + if (unlikely(!ret)) + return -ENOKEY; + + return 0; +} + +int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val) +{ + return guc_self_cfg(guc, key, 1, val); +} + +int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val) +{ + return guc_self_cfg(guc, key, 2, val); +} + +static void xe_guc_sw_0_irq_handler(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + + if (IS_SRIOV_VF(gt_to_xe(gt))) + xe_gt_sriov_vf_migrated_event_handler(gt); +} + +void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir) +{ + if (iir & GUC_INTR_GUC2HOST) + xe_guc_ct_irq_handler(&guc->ct); + + if (iir & GUC_INTR_SW_INT_0) + xe_guc_sw_0_irq_handler(guc); +} + +void xe_guc_sanitize(struct xe_guc *guc) +{ + xe_uc_fw_sanitize(&guc->fw); + xe_guc_ct_disable(&guc->ct); + xe_guc_submit_disable(guc); +} + +int xe_guc_reset_prepare(struct xe_guc *guc) +{ + return xe_guc_submit_reset_prepare(guc); +} + +void xe_guc_reset_wait(struct xe_guc *guc) +{ + xe_guc_submit_reset_wait(guc); +} + +void xe_guc_stop_prepare(struct xe_guc *guc) +{ + if (!IS_SRIOV_VF(guc_to_xe(guc))) + xe_guc_pc_stop(&guc->pc); +} + +void xe_guc_stop(struct xe_guc *guc) +{ + xe_guc_ct_stop(&guc->ct); + + xe_guc_submit_stop(guc); +} + +int xe_guc_start(struct xe_guc *guc) +{ + return xe_guc_submit_start(guc); +} + +/** + * xe_guc_runtime_suspend() - GuC runtime suspend + * @guc: The GuC object + * + * Stop further runs of submission tasks on given GuC and runtime suspend + * GuC CT. + */ +void xe_guc_runtime_suspend(struct xe_guc *guc) +{ + xe_guc_submit_pause(guc); + xe_guc_submit_disable(guc); + xe_guc_ct_runtime_suspend(&guc->ct); +} + +/** + * xe_guc_runtime_resume() - GuC runtime resume + * @guc: The GuC object + * + * Runtime resume GuC CT and allow further runs of submission tasks on + * given GuC. + */ +void xe_guc_runtime_resume(struct xe_guc *guc) +{ + /* + * Runtime PM flows are not applicable for VFs, so it's safe to + * directly enable IRQ. + */ + guc_enable_irq(guc); + + xe_guc_ct_runtime_resume(&guc->ct); + xe_guc_submit_enable(guc); + xe_guc_submit_unpause(guc); +} + +int xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 status; + int i; + + xe_uc_fw_print(&guc->fw, p); + + if (!IS_SRIOV_VF(gt_to_xe(gt))) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -EIO; + + status = xe_mmio_read32(>->mmio, GUC_STATUS); + + drm_printf(p, "\nGuC status 0x%08x:\n", status); + drm_printf(p, "\tBootrom status = 0x%x\n", + REG_FIELD_GET(GS_BOOTROM_MASK, status)); + drm_printf(p, "\tuKernel status = 0x%x\n", + REG_FIELD_GET(GS_UKERNEL_MASK, status)); + drm_printf(p, "\tMIA Core status = 0x%x\n", + REG_FIELD_GET(GS_MIA_MASK, status)); + drm_printf(p, "\tLog level = %d\n", + xe_guc_log_get_level(&guc->log)); + + drm_puts(p, "\nScratch registers:\n"); + for (i = 0; i < SOFT_SCRATCH_COUNT; i++) { + drm_printf(p, "\t%2d: \t0x%x\n", + i, xe_mmio_read32(>->mmio, SOFT_SCRATCH(i))); + } + } + + drm_puts(p, "\n"); + xe_guc_ct_print(&guc->ct, p, false); + + drm_puts(p, "\n"); + xe_guc_submit_print(guc, p); + + return 0; +} + +/** + * xe_guc_declare_wedged() - Declare GuC wedged + * @guc: the GuC object + * + * Wedge the GuC which stops all submission, saves desired debug state, and + * cleans up anything which could timeout. + */ +void xe_guc_declare_wedged(struct xe_guc *guc) +{ + xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); + + xe_guc_reset_prepare(guc); + xe_guc_ct_stop(&guc->ct); + xe_guc_submit_wedge(guc); +} + +/** + * xe_guc_using_main_gamctrl_queues() - Detect which reporting queues to use. + * @guc: The GuC object + * + * For Xe3p and beyond, we want to program the hardware to use the + * "Main GAMCTRL queue" rather than the legacy queue before we upload + * the GuC firmware. This will allow the GuC to use a new set of + * registers for pagefault handling and avoid some unnecessary + * complications with MCR register range handling. + * + * Return: true if can use new main gamctrl queues. + */ +bool xe_guc_using_main_gamctrl_queues(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + + /* + * For Xe3p media gt (35), the GuC and the CS subunits may be still Xe3 + * that lacks the Main GAMCTRL support. Reserved bits from the GMD_ID + * inform the IP version of the subunits. + */ + if (xe_gt_is_media_type(gt) && MEDIA_VER(gt_to_xe(gt)) == 35) { + u32 val = xe_mmio_read32(>->mmio, GMD_ID); + u32 subip = REG_FIELD_GET(GMD_ID_SUBIP_FLAG_MASK, val); + + if (!subip) + return true; + + xe_gt_WARN(gt, subip != 1, + "GMD_ID has unknown value in the SUBIP_FLAG field - 0x%x\n", + subip); + + return false; + } + + return GT_VER(gt) >= 35; +} + +bool xe_guc_has_paging_engine(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = gt_to_xe(gt); + + /* + * On newer platforms the GuC now has a dedicated engine class for the + * special PAGING engine, which is the driver reserved BCS engine used + * for KMD paging/binding operations. GuC requires KMD to refer to this + * using the special PAGING engine class. Note that there is no new hw + * engine here, this is purely a sw view in the GuC itself, which we + * need to respect. + */ + + if (IS_SRIOV_VF(xe)) + return xe_gt_sriov_vf_paging_engines(gt); + + return xe->info.platform >= XE_NOVALAKE_S && + GUC_FIRMWARE_VER_AT_LEAST(guc, 70, 69, 0); +} + +/** + * xe_hwe_guc_logical_instance - Get the GuC-aligned logical instance of a + * hardware engine. + * @hwe: Hardware engine. + * + * For GuC backend usage, we should no longer use the raw logical instance + * directly. This helper must be used to retrieve the logical instance of the + * hardware engine, taking care of any necessary adjustments (such as the GuC + * PAGING engine mapping). This is assumed to be used in conjunction with the + * GuC engine class. + * + * Return: Logical instance, taking into account for stuff like GuC PAGING + * engine mapping. + */ +u16 xe_hwe_guc_logical_instance(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + + if (xe_guc_has_paging_engine(&hwe->gt->uc.guc) && + xe_gt_is_usm_hwe(gt, hwe)) { + int shift = gt->usm.paging_hwe0->logical_instance; + + xe_gt_assert(gt, shift <= hwe->logical_instance); + + /* GUC_PAGING_CLASS:guc_logical_instance */ + return hwe->logical_instance - shift; + } + + return hwe->logical_instance; +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_guc_g2g_test.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h new file mode 100644 index 000000000..61e3ee19a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_H_ +#define _XE_GUC_H_ + +#include "xe_gt.h" +#include "xe_guc_types.h" +#include "xe_hw_engine_types.h" +#include "xe_macros.h" + +/* + * GuC version number components are defined to be only 8-bit size, + * so converting to a 32bit 8.8.8 integer allows simple (and safe) + * numerical comparisons. + */ +#define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat)) +#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch) +#define MAKE_GUC_VER_ARGS(ver...) \ + (BUILD_BUG_ON_ZERO(COUNT_ARGS(ver) < 2 || COUNT_ARGS(ver) > 3) + \ + MAKE_GUC_VER(PICK_ARG1(ver), PICK_ARG2(ver), IF_ARGS(PICK_ARG3(ver), 0, PICK_ARG3(ver)))) + +#define GUC_SUBMIT_VER(guc) \ + MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]) +#define GUC_FIRMWARE_VER(guc) \ + MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_RELEASE]) +#define GUC_FIRMWARE_VER_AT_LEAST(guc, ver...) \ + xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver)) + +struct drm_printer; + +void xe_guc_comm_init_early(struct xe_guc *guc); +int xe_guc_init_noalloc(struct xe_guc *guc); +int xe_guc_init(struct xe_guc *guc); +int xe_guc_init_post_hwconfig(struct xe_guc *guc); +int xe_guc_post_load_init(struct xe_guc *guc); +int xe_guc_reset(struct xe_guc *guc); +int xe_guc_upload(struct xe_guc *guc); +int xe_guc_min_load_for_hwconfig(struct xe_guc *guc); +int xe_guc_enable_communication(struct xe_guc *guc); +int xe_guc_opt_in_features_enable(struct xe_guc *guc); +void xe_guc_runtime_suspend(struct xe_guc *guc); +void xe_guc_runtime_resume(struct xe_guc *guc); +int xe_guc_suspend(struct xe_guc *guc); +int xe_guc_softreset(struct xe_guc *guc); +void xe_guc_notify(struct xe_guc *guc); +int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr); +int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len); +int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request, u32 len, + u32 *response_buf); +int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val); +int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val); +void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir); +void xe_guc_sanitize(struct xe_guc *guc); +int xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p); +int xe_guc_reset_prepare(struct xe_guc *guc); +void xe_guc_reset_wait(struct xe_guc *guc); +void xe_guc_stop_prepare(struct xe_guc *guc); +void xe_guc_stop(struct xe_guc *guc); +int xe_guc_start(struct xe_guc *guc); +void xe_guc_declare_wedged(struct xe_guc *guc); +bool xe_guc_using_main_gamctrl_queues(struct xe_guc *guc); +bool xe_guc_has_paging_engine(struct xe_guc *guc); + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +int xe_guc_g2g_test_notification(struct xe_guc *guc, u32 *payload, u32 len); +#endif + +u16 xe_hwe_to_guc_class(struct xe_hw_engine *hwe); +u16 xe_hwe_guc_logical_instance(struct xe_hw_engine *hwe); + +static inline struct xe_gt *guc_to_gt(struct xe_guc *guc) +{ + return container_of(guc, struct xe_gt, uc.guc); +} + +static inline struct xe_device *guc_to_xe(struct xe_guc *guc) +{ + return gt_to_xe(guc_to_gt(guc)); +} + +static inline struct drm_device *guc_to_drm(struct xe_guc *guc) +{ + return &guc_to_xe(guc)->drm; +} + +/** + * xe_guc_fw_version_at_least() - Check if GuC is at least of given version. + * @guc: the &xe_guc + * @ver: the version to check + * + * The @ver should be prepared using MAKE_GUC_VER(major, minor, patch). + * + * Return: true if loaded GuC firmware is at least of given version, + * false otherwise. + */ +static inline bool xe_guc_fw_version_at_least(const struct xe_guc *guc, u32 ver) +{ + return GUC_FIRMWARE_VER(guc) >= ver; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c new file mode 100644 index 000000000..ff8eee383 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -0,0 +1,1128 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_ads.h" + +#include + +#include + +#include + +#include "abi/guc_actions_abi.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" +#include "xe_bo.h" +#include "xe_configfs.h" +#include "xe_gt.h" +#include "xe_gt_ccs_mode.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_buf.h" +#include "xe_guc_capture.h" +#include "xe_guc_ct.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_wa.h" + +/* Slack of a few additional entries per engine */ +#define ADS_REGSET_EXTRA_MAX 8 + +static struct xe_guc * +ads_to_guc(struct xe_guc_ads *ads) +{ + return container_of(ads, struct xe_guc, ads); +} + +static struct xe_gt * +ads_to_gt(struct xe_guc_ads *ads) +{ + return container_of(ads, struct xe_gt, uc.guc.ads); +} + +static struct xe_device * +ads_to_xe(struct xe_guc_ads *ads) +{ + return gt_to_xe(ads_to_gt(ads)); +} + +static struct iosys_map * +ads_to_map(struct xe_guc_ads *ads) +{ + return &ads->bo->vmap; +} + +/* UM Queue parameters: */ +#define GUC_UM_QUEUE_SIZE (SZ_64K) +#define GUC_PAGE_RES_TIMEOUT_US (-1) + +/* + * The Additional Data Struct (ADS) has pointers for different buffers used by + * the GuC. One gem object (ads->bo) contains the ADS struct itself (guc_ads) + * and most of the extra buffers linked via the ADS struct's entries. The UM + * fault queues (PAGE_FAULT, PAGE_FAULT_RESPONSE, ACCESS_COUNTER rings) are + * kept in a separate BO (ads->um_queue_bo) so that the full memset of ads->bo + * performed on every GT reset does not discard fault descriptors already + * written into the rings by the GPU. + * + * Layout of the ADS blob (ads->bo): + * + * +---------------------------------------+ <== base + * | guc_ads | + * +---------------------------------------+ + * | guc_policies | + * +---------------------------------------+ + * | guc_gt_system_info | + * +---------------------------------------+ + * | guc_engine_usage | + * +---------------------------------------+ + * | guc_um_init_params | + * +---------------------------------------+ <== static + * | guc_mmio_reg[countA] (engine 0.0) | + * | guc_mmio_reg[countB] (engine 0.1) | + * | guc_mmio_reg[countC] (engine 1.0) | + * | ... | + * +---------------------------------------+ <== dynamic + * | padding | + * +---------------------------------------+ <== 4K aligned + * | golden contexts | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | w/a KLVs | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | capture lists | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | private data | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + */ +struct __guc_ads_blob { + struct guc_ads ads; + struct guc_policies policies; + struct guc_gt_system_info system_info; + struct guc_engine_usage engine_usage; + struct guc_um_init_params um_init_params; + /* From here on, location is dynamic! Refer to above diagram. */ + struct guc_mmio_reg regset[]; +} __packed; + +#define ads_blob_read(ads_, field_) \ + xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ + struct __guc_ads_blob, field_) + +#define ads_blob_write(ads_, field_, val_) \ + xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ + struct __guc_ads_blob, field_, val_) + +#define info_map_write(xe_, map_, field_, val_) \ + xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_) + +#define info_map_read(xe_, map_, field_) \ + xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_) + +static size_t guc_ads_regset_size(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + + xe_assert(xe, ads->regset_size); + + return ads->regset_size; +} + +static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads) +{ + return PAGE_ALIGN(ads->golden_lrc_size); +} + +static u32 guc_ads_waklv_size(struct xe_guc_ads *ads) +{ + return PAGE_ALIGN(ads->ads_waklv_size); +} + +static size_t guc_ads_capture_size(struct xe_guc_ads *ads) +{ + return PAGE_ALIGN(ads->capture_size); +} + +static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) +{ + return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); +} + +static size_t guc_ads_regset_offset(struct xe_guc_ads *ads) +{ + return offsetof(struct __guc_ads_blob, regset); +} + +static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads) +{ + size_t offset; + + offset = guc_ads_regset_offset(ads) + + guc_ads_regset_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads) +{ + u32 offset; + + offset = guc_ads_golden_lrc_offset(ads) + + guc_ads_golden_lrc_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) +{ + size_t offset; + + offset = guc_ads_waklv_offset(ads) + + guc_ads_waklv_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) +{ + size_t offset; + + offset = guc_ads_capture_offset(ads) + + guc_ads_capture_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_size(struct xe_guc_ads *ads) +{ + return guc_ads_private_data_offset(ads) + + guc_ads_private_data_size(ads); +} + +static size_t calculate_regset_size(struct xe_gt *gt) +{ + struct xe_reg_sr_entry *sr_entry; + unsigned long sr_idx; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + unsigned int count = 0; + + for_each_hw_engine(hwe, gt, id) + xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry) + count++; + + count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES; + + if (XE_GT_WA(gt, 1607983814)) + count += LNCFCMOCS_REG_COUNT; + + return count * sizeof(struct guc_mmio_reg); +} + +static inline enum xe_engine_class guc_class_to_engine_class(u16 guc_class) +{ + switch (guc_class) { + case GUC_RENDER_CLASS: + return XE_ENGINE_CLASS_RENDER; + case GUC_VIDEO_CLASS: + return XE_ENGINE_CLASS_VIDEO_DECODE; + case GUC_VIDEOENHANCE_CLASS: + return XE_ENGINE_CLASS_VIDEO_ENHANCE; + case GUC_BLITTER_CLASS: + case GUC_PAGING_CLASS: + return XE_ENGINE_CLASS_COPY; + case GUC_COMPUTE_CLASS: + return XE_ENGINE_CLASS_COMPUTE; + case GUC_GSC_OTHER_CLASS: + return XE_ENGINE_CLASS_OTHER; + default: + XE_WARN_ON(guc_class); + return -1; + } +} + +static u32 engine_enable_mask(struct xe_gt *gt, u16 guc_class) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 mask = 0; + + for_each_hw_engine(hwe, gt, id) + if (xe_hwe_to_guc_class(hwe) == guc_class) + mask |= BIT(hwe->instance); + + /* We expect at most one paging engine per GuC instance, for now */ + if (guc_class == GUC_PAGING_CLASS) + xe_gt_assert(gt, !mask || is_power_of_2(mask)); + + return mask; +} + +static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) +{ + struct xe_gt *gt = ads_to_gt(ads); + size_t total_size = 0, alloc_size, real_size; + u16 guc_class; + + for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) { + enum xe_engine_class class = + guc_class_to_engine_class(guc_class); + + if (!engine_enable_mask(gt, guc_class)) + continue; + + real_size = xe_gt_lrc_size(gt, class); + alloc_size = PAGE_ALIGN(real_size); + total_size += alloc_size; + } + + return total_size; +} + +static void guc_waklv_enable(struct xe_guc_ads *ads, + u32 data[], u32 data_len_dw, + u32 *offset, u32 *remain, + enum xe_guc_klv_ids klv_id) +{ + size_t size = sizeof(u32) * (1 + data_len_dw); + + if (*remain < size) { + drm_warn(&ads_to_xe(ads)->drm, + "w/a klv buffer too small to add klv id 0x%04X\n", klv_id); + return; + } + + /* 16:16 key/length */ + xe_map_wr(ads_to_xe(ads), ads_to_map(ads), *offset, u32, + FIELD_PREP(GUC_KLV_0_KEY, klv_id) | FIELD_PREP(GUC_KLV_0_LEN, data_len_dw)); + /* data_len_dw dwords of data */ + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), + *offset + sizeof(u32), data, data_len_dw * sizeof(u32)); + + *offset += size; + *remain -= size; +} + +static void guc_waklv_init(struct xe_guc_ads *ads) +{ + struct xe_gt *gt = ads_to_gt(ads); + u64 addr_ggtt; + u32 offset, remain, size; + + offset = guc_ads_waklv_offset(ads); + remain = guc_ads_waklv_size(ads); + + if (XE_GT_WA(gt, 16021333562)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); + if (XE_GT_WA(gt, 18024947630)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING); + if (XE_GT_WA(gt, 16022287689)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE); + + if (XE_GT_WA(gt, 14022866841)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO); + + /* + * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now, + * the default value for this register is determined to be 0xC40. This could change in the + * future, so GuC depends on KMD to send it the correct value. + */ + if (XE_GT_WA(gt, 13011645652)) { + u32 data = 0xC40; + + guc_waklv_enable(ads, &data, 1, &offset, &remain, + GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE); + } + + if (XE_GT_WA(gt, 14022293748) || XE_GT_WA(gt, 22019794406)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET); + + if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH); + if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) { + u32 data[] = { + 0x0, + 0xF, + }; + guc_waklv_enable(ads, data, ARRAY_SIZE(data), &offset, &remain, + GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG); + } + + if (XE_GT_WA(gt, 14020001231)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT); + if (XE_GT_WA(gt, 14025515070) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 53)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG); + + if (XE_GT_WA(gt, 22022079272) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 62)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_REMAP_RANGED_TLB_INV); + + /* The GuC does not enable the sem_tok_64 feature on NVL-S */ + if (XE_GT_WA(gt, 16029897822) && gt_to_xe(gt)->info.platform != XE_NOVALAKE_S && + GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 69)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64); + + /* + * On GuC firmware 70.66 and above, use the Feature KLV (shared with the + * WA KLV buffer); older firmware uses GUC_CTL_DISABLE_MULTI_QUEUE in + * the init params instead. + */ + if (!xe_configfs_get_enable_multi_queue(to_pci_dev(gt_to_xe(gt)->drm.dev)) && + GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 66)) { + u32 data = 1; + + guc_waklv_enable(ads, &data, 1, &offset, &remain, + GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE); + } + + size = guc_ads_waklv_size(ads) - remain; + if (!size) + return; + + offset = guc_ads_waklv_offset(ads); + addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; + + ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt)); + ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt)); + ads_blob_write(ads, ads.wa_klv_size, size); +} + +static int calculate_waklv_size(struct xe_guc_ads *ads) +{ + /* + * A single page is both the minimum size possible and + * is sufficiently large enough for all current platforms. + */ + return SZ_4K; +} + +#define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64) + +int xe_guc_ads_init(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *bo; + + ads->golden_lrc_size = calculate_golden_lrc_size(ads); + ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); + ads->regset_size = calculate_regset_size(gt); + ads->ads_waklv_size = calculate_waklv_size(ads); + + bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ads->bo = bo; + + if (xe->info.has_usm) { + /* + * Allocate a separate BO for the HW fault ring (UM queues). + * + * Round the size up to the next power of two so that on iGPU + * (system memory, no IOMMU) the TTM pool issues a single + * alloc_pages(order=N) call, maximising the chance of getting + * a physically contiguous block. GuC requires contiguous DPA. + */ + size_t um_size = IS_DGFX(xe) ? + GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX : + roundup_pow_of_two(GUC_UM_QUEUE_SIZE * + GUC_UM_HW_QUEUE_MAX); + + u32 um_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE | + XE_BO_FLAG_NEEDS_UC; + + bo = xe_managed_bo_create_pin_map(xe, tile, um_size, um_flags); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + /* + * On pre-Xe3p platforms, GAM (not GuC) accesses the UM queue + * ring via base_dpa, which must be a contiguous DMA address + * range. Verify that the allocated pages are contiguous in + * DMA address space. + */ + if (!xe_bo_is_vram(bo) && + !xe_guc_using_main_gamctrl_queues(ads_to_guc(ads)) && + unlikely(!xe_bo_sg_is_contiguous(bo, + GUC_UM_QUEUE_SIZE * + GUC_UM_HW_QUEUE_MAX))) { + drm_err(&xe->drm, + "UM fault queue memory is not contiguous in DMA address space; GAM requires contiguous DPA\n"); + return -ENOMEM; + } + + ads->um_queue_bo = bo; + } + + return 0; +} +ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load + * @ads: Additional data structures object + * + * Recalculate golden_lrc_size, capture_size and regset_size as the number + * hardware engines may have changed after the hwconfig was loaded. Also verify + * the new sizes fit in the already allocated ADS buffer object. + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads) +{ + struct xe_gt *gt = ads_to_gt(ads); + u32 prev_regset_size = ads->regset_size; + + xe_gt_assert(gt, ads->bo); + + ads->golden_lrc_size = calculate_golden_lrc_size(ads); + /* Calculate Capture size with worst size */ + ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); + ads->regset_size = calculate_regset_size(gt); + + xe_gt_assert(gt, ads->golden_lrc_size + + (ads->regset_size - prev_regset_size) <= + MAX_GOLDEN_LRC_SIZE); + + return 0; +} + +static void guc_policies_init(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + u32 global_flags = 0; + + ads_blob_write(ads, policies.dpc_promote_time, + GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); + ads_blob_write(ads, policies.max_num_work_items, + GLOBAL_POLICY_MAX_NUM_WI); + + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) + global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; + + ads_blob_write(ads, policies.global_flags, global_flags); + ads_blob_write(ads, policies.is_valid, 1); +} + +static void fill_engine_enable_masks(struct xe_gt *gt, + struct iosys_map *info_map) +{ + struct xe_device *xe = gt_to_xe(gt); + u16 guc_class; + + for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) + info_map_write(xe, info_map, engine_enabled_masks[guc_class], + engine_enable_mask(gt, guc_class)); +} + +u16 xe_hwe_to_guc_class(struct xe_hw_engine *hwe) +{ + if (xe_guc_has_paging_engine(&hwe->gt->uc.guc) && + xe_gt_is_usm_hwe(hwe->gt, hwe)) + return GUC_PAGING_CLASS; + + switch (hwe->class) { + case XE_ENGINE_CLASS_RENDER: + return GUC_RENDER_CLASS; + case XE_ENGINE_CLASS_VIDEO_DECODE: + return GUC_VIDEO_CLASS; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + return GUC_VIDEOENHANCE_CLASS; + case XE_ENGINE_CLASS_COPY: + return GUC_BLITTER_CLASS; + case XE_ENGINE_CLASS_COMPUTE: + return GUC_COMPUTE_CLASS; + case XE_ENGINE_CLASS_OTHER: + return GUC_GSC_OTHER_CLASS; + default: + XE_WARN_ON(hwe->class); + return -1; + } +} + +/* + * Write the offsets corresponding to the golden LRCs. The actual data is + * populated later by guc_golden_lrc_populate() + */ +static void guc_golden_lrc_init(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + size_t alloc_size, real_size; + u32 addr_ggtt, offset; + u16 guc_class; + + offset = guc_ads_golden_lrc_offset(ads); + addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; + + for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) { + enum xe_engine_class class = + guc_class_to_engine_class(guc_class); + + if (!info_map_read(xe, &info_map, + engine_enabled_masks[guc_class])) + continue; + + real_size = xe_gt_lrc_size(gt, class); + alloc_size = PAGE_ALIGN(real_size); + + /* + * This interface is slightly confusing. We need to pass the + * base address of the full golden context and the size of just + * the engine state, which is the section of the context image + * that starts after the execlists LRC registers. This is + * required to allow the GuC to restore just the engine state + * when a watchdog reset occurs. + */ + ads_blob_write(ads, ads.eng_state_size[guc_class], + xe_lrc_engine_state_size(gt, class)); + ads_blob_write(ads, ads.golden_context_lrca[guc_class], + addr_ggtt); + + addr_ggtt += alloc_size; + } +} + +static void guc_mapping_table_init_invalid(struct xe_gt *gt, + struct iosys_map *info_map) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int i, j; + + /* Table must be set to invalid values for entries not used */ + for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) + for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) + info_map_write(xe, info_map, mapping_table[i][j], + GUC_MAX_INSTANCES_PER_CLASS); +} + +static void guc_mapping_table_init(struct xe_gt *gt, + struct iosys_map *info_map) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + guc_mapping_table_init_invalid(gt, info_map); + + for_each_hw_engine(hwe, gt, id) { + u16 guc_logical_instance; + u8 guc_class; + + guc_class = xe_hwe_to_guc_class(hwe); + guc_logical_instance = xe_hwe_guc_logical_instance(hwe); + + info_map_write(xe, info_map, + mapping_table[guc_class][guc_logical_instance], + hwe->instance); + } +} + +static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_map, + enum guc_capture_list_class_type capture_class) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 mask; + + switch (capture_class) { + case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: + mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS]); + mask |= info_map_read(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); + break; + case GUC_CAPTURE_LIST_CLASS_VIDEO: + mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); + break; + case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: + mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); + break; + case GUC_CAPTURE_LIST_CLASS_BLITTER: + mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); + break; + case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: + mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); + break; + case GUC_CAPTURE_LIST_CLASS_PAGING: + mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_PAGING_CLASS]); + break; + default: + mask = 0; + } + + return mask; +} + +static inline bool get_capture_list(struct xe_guc_ads *ads, struct xe_guc *guc, struct xe_gt *gt, + int owner, int type, int class, u32 *total_size, size_t *size, + void **pptr) +{ + *size = 0; + + if (!xe_guc_capture_getlistsize(guc, owner, type, class, size)) { + if (*total_size + *size > ads->capture_size) + xe_gt_dbg(gt, "Capture size overflow :%zu vs %d\n", + *total_size + *size, ads->capture_size); + else if (!xe_guc_capture_getlist(guc, owner, type, class, pptr)) + return false; + } + + return true; +} + +static int guc_capture_prep_lists(struct xe_guc_ads *ads) +{ + struct xe_guc *guc = ads_to_guc(ads); + struct xe_gt *gt = ads_to_gt(ads); + u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; + struct iosys_map info_map; + size_t size = 0; + void *ptr; + int i, j; + + /* + * GuC Capture's steered reg-list needs to be allocated and initialized + * after the GuC-hwconfig is available which guaranteed from here. + */ + xe_guc_capture_steered_list_init(ads_to_guc(ads)); + + capture_offset = guc_ads_capture_offset(ads); + ads_ggtt = xe_bo_ggtt_addr(ads->bo); + info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + + /* first, set aside the first page for a capture_list with zero descriptors */ + total_size = PAGE_SIZE; + if (!xe_guc_capture_getnullheader(guc, &ptr, &size)) + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, size); + + null_ggtt = ads_ggtt + capture_offset; + capture_offset += PAGE_SIZE; + + /* + * Populate capture list : at this point adps is already allocated and + * mapped to worst case size + */ + for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { + bool write_empty_list; + + for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) { + u32 engine_mask = guc_get_capture_engine_mask(gt, &info_map, j); + /* null list if we dont have said engine or list */ + if (!engine_mask) { + ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); + ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); + continue; + } + + /* engine exists: start with engine-class registers */ + write_empty_list = get_capture_list(ads, guc, gt, i, + GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + j, &total_size, &size, &ptr); + if (!write_empty_list) { + ads_blob_write(ads, ads.capture_class[i][j], + ads_ggtt + capture_offset); + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, + ptr, size); + total_size += size; + capture_offset += size; + } else { + ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); + } + + /* engine exists: next, engine-instance registers */ + write_empty_list = get_capture_list(ads, guc, gt, i, + GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE, + j, &total_size, &size, &ptr); + if (!write_empty_list) { + ads_blob_write(ads, ads.capture_instance[i][j], + ads_ggtt + capture_offset); + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, + ptr, size); + total_size += size; + capture_offset += size; + } else { + ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); + } + } + + /* global registers is last in our PF/VF loops */ + write_empty_list = get_capture_list(ads, guc, gt, i, + GUC_STATE_CAPTURE_TYPE_GLOBAL, + 0, &total_size, &size, &ptr); + if (!write_empty_list) { + ads_blob_write(ads, ads.capture_global[i], ads_ggtt + capture_offset); + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, + size); + total_size += size; + capture_offset += size; + } else { + ads_blob_write(ads, ads.capture_global[i], null_ggtt); + } + } + + if (ads->capture_size != PAGE_ALIGN(total_size)) + xe_gt_dbg(gt, "Updated ADS capture size %d (was %d)\n", + PAGE_ALIGN(total_size), ads->capture_size); + return PAGE_ALIGN(total_size); +} + +static void guc_mmio_regset_write_one(struct xe_guc_ads *ads, + struct iosys_map *regset_map, + struct xe_reg reg, + unsigned int n_entry) +{ + struct guc_mmio_reg entry = { + .offset = reg.addr, + .flags = reg.masked ? GUC_REGSET_MASKED : 0, + }; + + if (reg.mcr) { + struct xe_reg_mcr mcr_reg = XE_REG_MCR(reg.addr); + u8 group, instance; + + bool steer = xe_gt_mcr_get_nonterminated_steering(ads_to_gt(ads), mcr_reg, + &group, &instance); + + if (steer) { + entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, group); + entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, instance); + entry.flags |= GUC_REGSET_STEERING_NEEDED; + } + } + + xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry), + &entry, sizeof(entry)); +} + +static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, + struct iosys_map *regset_map, + struct xe_hw_engine *hwe) +{ + struct xe_hw_engine *hwe_rcs_reset_domain = + xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER); + struct xe_reg_sr_entry *entry; + unsigned long idx; + unsigned int count = 0; + const struct { + struct xe_reg reg; + bool skip; + } *e, extra_regs[] = { + { .reg = RING_HWS_PGA(hwe->mmio_base), }, + { .reg = RING_IMR(hwe->mmio_base), }, + { .reg = CCS_MODE, + .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) }, + }; + u32 i; + + BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX); + + xa_for_each(&hwe->reg_sr.xa, idx, entry) + guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++); + + for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) { + if (e->skip) + continue; + + guc_mmio_regset_write_one(ads, regset_map, e->reg, count++); + } + + if (XE_GT_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) { + for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) { + guc_mmio_regset_write_one(ads, regset_map, + XELP_LNCFCMOCS(i), count++); + } + } + + if (XE_GT_WA(hwe->gt, 16023105232)) + guc_mmio_regset_write_one(ads, regset_map, + RING_IDLEDLY(hwe->mmio_base), + count++); + + return count; +} + +static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) +{ + size_t regset_offset = guc_ads_regset_offset(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset; + struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + regset_offset); + unsigned int regset_used = 0; + + for_each_hw_engine(hwe, gt, id) { + unsigned int count; + u8 gc; + + /* + * 1. Write all MMIO entries for this exec queue to the table. No + * need to worry about fused-off engines and when there are + * entries in the regset: the reg_state_list has been zero'ed + * by xe_guc_ads_populate() + */ + count = guc_mmio_regset_write(ads, ®set_map, hwe); + if (!count) + continue; + + /* + * 2. Record in the header (ads.reg_state_list) the address + * location and number of entries + */ + gc = xe_hwe_to_guc_class(hwe); + ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr); + ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count); + + addr += count * sizeof(struct guc_mmio_reg); + iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg)); + + regset_used += count * sizeof(struct guc_mmio_reg); + } + + xe_gt_assert(gt, regset_used <= ads->regset_size); +} + +static void guc_um_init_params(struct xe_guc_ads *ads) +{ + struct xe_bo *um_bo = ads->um_queue_bo; + struct xe_guc *guc = ads_to_guc(ads); + struct xe_device *xe = ads_to_xe(ads); + u64 base_dpa; + u32 base_ggtt; + bool with_dpa; + int i; + + with_dpa = !xe_guc_using_main_gamctrl_queues(guc); + + if (um_bo) { + /* All USM platforms: UM queues in dedicated um_queue_bo */ + base_ggtt = xe_bo_ggtt_addr(um_bo); + base_dpa = xe_bo_main_addr(um_bo, PAGE_SIZE); + } else { + /* Platform does not support USM: no UM queues, nothing to do */ + return; + } + + for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { + /* + * Some platforms support USM but not access counters. + * Skip ACCESS_COUNTER queue initialization for such + * platforms, leaving queue_params[2] zero-initialized + * to signal unavailability to the GuC. + */ + if (i == GUC_UM_HW_QUEUE_ACCESS_COUNTER && + !xe->info.has_access_counter) + continue; + + ads_blob_write(ads, um_init_params.queue_params[i].base_dpa, + with_dpa ? (base_dpa + (i * GUC_UM_QUEUE_SIZE)) : 0); + ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address, + base_ggtt + (i * GUC_UM_QUEUE_SIZE)); + ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes, + GUC_UM_QUEUE_SIZE); + } + + ads_blob_write(ads, um_init_params.page_response_timeout_in_us, + GUC_PAGE_RES_TIMEOUT_US); +} + +static void guc_doorbell_init(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + + if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) { + u32 distdbreg = + xe_mmio_read32(>->mmio, DIST_DBS_POPULATED); + + ads_blob_write(ads, + system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], + REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1); + } +} + +/** + * xe_guc_ads_populate_minimal - populate minimal ADS + * @ads: Additional data structures object + * + * This function populates a minimal ADS that does not support submissions but + * enough so the GuC can load and the hwconfig table can be read. + */ +void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads) +{ + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + u32 base = xe_bo_ggtt_addr(ads->bo); + + xe_gt_assert(gt, ads->bo); + + xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); + guc_policies_init(ads); + guc_golden_lrc_init(ads); + guc_mapping_table_init_invalid(gt, &info_map); + guc_doorbell_init(ads); + + ads_blob_write(ads, ads.scheduler_policies, base + + offsetof(struct __guc_ads_blob, policies)); + ads_blob_write(ads, ads.gt_system_info, base + + offsetof(struct __guc_ads_blob, system_info)); + ads_blob_write(ads, ads.private_data, base + + guc_ads_private_data_offset(ads)); +} + +void xe_guc_ads_populate(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + u32 base = xe_bo_ggtt_addr(ads->bo); + + xe_gt_assert(gt, ads->bo); + + xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); + guc_policies_init(ads); + fill_engine_enable_masks(gt, &info_map); + guc_mmio_reg_state_init(ads); + guc_golden_lrc_init(ads); + guc_mapping_table_init(gt, &info_map); + guc_capture_prep_lists(ads); + guc_doorbell_init(ads); + guc_waklv_init(ads); + + if (xe->info.has_usm) { + guc_um_init_params(ads); + ads_blob_write(ads, ads.um_init_data, base + + offsetof(struct __guc_ads_blob, um_init_params)); + } + + ads_blob_write(ads, ads.scheduler_policies, base + + offsetof(struct __guc_ads_blob, policies)); + ads_blob_write(ads, ads.gt_system_info, base + + offsetof(struct __guc_ads_blob, system_info)); + ads_blob_write(ads, ads.private_data, base + + guc_ads_private_data_offset(ads)); +} + +/* + * After the golden LRC's are recorded for each engine class by the first + * submission, copy them to the ADS, as initialized earlier by + * guc_golden_lrc_init(). + */ +static void guc_golden_lrc_populate(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + size_t total_size = 0, alloc_size, real_size; + u32 offset; + u16 guc_class; + + offset = guc_ads_golden_lrc_offset(ads); + + for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) { + enum xe_engine_class class = + guc_class_to_engine_class(guc_class); + + if (!info_map_read(xe, &info_map, + engine_enabled_masks[guc_class])) + continue; + + xe_gt_assert(gt, gt->default_lrc[class]); + + real_size = xe_gt_lrc_size(gt, class); + alloc_size = PAGE_ALIGN(real_size); + total_size += alloc_size; + + xe_map_memcpy_to(xe, ads_to_map(ads), offset, + gt->default_lrc[class], real_size); + + offset += alloc_size; + } + + xe_gt_assert(gt, total_size == ads->golden_lrc_size); +} + +void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads) +{ + guc_golden_lrc_populate(ads); +} + +static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset) +{ + struct xe_guc_ct *ct = &ads_to_guc(ads)->ct; + u32 action[] = { + XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, + policy_offset + }; + + return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); +} + +/** + * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy + * @ads: Additional data structures object + * @enable_engine_reset: true to enable engine resets, false otherwise + * + * This function update the GuC's engine reset policy. + * + * Return: 0 on success, and negative error code otherwise. + */ +int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads, + bool enable_engine_reset) +{ + struct guc_policies *policies; + struct xe_guc *guc = ads_to_guc(ads); + CLASS(xe_guc_buf, buf)(&guc->buf, sizeof(*policies)); + + if (!xe_guc_buf_is_valid(buf)) + return -ENOBUFS; + + policies = xe_guc_buf_cpu_ptr(buf); + memset(policies, 0, sizeof(*policies)); + + policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time); + policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items); + policies->is_valid = 1; + + if (enable_engine_reset) + policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET; + else + policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; + + return guc_ads_action_update_policies(ads, xe_guc_buf_flush(buf)); +} diff --git a/drivers/gpu/drm/xe/xe_guc_ads.h b/drivers/gpu/drm/xe/xe_guc_ads.h new file mode 100644 index 000000000..7a39f361c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ads.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_ADS_H_ +#define _XE_GUC_ADS_H_ + +#include + +struct xe_guc_ads; + +int xe_guc_ads_init(struct xe_guc_ads *ads); +int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads); +void xe_guc_ads_populate(struct xe_guc_ads *ads); +void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads); +void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads); +int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads, + bool enable_engine_reset); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ads_types.h b/drivers/gpu/drm/xe/xe_guc_ads_types.h new file mode 100644 index 000000000..845c1fbd9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ads_types.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_ADS_TYPES_H_ +#define _XE_GUC_ADS_TYPES_H_ + +#include + +struct xe_bo; + +/** + * struct xe_guc_ads - GuC additional data structures (ADS) + */ +struct xe_guc_ads { + /** @bo: Xe BO for GuC ads blob */ + struct xe_bo *bo; + /** + * @um_queue_bo: Dedicated BO for the HW fault ring (UM queues). + * NULL if the platform does not support USM. + */ + struct xe_bo *um_queue_bo; + /** @golden_lrc_size: golden LRC size */ + size_t golden_lrc_size; + /** @regset_size: size of register set passed to GuC for save/restore */ + u32 regset_size; + /** @ads_waklv_size: total waklv size supported by platform */ + u32 ads_waklv_size; + /** @capture_size: size of register set passed to GuC for capture */ + u32 capture_size; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_buf.c b/drivers/gpu/drm/xe/xe_guc_buf.c new file mode 100644 index 000000000..11f77decd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_buf.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include +#include + +#include "xe_bo.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_buf.h" +#include "xe_sa.h" + +#define XE_GUC_BUF_CACHE_DEFAULT_SIZE SZ_8K + +static struct xe_guc *cache_to_guc(struct xe_guc_buf_cache *cache) +{ + return container_of(cache, struct xe_guc, buf); +} + +static struct xe_gt *cache_to_gt(struct xe_guc_buf_cache *cache) +{ + return guc_to_gt(cache_to_guc(cache)); +} + +static int guc_buf_cache_init(struct xe_guc_buf_cache *cache, u32 size) +{ + struct xe_gt *gt = cache_to_gt(cache); + struct xe_sa_manager *sam; + + sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32), 0); + if (IS_ERR(sam)) + return PTR_ERR(sam); + cache->sam = sam; + + xe_gt_dbg(gt, "reusable buffer with %u dwords at %#x for %ps\n", + xe_guc_buf_cache_dwords(cache), xe_bo_ggtt_addr(sam->bo), + __builtin_return_address(0)); + return 0; +} + +/** + * xe_guc_buf_cache_init() - Initialize the GuC Buffer Cache. + * @cache: the &xe_guc_buf_cache to initialize + * + * The Buffer Cache allows to obtain a reusable buffer that can be used to pass + * data to GuC or read data from GuC without a need to create a ad-hoc allocation. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_buf_cache_init(struct xe_guc_buf_cache *cache) +{ + return guc_buf_cache_init(cache, XE_GUC_BUF_CACHE_DEFAULT_SIZE); +} + +/** + * xe_guc_buf_cache_init_with_size() - Initialize the GuC Buffer Cache. + * @cache: the &xe_guc_buf_cache to initialize + * @size: size in bytes + * + * Like xe_guc_buf_cache_init(), except it allows the caller to make the cache + * buffer larger, allowing to accommodate larger objects. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_buf_cache_init_with_size(struct xe_guc_buf_cache *cache, u32 size) +{ + return guc_buf_cache_init(cache, max(XE_GUC_BUF_CACHE_DEFAULT_SIZE, size)); +} + +/** + * xe_guc_buf_cache_dwords() - Number of dwords the GuC Buffer Cache supports. + * @cache: the &xe_guc_buf_cache to query + * + * Return: a size of the largest reusable buffer (in dwords) + */ +u32 xe_guc_buf_cache_dwords(struct xe_guc_buf_cache *cache) +{ + return cache->sam ? cache->sam->base.size / sizeof(u32) : 0; +} + +/** + * xe_guc_buf_reserve() - Reserve a new sub-allocation. + * @cache: the &xe_guc_buf_cache where reserve sub-allocation + * @dwords: the requested size of the buffer in dwords + * + * Use xe_guc_buf_is_valid() to check if returned buffer reference is valid. + * Must use xe_guc_buf_release() to release a sub-allocation. + * + * Return: a &xe_guc_buf of new sub-allocation. + */ +struct xe_guc_buf xe_guc_buf_reserve(struct xe_guc_buf_cache *cache, u32 dwords) +{ + struct drm_suballoc *sa; + + if (cache->sam) + sa = __xe_sa_bo_new(cache->sam, dwords * sizeof(u32), GFP_ATOMIC); + else + sa = ERR_PTR(-EOPNOTSUPP); + + return (struct xe_guc_buf){ .sa = sa }; +} + +/** + * xe_guc_buf_from_data() - Reserve a new sub-allocation using data. + * @cache: the &xe_guc_buf_cache where reserve sub-allocation + * @data: the data to flush the sub-allocation + * @size: the size of the data + * + * Similar to xe_guc_buf_reserve() but flushes @data to the GPU memory. + * + * Return: a &xe_guc_buf of new sub-allocation. + */ +struct xe_guc_buf xe_guc_buf_from_data(struct xe_guc_buf_cache *cache, + const void *data, size_t size) +{ + struct drm_suballoc *sa; + + sa = __xe_sa_bo_new(cache->sam, size, GFP_ATOMIC); + if (!IS_ERR(sa)) + memcpy(xe_sa_bo_cpu_addr(sa), data, size); + + return (struct xe_guc_buf){ .sa = sa }; +} + +/** + * xe_guc_buf_release() - Release a sub-allocation. + * @buf: the &xe_guc_buf to release + * + * Releases a sub-allocation reserved by the xe_guc_buf_reserve(). + */ +void xe_guc_buf_release(const struct xe_guc_buf buf) +{ + if (xe_guc_buf_is_valid(buf)) + xe_sa_bo_free(buf.sa, NULL); +} + +/** + * xe_guc_buf_sync_read() - Copy the data from the GPU memory to the sub-allocation. + * @buf: the &xe_guc_buf to sync + * + * Return: a CPU pointer of the sub-allocation. + */ +void *xe_guc_buf_sync_read(const struct xe_guc_buf buf) +{ + xe_sa_bo_sync_read(buf.sa); + + return xe_sa_bo_cpu_addr(buf.sa); +} + +/** + * xe_guc_buf_flush() - Copy the data from the sub-allocation to the GPU memory. + * @buf: the &xe_guc_buf to flush + * + * Return: a GPU address of the sub-allocation. + */ +u64 xe_guc_buf_flush(const struct xe_guc_buf buf) +{ + xe_sa_bo_flush_write(buf.sa); + return xe_sa_bo_gpu_addr(buf.sa); +} + +/** + * xe_guc_buf_cpu_ptr() - Obtain a CPU pointer to the sub-allocation. + * @buf: the &xe_guc_buf to query + * + * Return: a CPU pointer of the sub-allocation. + */ +void *xe_guc_buf_cpu_ptr(const struct xe_guc_buf buf) +{ + return xe_sa_bo_cpu_addr(buf.sa); +} + +/** + * xe_guc_buf_gpu_addr() - Obtain a GPU address of the sub-allocation. + * @buf: the &xe_guc_buf to query + * + * Return: a GPU address of the sub-allocation. + */ +u64 xe_guc_buf_gpu_addr(const struct xe_guc_buf buf) +{ + return xe_sa_bo_gpu_addr(buf.sa); +} + +/** + * xe_guc_cache_gpu_addr_from_ptr() - Lookup a GPU address using the pointer. + * @cache: the &xe_guc_buf_cache with sub-allocations + * @ptr: the CPU pointer of the sub-allocation + * @size: the size of the data + * + * Return: a GPU address on success or 0 if the pointer was unrelated. + */ +u64 xe_guc_cache_gpu_addr_from_ptr(struct xe_guc_buf_cache *cache, const void *ptr, u32 size) +{ + ptrdiff_t offset = ptr - cache->sam->cpu_ptr; + + if (offset < 0 || offset + size > cache->sam->base.size) + return 0; + + return xe_sa_manager_gpu_addr(cache->sam) + offset; +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_guc_buf_kunit.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_buf.h b/drivers/gpu/drm/xe/xe_guc_buf.h new file mode 100644 index 000000000..e3cca553f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_buf.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GUC_BUF_H_ +#define _XE_GUC_BUF_H_ + +#include +#include + +#include "xe_guc_buf_types.h" + +int xe_guc_buf_cache_init(struct xe_guc_buf_cache *cache); +int xe_guc_buf_cache_init_with_size(struct xe_guc_buf_cache *cache, u32 size); +u32 xe_guc_buf_cache_dwords(struct xe_guc_buf_cache *cache); +struct xe_guc_buf xe_guc_buf_reserve(struct xe_guc_buf_cache *cache, u32 dwords); +struct xe_guc_buf xe_guc_buf_from_data(struct xe_guc_buf_cache *cache, + const void *data, size_t size); +void xe_guc_buf_release(const struct xe_guc_buf buf); + +/** + * xe_guc_buf_is_valid() - Check if a buffer reference is valid. + * @buf: the &xe_guc_buf reference to check + * + * Return: true if @ref represents a valid sub-allication. + */ +static inline bool xe_guc_buf_is_valid(const struct xe_guc_buf buf) +{ + return !IS_ERR_OR_NULL(buf.sa); +} + +void *xe_guc_buf_cpu_ptr(const struct xe_guc_buf buf); +void *xe_guc_buf_sync_read(const struct xe_guc_buf buf); +u64 xe_guc_buf_flush(const struct xe_guc_buf buf); +u64 xe_guc_buf_gpu_addr(const struct xe_guc_buf buf); +u64 xe_guc_cache_gpu_addr_from_ptr(struct xe_guc_buf_cache *cache, const void *ptr, u32 size); + +DEFINE_CLASS(xe_guc_buf, struct xe_guc_buf, + xe_guc_buf_release(_T), + xe_guc_buf_reserve(cache, num), + struct xe_guc_buf_cache *cache, u32 num); + +DEFINE_CLASS(xe_guc_buf_from_data, struct xe_guc_buf, + xe_guc_buf_release(_T), + xe_guc_buf_from_data(cache, data, size), + struct xe_guc_buf_cache *cache, const void *data, size_t size); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_buf_types.h b/drivers/gpu/drm/xe/xe_guc_buf_types.h new file mode 100644 index 000000000..9e123d71c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_buf_types.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GUC_BUF_TYPES_H_ +#define _XE_GUC_BUF_TYPES_H_ + +struct drm_suballoc; +struct xe_sa_manager; + +/** + * struct xe_guc_buf_cache - GuC Data Buffer Cache. + */ +struct xe_guc_buf_cache { + /* private: internal sub-allocation manager */ + struct xe_sa_manager *sam; +}; + +/** + * struct xe_guc_buf - GuC Data Buffer Reference. + */ +struct xe_guc_buf { + /* private: internal sub-allocation reference */ + struct drm_suballoc *sa; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c new file mode 100644 index 000000000..82df19b30 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -0,0 +1,2057 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2024 Intel Corporation + */ + +#include + +#include +#include + +#include "abi/guc_actions_abi.h" +#include "abi/guc_capture_abi.h" +#include "abi/guc_log_abi.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" + +#include "xe_bo_types.h" +#include "xe_device.h" +#include "xe_exec_queue_types.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_capture.h" +#include "xe_guc_capture_types.h" +#include "xe_guc_ct.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_guc_log.h" +#include "xe_guc_submit_types.h" +#include "xe_guc_submit.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_sched_job.h" + +/* + * struct __guc_capture_bufstate + * + * Book-keeping structure used to track read and write pointers + * as we extract error capture data from the GuC-log-buffer's + * error-capture region as a stream of dwords. + */ +struct __guc_capture_bufstate { + u32 size; + u32 data_offset; + u32 rd; + u32 wr; +}; + +/* + * struct __guc_capture_parsed_output - extracted error capture node + * + * A single unit of extracted error-capture output data grouped together + * at an engine-instance level. We keep these nodes in a linked list. + * See cachelist and outlist below. + */ +struct __guc_capture_parsed_output { + /* + * A single set of 3 capture lists: a global-list + * an engine-class-list and an engine-instance list. + * outlist in __guc_capture_parsed_output will keep + * a linked list of these nodes that will eventually + * be detached from outlist and attached into to + * xe_codedump in response to a context reset + */ + struct list_head link; + bool is_partial; + u32 eng_class; + u32 eng_inst; + u32 guc_id; + u32 lrca; + u32 type; + bool locked; + enum xe_hw_engine_snapshot_source_id source; + struct gcap_reg_list_info { + u32 vfid; + u32 num_regs; + struct guc_mmio_reg *regs; + } reginfo[GUC_STATE_CAPTURE_TYPE_MAX]; +#define GCAP_PARSED_REGLIST_INDEX_GLOBAL BIT(GUC_STATE_CAPTURE_TYPE_GLOBAL) +#define GCAP_PARSED_REGLIST_INDEX_ENGCLASS BIT(GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS) +}; + +/* + * Define all device tables of GuC error capture register lists + * NOTE: + * For engine-registers, GuC only needs the register offsets + * from the engine-mmio-base + * + * 64 bit registers need 2 entries for low 32 bit register and high 32 bit + * register, for example: + * Register data_type flags mask Register name + * { XXX_REG_LO(0), REG_64BIT_LOW_DW, 0, 0, NULL}, + * { XXX_REG_HI(0), REG_64BIT_HI_DW,, 0, 0, "XXX_REG"}, + * 1. data_type: Indicate is hi/low 32 bit for a 64 bit register + * A 64 bit register define requires 2 consecutive entries, + * with low dword first and hi dword the second. + * 2. Register name: null for incompleted define + * 3. Incorrect order will trigger XE_WARN. + */ +#define COMMON_XELP_BASE_GLOBAL \ + { FORCEWAKE_GT, REG_32BIT, 0, 0, 0, "FORCEWAKE_GT"} + +#define COMMON_BASE_ENGINE_INSTANCE \ + { RING_HWSTAM(0), REG_32BIT, 0, 0, 0, "HWSTAM"}, \ + { RING_HWS_PGA(0), REG_32BIT, 0, 0, 0, "RING_HWS_PGA"}, \ + { RING_HEAD(0), REG_32BIT, 0, 0, 0, "RING_HEAD"}, \ + { RING_TAIL(0), REG_32BIT, 0, 0, 0, "RING_TAIL"}, \ + { RING_CTL(0), REG_32BIT, 0, 0, 0, "RING_CTL"}, \ + { RING_MI_MODE(0), REG_32BIT, 0, 0, 0, "RING_MI_MODE"}, \ + { GFX_MODE(0), REG_32BIT, 0, 0, 0, "GFX_MODE"}, \ + { RING_ESR(0), REG_32BIT, 0, 0, 0, "RING_ESR"}, \ + { RING_EMR(0), REG_32BIT, 0, 0, 0, "RING_EMR"}, \ + { RING_EIR(0), REG_32BIT, 0, 0, 0, "RING_EIR"}, \ + { RING_IMR(0), REG_32BIT, 0, 0, 0, "RING_IMR"}, \ + { RING_IPEHR(0), REG_32BIT, 0, 0, 0, "IPEHR"}, \ + { RING_INSTDONE(0), REG_32BIT, 0, 0, 0, "RING_INSTDONE"}, \ + { INDIRECT_RING_STATE(0), REG_32BIT, 0, 0, 0, "INDIRECT_RING_STATE"}, \ + { RING_CURRENT_LRCA(0), REG_32BIT, 0, 0, 0, "CURRENT_LRCA"}, \ + { RING_ACTHD(0), REG_64BIT_LOW_DW, 0, 0, 0, NULL}, \ + { RING_ACTHD_UDW(0), REG_64BIT_HI_DW, 0, 0, 0, "ACTHD"}, \ + { RING_BBADDR(0), REG_64BIT_LOW_DW, 0, 0, 0, NULL}, \ + { RING_BBADDR_UDW(0), REG_64BIT_HI_DW, 0, 0, 0, "RING_BBADDR"}, \ + { RING_START(0), REG_64BIT_LOW_DW, 0, 0, 0, NULL}, \ + { RING_START_UDW(0), REG_64BIT_HI_DW, 0, 0, 0, "RING_START"}, \ + { RING_DMA_FADD(0), REG_64BIT_LOW_DW, 0, 0, 0, NULL}, \ + { RING_DMA_FADD_UDW(0), REG_64BIT_HI_DW, 0, 0, 0, "RING_DMA_FADD"}, \ + { RING_EXECLIST_STATUS_LO(0), REG_64BIT_LOW_DW, 0, 0, 0, NULL}, \ + { RING_EXECLIST_STATUS_HI(0), REG_64BIT_HI_DW, 0, 0, 0, "RING_EXECLIST_STATUS"}, \ + { RING_EXECLIST_SQ_CONTENTS_LO(0), REG_64BIT_LOW_DW, 0, 0, 0, NULL}, \ + { RING_EXECLIST_SQ_CONTENTS_HI(0), REG_64BIT_HI_DW, 0, 0, 0, "RING_EXECLIST_SQ_CONTENTS"} + +#define COMMON_XELP_RC_CLASS \ + { RCU_MODE, REG_32BIT, 0, 0, 0, "RCU_MODE"} + +#define COMMON_XELP_RC_CLASS_INSTDONE \ + { SC_INSTDONE, REG_32BIT, 0, 0, 0, "SC_INSTDONE"}, \ + { SC_INSTDONE_EXTRA, REG_32BIT, 0, 0, 0, "SC_INSTDONE_EXTRA"}, \ + { SC_INSTDONE_EXTRA2, REG_32BIT, 0, 0, 0, "SC_INSTDONE_EXTRA2"} + +#define XELP_VEC_CLASS_REGS \ + { SFC_DONE(0), 0, 0, 0, 0, "SFC_DONE[0]"}, \ + { SFC_DONE(1), 0, 0, 0, 0, "SFC_DONE[1]"}, \ + { SFC_DONE(2), 0, 0, 0, 0, "SFC_DONE[2]"}, \ + { SFC_DONE(3), 0, 0, 0, 0, "SFC_DONE[3]"} + +#define XE3P_BASE_ENGINE_INSTANCE \ + { RING_CSMQDEBUG(0), REG_32BIT, 0, 0, 0, "CSMQDEBUG"} + +/* XE_LP Global */ +static const struct __guc_mmio_reg_descr xe_lp_global_regs[] = { + COMMON_XELP_BASE_GLOBAL, +}; + +/* Render / Compute Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe_rc_inst_regs[] = { + COMMON_BASE_ENGINE_INSTANCE, +}; + +/* Render / Compute Engine-Class */ +static const struct __guc_mmio_reg_descr xe_rc_class_regs[] = { + COMMON_XELP_RC_CLASS, + COMMON_XELP_RC_CLASS_INSTDONE, +}; + +/* Render / Compute Engine-Class for xehpg */ +static const struct __guc_mmio_reg_descr xe_hpg_rc_class_regs[] = { + COMMON_XELP_RC_CLASS, +}; + +/* Media Decode/Encode Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe_vd_inst_regs[] = { + COMMON_BASE_ENGINE_INSTANCE, +}; + +/* Video Enhancement Engine-Class */ +static const struct __guc_mmio_reg_descr xe_vec_class_regs[] = { + XELP_VEC_CLASS_REGS, +}; + +/* Video Enhancement Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe_vec_inst_regs[] = { + COMMON_BASE_ENGINE_INSTANCE, +}; + +/* Blitter Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe_blt_inst_regs[] = { + COMMON_BASE_ENGINE_INSTANCE, +}; + +/* XE_LP - GSC Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe_lp_gsc_inst_regs[] = { + COMMON_BASE_ENGINE_INSTANCE, +}; + +/* Render / Compute Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe3p_rc_inst_regs[] = { + COMMON_BASE_ENGINE_INSTANCE, + XE3P_BASE_ENGINE_INSTANCE, +}; + +/* + * Empty list to prevent warnings about unknown class/instance types + * as not all class/instance types have entries on all platforms. + */ +static const struct __guc_mmio_reg_descr empty_regs_list[] = { +}; + +#define TO_GCAP_DEF_OWNER(x) (GUC_CAPTURE_LIST_INDEX_##x) +#define TO_GCAP_DEF_TYPE(x) (GUC_STATE_CAPTURE_TYPE_##x) +#define MAKE_REGLIST(regslist, regsowner, regstype, class) \ + { \ + regslist, \ + ARRAY_SIZE(regslist), \ + TO_GCAP_DEF_OWNER(regsowner), \ + TO_GCAP_DEF_TYPE(regstype), \ + class \ + } + +/* List of lists for legacy graphic product version < 1255 */ +static const struct __guc_mmio_reg_descr_group xe_lp_lists[] = { + MAKE_REGLIST(xe_lp_global_regs, PF, GLOBAL, 0), + MAKE_REGLIST(xe_rc_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(xe_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_vec_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(xe_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(xe_lp_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + {} +}; + + /* List of lists for graphic product version >= 1255 */ +static const struct __guc_mmio_reg_descr_group xe_hpg_lists[] = { + MAKE_REGLIST(xe_lp_global_regs, PF, GLOBAL, 0), + MAKE_REGLIST(xe_hpg_rc_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(xe_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_vec_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(xe_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(xe_lp_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_PAGING), + MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_PAGING), + {} +}; + + /* List of lists for Xe3p and beyond */ +static const struct __guc_mmio_reg_descr_group xe3p_lists[] = { + MAKE_REGLIST(xe_lp_global_regs, PF, GLOBAL, 0), + MAKE_REGLIST(xe_hpg_rc_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(xe3p_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_vec_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(xe_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(xe_lp_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_PAGING), + MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_PAGING), + {} +}; +static const char * const capture_list_type_names[] = { + "Global", + "Class", + "Instance", +}; + +static const char * const capture_engine_class_names[] = { + "Render/Compute", + "Video", + "VideoEnhance", + "Blitter", + "GSC-Other", + "Paging", +}; + +struct __guc_capture_ads_cache { + bool is_valid; + void *ptr; + size_t size; + int status; +}; + +struct xe_guc_state_capture { + const struct __guc_mmio_reg_descr_group *reglists; + /** + * NOTE: steered registers have multiple instances depending on the HW configuration + * (slices or dual-sub-slices) and thus depends on HW fuses discovered + */ + struct __guc_mmio_reg_descr_group *extlists; + struct __guc_capture_ads_cache ads_cache[GUC_CAPTURE_LIST_INDEX_MAX] + [GUC_STATE_CAPTURE_TYPE_MAX] + [GUC_CAPTURE_LIST_CLASS_MAX]; + void *ads_null_cache; + struct list_head cachelist; +#define PREALLOC_NODES_MAX_COUNT (3 * GUC_MAX_ENGINE_CLASSES * GUC_MAX_INSTANCES_PER_CLASS) +#define PREALLOC_NODES_DEFAULT_NUMREGS 64 + + int max_mmio_per_node; + struct list_head outlist; +}; + +static void +guc_capture_remove_stale_matches_from_list(struct xe_guc_state_capture *gc, + struct __guc_capture_parsed_output *node); + +static const struct __guc_mmio_reg_descr_group * +guc_capture_get_device_reglist(struct xe_device *xe) +{ + if (GRAPHICS_VER(xe) >= 35) + return xe3p_lists; + else if (GRAPHICS_VERx100(xe) >= 1255) + return xe_hpg_lists; + else + return xe_lp_lists; +} + +static const struct __guc_mmio_reg_descr_group * +guc_capture_get_one_list(const struct __guc_mmio_reg_descr_group *reglists, + u32 owner, u32 type, enum guc_capture_list_class_type capture_class) +{ + int i; + + if (!reglists) + return NULL; + + for (i = 0; reglists[i].list; ++i) { + if (reglists[i].owner == owner && reglists[i].type == type && + (reglists[i].engine == capture_class || + reglists[i].type == GUC_STATE_CAPTURE_TYPE_GLOBAL)) + return ®lists[i]; + } + + return NULL; +} + +const struct __guc_mmio_reg_descr_group * +xe_guc_capture_get_reg_desc_list(struct xe_gt *gt, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, bool is_ext) +{ + const struct __guc_mmio_reg_descr_group *reglists; + + if (is_ext) { + struct xe_guc *guc = >->uc.guc; + + reglists = guc->capture->extlists; + } else { + reglists = guc_capture_get_device_reglist(gt_to_xe(gt)); + } + return guc_capture_get_one_list(reglists, owner, type, capture_class); +} + +struct __ext_steer_reg { + const char *name; + struct xe_reg_mcr reg; +}; + +static const struct __ext_steer_reg xe_extregs[] = { + {"SAMPLER_INSTDONE", SAMPLER_INSTDONE}, + {"ROW_INSTDONE", ROW_INSTDONE} +}; + +static const struct __ext_steer_reg xehpg_extregs[] = { + {"SC_INSTDONE", XEHPG_SC_INSTDONE}, + {"SC_INSTDONE_EXTRA", XEHPG_SC_INSTDONE_EXTRA}, + {"SC_INSTDONE_EXTRA2", XEHPG_SC_INSTDONE_EXTRA2}, + {"INSTDONE_GEOM_SVGUNIT", XEHPG_INSTDONE_GEOM_SVGUNIT} +}; + +static void __fill_ext_reg(struct __guc_mmio_reg_descr *ext, + const struct __ext_steer_reg *extlist, + u32 dss_id, u16 slice_id, u16 subslice_id) +{ + if (!ext || !extlist) + return; + + ext->reg = XE_REG(extlist->reg.__reg.addr); + ext->flags = FIELD_PREP(GUC_REGSET_STEERING_NEEDED, 1); + ext->flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, slice_id); + ext->flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, subslice_id); + ext->dss_id = dss_id; + ext->regname = extlist->name; +} + +static int +__alloc_ext_regs(struct drm_device *drm, struct __guc_mmio_reg_descr_group *newlist, + const struct __guc_mmio_reg_descr_group *rootlist, int num_regs) +{ + struct __guc_mmio_reg_descr *list; + + list = drmm_kzalloc(drm, num_regs * sizeof(struct __guc_mmio_reg_descr), GFP_KERNEL); + if (!list) + return -ENOMEM; + + newlist->list = list; + newlist->num_regs = num_regs; + newlist->owner = rootlist->owner; + newlist->engine = rootlist->engine; + newlist->type = rootlist->type; + + return 0; +} + +static int guc_capture_get_steer_reg_num(struct xe_device *xe) +{ + int num = ARRAY_SIZE(xe_extregs); + + if (GRAPHICS_VERx100(xe) >= 1255) + num += ARRAY_SIZE(xehpg_extregs); + + return num; +} + +static void guc_capture_alloc_steered_lists(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u16 slice, subslice; + int dss, i, total = 0; + const struct __guc_mmio_reg_descr_group *lists = guc->capture->reglists; + const struct __guc_mmio_reg_descr_group *list; + struct __guc_mmio_reg_descr_group *extlists; + struct __guc_mmio_reg_descr *extarray; + bool has_xehpg_extregs = GRAPHICS_VERx100(gt_to_xe(gt)) >= 1255; + struct drm_device *drm = >_to_xe(gt)->drm; + bool has_rcs_ccs = false; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + /* + * If GT has no rcs/ccs, no need to alloc steered list. + * Currently, only rcs/ccs has steering register, if in the future, + * other engine types has steering register, this condition check need + * to be extended + */ + for_each_hw_engine(hwe, gt, id) { + if (xe_hwe_to_guc_capture_class(hwe) == + GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE) { + has_rcs_ccs = true; + break; + } + } + + if (!has_rcs_ccs) + return; + + /* steered registers currently only exist for the render-class */ + list = guc_capture_get_one_list(lists, GUC_CAPTURE_LIST_INDEX_PF, + GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE); + /* + * Skip if this platform has no engine class registers or if extlists + * was previously allocated + */ + if (!list || guc->capture->extlists) + return; + + { + xe_dss_mask_t all_dss; + + total = bitmap_weighted_or(all_dss, gt->fuse_topo.g_dss_mask, + gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS) * + guc_capture_get_steer_reg_num(guc_to_xe(guc)); + } + + if (!total) + return; + + /* allocate an extra for an end marker */ + extlists = drmm_kzalloc(drm, 2 * sizeof(struct __guc_mmio_reg_descr_group), GFP_KERNEL); + if (!extlists) + return; + + if (__alloc_ext_regs(drm, &extlists[0], list, total)) { + drmm_kfree(drm, extlists); + return; + } + + /* For steering registers, the list is generated at run-time */ + extarray = (struct __guc_mmio_reg_descr *)extlists[0].list; + for_each_dss_steering(dss, gt, slice, subslice) { + for (i = 0; i < ARRAY_SIZE(xe_extregs); ++i) { + __fill_ext_reg(extarray, &xe_extregs[i], dss, slice, subslice); + ++extarray; + } + + if (has_xehpg_extregs) + for (i = 0; i < ARRAY_SIZE(xehpg_extregs); ++i) { + __fill_ext_reg(extarray, &xehpg_extregs[i], dss, slice, subslice); + ++extarray; + } + } + + extlists[0].num_regs = total; + + xe_gt_dbg(guc_to_gt(guc), "capture found %d ext-regs.\n", total); + guc->capture->extlists = extlists; +} + +static int +guc_capture_list_init(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, struct guc_mmio_reg *ptr, + u16 num_entries) +{ + u32 ptr_idx = 0, list_idx = 0; + const struct __guc_mmio_reg_descr_group *reglists = guc->capture->reglists; + struct __guc_mmio_reg_descr_group *extlists = guc->capture->extlists; + const struct __guc_mmio_reg_descr_group *match; + u32 list_num; + + if (!reglists) + return -ENODEV; + + match = guc_capture_get_one_list(reglists, owner, type, capture_class); + if (!match) + return -ENODATA; + + list_num = match->num_regs; + for (list_idx = 0; ptr_idx < num_entries && list_idx < list_num; ++list_idx, ++ptr_idx) { + ptr[ptr_idx].offset = match->list[list_idx].reg.addr; + ptr[ptr_idx].value = 0xDEADF00D; + ptr[ptr_idx].flags = match->list[list_idx].flags; + ptr[ptr_idx].mask = match->list[list_idx].mask; + } + + match = guc_capture_get_one_list(extlists, owner, type, capture_class); + if (match) + for (ptr_idx = list_num, list_idx = 0; + ptr_idx < num_entries && list_idx < match->num_regs; + ++ptr_idx, ++list_idx) { + ptr[ptr_idx].offset = match->list[list_idx].reg.addr; + ptr[ptr_idx].value = 0xDEADF00D; + ptr[ptr_idx].flags = match->list[list_idx].flags; + ptr[ptr_idx].mask = match->list[list_idx].mask; + } + + if (ptr_idx < num_entries) + xe_gt_dbg(guc_to_gt(guc), "Got short capture reglist init: %d out-of %d.\n", + ptr_idx, num_entries); + + return 0; +} + +static int +guc_cap_list_num_regs(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class) +{ + const struct __guc_mmio_reg_descr_group *match; + int num_regs = 0; + + match = guc_capture_get_one_list(guc->capture->reglists, owner, type, capture_class); + if (match) + num_regs = match->num_regs; + + match = guc_capture_get_one_list(guc->capture->extlists, owner, type, capture_class); + if (match) + num_regs += match->num_regs; + else + /* + * If a caller wants the full register dump size but we have + * not yet got the hw-config, which is before max_mmio_per_node + * is initialized, then provide a worst-case number for + * extlists based on max dss fuse bits, but only ever for + * render/compute + */ + if (owner == GUC_CAPTURE_LIST_INDEX_PF && + type == GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS && + capture_class == GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE && + !guc->capture->max_mmio_per_node) + num_regs += guc_capture_get_steer_reg_num(guc_to_xe(guc)) * + XE_MAX_DSS_FUSE_BITS; + + return num_regs; +} + +static int +guc_capture_getlistsize(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, + size_t *size, bool is_purpose_est) +{ + struct xe_guc_state_capture *gc = guc->capture; + struct xe_gt *gt = guc_to_gt(guc); + struct __guc_capture_ads_cache *cache; + int num_regs; + + xe_gt_assert(gt, type < GUC_STATE_CAPTURE_TYPE_MAX); + xe_gt_assert(gt, capture_class < GUC_CAPTURE_LIST_CLASS_MAX); + + cache = &gc->ads_cache[owner][type][capture_class]; + if (!gc->reglists) { + xe_gt_warn(gt, "No capture reglist for this device\n"); + return -ENODEV; + } + + if (cache->is_valid) { + *size = cache->size; + return cache->status; + } + + if (!is_purpose_est && owner == GUC_CAPTURE_LIST_INDEX_PF && + !guc_capture_get_one_list(gc->reglists, owner, type, capture_class)) { + if (type == GUC_STATE_CAPTURE_TYPE_GLOBAL) + xe_gt_warn(gt, "Missing capture reglist: global!\n"); + else + xe_gt_warn(gt, "Missing capture reglist: %s(%u):%s(%u)!\n", + capture_list_type_names[type], type, + capture_engine_class_names[capture_class], capture_class); + return -ENODEV; + } + + num_regs = guc_cap_list_num_regs(guc, owner, type, capture_class); + /* intentional empty lists can exist depending on hw config */ + if (!num_regs) + return -ENODATA; + + if (size) + *size = PAGE_ALIGN((sizeof(struct guc_debug_capture_list)) + + (num_regs * sizeof(struct guc_mmio_reg))); + + return 0; +} + +/** + * xe_guc_capture_getlistsize - Get list size for owner/type/class combination + * @guc: The GuC object + * @owner: PF/VF owner + * @type: GuC capture register type + * @capture_class: GuC capture engine class id + * @size: Point to the size + * + * This function will get the list for the owner/type/class combination, and + * return the page aligned list size. + * + * Returns: 0 on success or a negative error code on failure. + */ +int +xe_guc_capture_getlistsize(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, size_t *size) +{ + return guc_capture_getlistsize(guc, owner, type, capture_class, size, false); +} + +/** + * xe_guc_capture_getlist - Get register capture list for owner/type/class + * combination + * @guc: The GuC object + * @owner: PF/VF owner + * @type: GuC capture register type + * @capture_class: GuC capture engine class id + * @outptr: Point to cached register capture list + * + * This function will get the register capture list for the owner/type/class + * combination. + * + * Returns: 0 on success or a negative error code on failure. + */ +int +xe_guc_capture_getlist(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, void **outptr) +{ + struct xe_guc_state_capture *gc = guc->capture; + struct __guc_capture_ads_cache *cache = &gc->ads_cache[owner][type][capture_class]; + struct guc_debug_capture_list *listnode; + int ret, num_regs; + u8 *caplist, *tmp; + size_t size = 0; + + if (!gc->reglists) + return -ENODEV; + + if (cache->is_valid) { + *outptr = cache->ptr; + return cache->status; + } + + ret = xe_guc_capture_getlistsize(guc, owner, type, capture_class, &size); + if (ret) { + cache->is_valid = true; + cache->ptr = NULL; + cache->size = 0; + cache->status = ret; + return ret; + } + + caplist = drmm_kzalloc(guc_to_drm(guc), size, GFP_KERNEL); + if (!caplist) + return -ENOMEM; + + /* populate capture list header */ + tmp = caplist; + num_regs = guc_cap_list_num_regs(guc, owner, type, capture_class); + listnode = (struct guc_debug_capture_list *)tmp; + listnode->header.info = FIELD_PREP(GUC_CAPTURELISTHDR_NUMDESCR, (u32)num_regs); + + /* populate list of register descriptor */ + tmp += sizeof(struct guc_debug_capture_list); + guc_capture_list_init(guc, owner, type, capture_class, + (struct guc_mmio_reg *)tmp, num_regs); + + /* cache this list */ + cache->is_valid = true; + cache->ptr = caplist; + cache->size = size; + cache->status = 0; + + *outptr = caplist; + + return 0; +} + +/** + * xe_guc_capture_getnullheader - Get a null list for register capture + * @guc: The GuC object + * @outptr: Point to cached register capture list + * @size: Point to the size + * + * This function will alloc for a null list for register capture. + * + * Returns: 0 on success or a negative error code on failure. + */ +int +xe_guc_capture_getnullheader(struct xe_guc *guc, void **outptr, size_t *size) +{ + struct xe_guc_state_capture *gc = guc->capture; + int tmp = sizeof(u32) * 4; + void *null_header; + + if (gc->ads_null_cache) { + *outptr = gc->ads_null_cache; + *size = tmp; + return 0; + } + + null_header = drmm_kzalloc(guc_to_drm(guc), tmp, GFP_KERNEL); + if (!null_header) + return -ENOMEM; + + gc->ads_null_cache = null_header; + *outptr = null_header; + *size = tmp; + + return 0; +} + +/** + * xe_guc_capture_ads_input_worst_size - Calculate the worst size for GuC register capture + * @guc: point to xe_guc structure + * + * Calculate the worst size for GuC register capture by including all possible engines classes. + * + * Returns: Calculated size + */ +size_t xe_guc_capture_ads_input_worst_size(struct xe_guc *guc) +{ + size_t total_size, class_size, instance_size, global_size; + int i, j; + + /* + * This function calculates the worst case register lists size by + * including all possible engines classes. It is called during the + * first of a two-phase GuC (and ADS-population) initialization + * sequence, that is, during the pre-hwconfig phase before we have + * the exact engine fusing info. + */ + total_size = PAGE_SIZE; /* Pad a page in front for empty lists */ + for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { + for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) { + if (!xe_guc_has_paging_engine(guc) && + j == GUC_CAPTURE_LIST_CLASS_PAGING) + continue; + + if (xe_guc_capture_getlistsize(guc, i, + GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + j, &class_size) < 0) + class_size = 0; + if (xe_guc_capture_getlistsize(guc, i, + GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE, + j, &instance_size) < 0) + instance_size = 0; + total_size += class_size + instance_size; + } + if (xe_guc_capture_getlistsize(guc, i, + GUC_STATE_CAPTURE_TYPE_GLOBAL, + 0, &global_size) < 0) + global_size = 0; + total_size += global_size; + } + + return PAGE_ALIGN(total_size); +} + +static int guc_capture_output_size_est(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + int capture_size = 0; + size_t tmp = 0; + + if (!guc->capture) + return -ENODEV; + + /* + * If every single engine-instance suffered a failure in quick succession but + * were all unrelated, then a burst of multiple error-capture events would dump + * registers for every one engine instance, one at a time. In this case, GuC + * would even dump the global-registers repeatedly. + * + * For each engine instance, there would be 1 x guc_state_capture_group_t output + * followed by 3 x guc_state_capture_t lists. The latter is how the register + * dumps are split across different register types (where the '3' are global vs class + * vs instance). + */ + for_each_hw_engine(hwe, gt, id) { + enum guc_capture_list_class_type capture_class; + + capture_class = xe_hwe_to_guc_capture_class(hwe); + capture_size += sizeof(struct guc_state_capture_group_header_t) + + (3 * sizeof(struct guc_state_capture_header_t)); + + if (!guc_capture_getlistsize(guc, 0, GUC_STATE_CAPTURE_TYPE_GLOBAL, + 0, &tmp, true)) + capture_size += tmp; + if (!guc_capture_getlistsize(guc, 0, GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + capture_class, &tmp, true)) + capture_size += tmp; + if (!guc_capture_getlistsize(guc, 0, GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE, + capture_class, &tmp, true)) + capture_size += tmp; + } + + return capture_size; +} + +/* + * Add on a 3x multiplier to allow for multiple back-to-back captures occurring + * before the Xe can read the data out and process it + */ +#define GUC_CAPTURE_OVERBUFFER_MULTIPLIER 3 + +static void check_guc_capture_size(struct xe_guc *guc) +{ + int capture_size = guc_capture_output_size_est(guc); + int spare_size = capture_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER; + u32 buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE; + + /* + * NOTE: capture_size is much smaller than the capture region + * allocation (DG2: <80K vs 1MB). + * Additionally, its based on space needed to fit all engines getting + * reset at once within the same G2H handler task slot. This is very + * unlikely. However, if GuC really does run out of space for whatever + * reason, we will see an separate warning message when processing the + * G2H event capture-notification, search for: + * xe_guc_STATE_CAPTURE_EVENT_STATUS_NOSPACE. + */ + if (capture_size < 0) + xe_gt_dbg(guc_to_gt(guc), + "Failed to calculate error state capture buffer minimum size: %d!\n", + capture_size); + if (capture_size > buffer_size) + xe_gt_dbg(guc_to_gt(guc), "Error state capture buffer maybe small: %d < %d\n", + buffer_size, capture_size); + else if (spare_size > buffer_size) + xe_gt_dbg(guc_to_gt(guc), + "Error state capture buffer lacks spare size: %d < %d (min = %d)\n", + buffer_size, spare_size, capture_size); +} + +static void +guc_capture_add_node_to_list(struct __guc_capture_parsed_output *node, + struct list_head *list) +{ + list_add(&node->link, list); +} + +static void +guc_capture_add_node_to_outlist(struct xe_guc_state_capture *gc, + struct __guc_capture_parsed_output *node) +{ + guc_capture_remove_stale_matches_from_list(gc, node); + guc_capture_add_node_to_list(node, &gc->outlist); +} + +static void +guc_capture_add_node_to_cachelist(struct xe_guc_state_capture *gc, + struct __guc_capture_parsed_output *node) +{ + guc_capture_add_node_to_list(node, &gc->cachelist); +} + +static void +guc_capture_free_outlist_node(struct xe_guc_state_capture *gc, + struct __guc_capture_parsed_output *n) +{ + if (n) { + n->locked = 0; + list_del(&n->link); + /* put node back to cache list */ + guc_capture_add_node_to_cachelist(gc, n); + } +} + +static void +guc_capture_remove_stale_matches_from_list(struct xe_guc_state_capture *gc, + struct __guc_capture_parsed_output *node) +{ + struct __guc_capture_parsed_output *n, *ntmp; + int guc_id = node->guc_id; + + list_for_each_entry_safe(n, ntmp, &gc->outlist, link) { + if (n != node && !n->locked && n->guc_id == guc_id) + guc_capture_free_outlist_node(gc, n); + } +} + +static void +guc_capture_init_node(struct xe_guc *guc, struct __guc_capture_parsed_output *node) +{ + struct guc_mmio_reg *tmp[GUC_STATE_CAPTURE_TYPE_MAX]; + int i; + + for (i = 0; i < GUC_STATE_CAPTURE_TYPE_MAX; ++i) { + tmp[i] = node->reginfo[i].regs; + memset(tmp[i], 0, sizeof(struct guc_mmio_reg) * + guc->capture->max_mmio_per_node); + } + memset(node, 0, sizeof(*node)); + for (i = 0; i < GUC_STATE_CAPTURE_TYPE_MAX; ++i) + node->reginfo[i].regs = tmp[i]; + + INIT_LIST_HEAD(&node->link); +} + +/** + * DOC: Init, G2H-event and reporting flows for GuC-error-capture + * + * KMD Init time flows: + * -------------------- + * --> alloc A: GuC input capture regs lists (registered to GuC via ADS). + * xe_guc_ads acquires the register lists by calling + * xe_guc_capture_getlistsize and xe_guc_capture_getlist 'n' times, + * where n = 1 for global-reg-list + + * num_engine_classes for class-reg-list + + * num_engine_classes for instance-reg-list + * (since all instances of the same engine-class type + * have an identical engine-instance register-list). + * ADS module also calls separately for PF vs VF. + * + * --> alloc B: GuC output capture buf (registered via guc_init_params(log_param)) + * Size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE (warns if on too-small) + * Note2: 'x 3' to hold multiple capture groups + * + * GUC Runtime notify capture: + * -------------------------- + * --> G2H STATE_CAPTURE_NOTIFICATION + * L--> xe_guc_capture_process + * L--> Loop through B (head..tail) and for each engine instance's + * err-state-captured register-list we find, we alloc 'C': + * --> alloc C: A capture-output-node structure that includes misc capture info along + * with 3 register list dumps (global, engine-class and engine-instance) + * This node is created from a pre-allocated list of blank nodes in + * guc->capture->cachelist and populated with the error-capture + * data from GuC and then it's added into guc->capture->outlist linked + * list. This list is used for matchup and printout by xe_devcoredump_read + * and xe_engine_snapshot_print, (when user invokes the devcoredump sysfs). + * + * GUC --> notify context reset: + * ----------------------------- + * --> guc_exec_queue_timedout_job + * L--> xe_devcoredump + * L--> devcoredump_snapshot + * --> xe_hw_engine_snapshot_capture + * --> xe_engine_manual_capture(For manual capture) + * + * User Sysfs / Debugfs + * -------------------- + * --> xe_devcoredump_read-> + * L--> xxx_snapshot_print + * L--> xe_engine_snapshot_print + * Print register lists values saved at + * guc->capture->outlist + * + */ + +static int guc_capture_buf_cnt(struct __guc_capture_bufstate *buf) +{ + if (buf->wr >= buf->rd) + return (buf->wr - buf->rd); + return (buf->size - buf->rd) + buf->wr; +} + +static int guc_capture_buf_cnt_to_end(struct __guc_capture_bufstate *buf) +{ + if (buf->rd > buf->wr) + return (buf->size - buf->rd); + return (buf->wr - buf->rd); +} + +/* + * GuC's error-capture output is a ring buffer populated in a byte-stream fashion: + * + * The GuC Log buffer region for error-capture is managed like a ring buffer. + * The GuC firmware dumps error capture logs into this ring in a byte-stream flow. + * Additionally, as per the current and foreseeable future, all packed error- + * capture output structures are dword aligned. + * + * That said, if the GuC firmware is in the midst of writing a structure that is larger + * than one dword but the tail end of the err-capture buffer-region has lesser space left, + * we would need to extract that structure one dword at a time straddled across the end, + * onto the start of the ring. + * + * Below function, guc_capture_log_remove_bytes is a helper for that. All callers of this + * function would typically do a straight-up memcpy from the ring contents and will only + * call this helper if their structure-extraction is straddling across the end of the + * ring. GuC firmware does not add any padding. The reason for the no-padding is to ease + * scalability for future expansion of output data types without requiring a redesign + * of the flow controls. + */ +static int +guc_capture_log_remove_bytes(struct xe_guc *guc, struct __guc_capture_bufstate *buf, + void *out, int bytes_needed) +{ +#define GUC_CAPTURE_LOG_BUF_COPY_RETRY_MAX 3 + + int fill_size = 0, tries = GUC_CAPTURE_LOG_BUF_COPY_RETRY_MAX; + int copy_size, avail; + + xe_assert(guc_to_xe(guc), bytes_needed % sizeof(u32) == 0); + + if (bytes_needed > guc_capture_buf_cnt(buf)) + return -1; + + while (bytes_needed > 0 && tries--) { + int misaligned; + + avail = guc_capture_buf_cnt_to_end(buf); + misaligned = avail % sizeof(u32); + /* wrap if at end */ + if (!avail) { + /* output stream clipped */ + if (!buf->rd) + return fill_size; + buf->rd = 0; + continue; + } + + /* Only copy to u32 aligned data */ + copy_size = avail < bytes_needed ? avail - misaligned : bytes_needed; + xe_map_memcpy_from(guc_to_xe(guc), out + fill_size, &guc->log.bo->vmap, + buf->data_offset + buf->rd, copy_size); + buf->rd += copy_size; + fill_size += copy_size; + bytes_needed -= copy_size; + + if (misaligned) + xe_gt_warn(guc_to_gt(guc), + "Bytes extraction not dword aligned, clipping.\n"); + } + + return fill_size; +} + +static int +guc_capture_log_get_group_hdr(struct xe_guc *guc, struct __guc_capture_bufstate *buf, + struct guc_state_capture_group_header_t *ghdr) +{ + int fullsize = sizeof(struct guc_state_capture_group_header_t); + + if (guc_capture_log_remove_bytes(guc, buf, ghdr, fullsize) != fullsize) + return -1; + return 0; +} + +static int +guc_capture_log_get_data_hdr(struct xe_guc *guc, struct __guc_capture_bufstate *buf, + struct guc_state_capture_header_t *hdr) +{ + int fullsize = sizeof(struct guc_state_capture_header_t); + + if (guc_capture_log_remove_bytes(guc, buf, hdr, fullsize) != fullsize) + return -1; + return 0; +} + +static int +guc_capture_log_get_register(struct xe_guc *guc, struct __guc_capture_bufstate *buf, + struct guc_mmio_reg *reg) +{ + int fullsize = sizeof(struct guc_mmio_reg); + + if (guc_capture_log_remove_bytes(guc, buf, reg, fullsize) != fullsize) + return -1; + return 0; +} + +static struct __guc_capture_parsed_output * +guc_capture_get_prealloc_node(struct xe_guc *guc) +{ + struct __guc_capture_parsed_output *found = NULL; + + if (!list_empty(&guc->capture->cachelist)) { + struct __guc_capture_parsed_output *n, *ntmp; + + /* get first avail node from the cache list */ + list_for_each_entry_safe(n, ntmp, &guc->capture->cachelist, link) { + found = n; + break; + } + } else { + struct __guc_capture_parsed_output *n, *ntmp; + + /* + * traverse reversed and steal back the oldest node already + * allocated + */ + list_for_each_entry_safe_reverse(n, ntmp, &guc->capture->outlist, link) { + if (!n->locked) + found = n; + } + } + if (found) { + list_del(&found->link); + guc_capture_init_node(guc, found); + } + + return found; +} + +static struct __guc_capture_parsed_output * +guc_capture_clone_node(struct xe_guc *guc, struct __guc_capture_parsed_output *original, + u32 keep_reglist_mask) +{ + struct __guc_capture_parsed_output *new; + int i; + + new = guc_capture_get_prealloc_node(guc); + if (!new) + return NULL; + if (!original) + return new; + + new->is_partial = original->is_partial; + + /* copy reg-lists that we want to clone */ + for (i = 0; i < GUC_STATE_CAPTURE_TYPE_MAX; ++i) { + if (keep_reglist_mask & BIT(i)) { + XE_WARN_ON(original->reginfo[i].num_regs > + guc->capture->max_mmio_per_node); + + memcpy(new->reginfo[i].regs, original->reginfo[i].regs, + original->reginfo[i].num_regs * sizeof(struct guc_mmio_reg)); + + new->reginfo[i].num_regs = original->reginfo[i].num_regs; + new->reginfo[i].vfid = original->reginfo[i].vfid; + + if (i == GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS) { + new->eng_class = original->eng_class; + } else if (i == GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE) { + new->eng_inst = original->eng_inst; + new->guc_id = original->guc_id; + new->lrca = original->lrca; + } + } + } + + return new; +} + +static int +guc_capture_extract_reglists(struct xe_guc *guc, struct __guc_capture_bufstate *buf) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct guc_state_capture_group_header_t ghdr = {0}; + struct guc_state_capture_header_t hdr = {0}; + struct __guc_capture_parsed_output *node = NULL; + struct guc_mmio_reg *regs = NULL; + int i, numlists, numregs, ret = 0; + enum guc_state_capture_type datatype; + struct guc_mmio_reg tmp; + bool is_partial = false; + + i = guc_capture_buf_cnt(buf); + if (!i) + return -ENODATA; + + if (i % sizeof(u32)) { + xe_gt_warn(gt, "Got mis-aligned register capture entries\n"); + ret = -EIO; + goto bailout; + } + + /* first get the capture group header */ + if (guc_capture_log_get_group_hdr(guc, buf, &ghdr)) { + ret = -EIO; + goto bailout; + } + /* + * we would typically expect a layout as below where n would be expected to be + * anywhere between 3 to n where n > 3 if we are seeing multiple dependent engine + * instances being reset together. + * ____________________________________________ + * | Capture Group | + * | ________________________________________ | + * | | Capture Group Header: | | + * | | - num_captures = 5 | | + * | |______________________________________| | + * | ________________________________________ | + * | | Capture1: | | + * | | Hdr: GLOBAL, numregs=a | | + * | | ____________________________________ | | + * | | | Reglist | | | + * | | | - reg1, reg2, ... rega | | | + * | | |__________________________________| | | + * | |______________________________________| | + * | ________________________________________ | + * | | Capture2: | | + * | | Hdr: CLASS=RENDER/COMPUTE, numregs=b| | + * | | ____________________________________ | | + * | | | Reglist | | | + * | | | - reg1, reg2, ... regb | | | + * | | |__________________________________| | | + * | |______________________________________| | + * | ________________________________________ | + * | | Capture3: | | + * | | Hdr: INSTANCE=RCS, numregs=c | | + * | | ____________________________________ | | + * | | | Reglist | | | + * | | | - reg1, reg2, ... regc | | | + * | | |__________________________________| | | + * | |______________________________________| | + * | ________________________________________ | + * | | Capture4: | | + * | | Hdr: CLASS=RENDER/COMPUTE, numregs=d| | + * | | ____________________________________ | | + * | | | Reglist | | | + * | | | - reg1, reg2, ... regd | | | + * | | |__________________________________| | | + * | |______________________________________| | + * | ________________________________________ | + * | | Capture5: | | + * | | Hdr: INSTANCE=CCS0, numregs=e | | + * | | ____________________________________ | | + * | | | Reglist | | | + * | | | - reg1, reg2, ... rege | | | + * | | |__________________________________| | | + * | |______________________________________| | + * |__________________________________________| + */ + is_partial = FIELD_GET(GUC_STATE_CAPTURE_GROUP_HEADER_CAPTURE_GROUP_TYPE, ghdr.info); + numlists = FIELD_GET(GUC_STATE_CAPTURE_GROUP_HEADER_NUM_CAPTURES, ghdr.info); + + while (numlists--) { + if (guc_capture_log_get_data_hdr(guc, buf, &hdr)) { + ret = -EIO; + break; + } + + datatype = FIELD_GET(GUC_STATE_CAPTURE_HEADER_CAPTURE_TYPE, hdr.info); + if (datatype > GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE) { + /* unknown capture type - skip over to next capture set */ + numregs = FIELD_GET(GUC_STATE_CAPTURE_HEADER_NUM_MMIO_ENTRIES, + hdr.num_mmio_entries); + while (numregs--) { + if (guc_capture_log_get_register(guc, buf, &tmp)) { + ret = -EIO; + break; + } + } + continue; + } else if (node) { + /* + * Based on the current capture type and what we have so far, + * decide if we should add the current node into the internal + * linked list for match-up when xe_devcoredump calls later + * (and alloc a blank node for the next set of reglists) + * or continue with the same node or clone the current node + * but only retain the global or class registers (such as the + * case of dependent engine resets). + */ + if (datatype == GUC_STATE_CAPTURE_TYPE_GLOBAL) { + guc_capture_add_node_to_outlist(guc->capture, node); + node = NULL; + } else if (datatype == GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS && + node->reginfo[GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS].num_regs) { + /* Add to list, clone node and duplicate global list */ + guc_capture_add_node_to_outlist(guc->capture, node); + node = guc_capture_clone_node(guc, node, + GCAP_PARSED_REGLIST_INDEX_GLOBAL); + } else if (datatype == GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE && + node->reginfo[GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE].num_regs) { + /* Add to list, clone node and duplicate global + class lists */ + guc_capture_add_node_to_outlist(guc->capture, node); + node = guc_capture_clone_node(guc, node, + (GCAP_PARSED_REGLIST_INDEX_GLOBAL | + GCAP_PARSED_REGLIST_INDEX_ENGCLASS)); + } + } + + if (!node) { + node = guc_capture_get_prealloc_node(guc); + if (!node) { + ret = -ENOMEM; + break; + } + if (datatype != GUC_STATE_CAPTURE_TYPE_GLOBAL) + xe_gt_dbg(gt, "Register capture missing global dump: %08x!\n", + datatype); + } + node->is_partial = is_partial; + node->reginfo[datatype].vfid = FIELD_GET(GUC_STATE_CAPTURE_HEADER_VFID, hdr.owner); + node->source = XE_ENGINE_CAPTURE_SOURCE_GUC; + node->type = datatype; + + switch (datatype) { + case GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE: + node->eng_class = FIELD_GET(GUC_STATE_CAPTURE_HEADER_ENGINE_CLASS, + hdr.info); + node->eng_inst = FIELD_GET(GUC_STATE_CAPTURE_HEADER_ENGINE_INSTANCE, + hdr.info); + node->lrca = hdr.lrca; + node->guc_id = hdr.guc_id; + break; + case GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS: + node->eng_class = FIELD_GET(GUC_STATE_CAPTURE_HEADER_ENGINE_CLASS, + hdr.info); + break; + default: + break; + } + + numregs = FIELD_GET(GUC_STATE_CAPTURE_HEADER_NUM_MMIO_ENTRIES, + hdr.num_mmio_entries); + if (numregs > guc->capture->max_mmio_per_node) { + xe_gt_dbg(gt, "Register capture list extraction clipped by prealloc!\n"); + numregs = guc->capture->max_mmio_per_node; + } + node->reginfo[datatype].num_regs = numregs; + regs = node->reginfo[datatype].regs; + i = 0; + while (numregs--) { + if (guc_capture_log_get_register(guc, buf, ®s[i++])) { + ret = -EIO; + break; + } + } + } + +bailout: + if (node) { + /* If we have data, add to linked list for match-up when xe_devcoredump calls */ + for (i = GUC_STATE_CAPTURE_TYPE_GLOBAL; i < GUC_STATE_CAPTURE_TYPE_MAX; ++i) { + if (node->reginfo[i].regs) { + guc_capture_add_node_to_outlist(guc->capture, node); + node = NULL; + break; + } + } + if (node) /* else return it back to cache list */ + guc_capture_add_node_to_cachelist(guc->capture, node); + } + return ret; +} + +static int __guc_capture_flushlog_complete(struct xe_guc *guc) +{ + u32 action[] = { + XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE, + GUC_LOG_TYPE_STATE_CAPTURE + }; + + return xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); +} + +static void __guc_capture_process_output(struct xe_guc *guc) +{ + unsigned int buffer_size, read_offset, write_offset, full_count; + struct xe_uc *uc = container_of(guc, typeof(*uc), guc); + struct guc_log_buffer_state log_buf_state_local; + struct __guc_capture_bufstate buf; + bool new_overflow; + int ret, tmp; + u32 log_buf_state_offset; + u32 src_data_offset; + + log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_TYPE_STATE_CAPTURE; + src_data_offset = XE_GUC_LOG_STATE_CAPTURE_OFFSET; + + /* + * Make a copy of the state structure, inside GuC log buffer + * (which is uncached mapped), on the stack to avoid reading + * from it multiple times. + */ + xe_map_memcpy_from(guc_to_xe(guc), &log_buf_state_local, &guc->log.bo->vmap, + log_buf_state_offset, sizeof(struct guc_log_buffer_state)); + + buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE; + read_offset = log_buf_state_local.read_ptr; + write_offset = log_buf_state_local.sampled_write_ptr; + full_count = FIELD_GET(GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT, log_buf_state_local.flags); + + /* Bookkeeping stuff */ + tmp = FIELD_GET(GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE, log_buf_state_local.flags); + guc->log.stats[GUC_LOG_TYPE_STATE_CAPTURE].flush += tmp; + new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_TYPE_STATE_CAPTURE, + full_count); + + /* Now copy the actual logs. */ + if (unlikely(new_overflow)) { + /* copy the whole buffer in case of overflow */ + read_offset = 0; + write_offset = buffer_size; + } else if (unlikely((read_offset > buffer_size) || + (write_offset > buffer_size))) { + xe_gt_err(guc_to_gt(guc), + "Register capture buffer in invalid state: read = 0x%X, size = 0x%X!\n", + read_offset, buffer_size); + /* copy whole buffer as offsets are unreliable */ + read_offset = 0; + write_offset = buffer_size; + } + + buf.size = buffer_size; + buf.rd = read_offset; + buf.wr = write_offset; + buf.data_offset = src_data_offset; + + if (!xe_guc_read_stopped(guc)) { + do { + ret = guc_capture_extract_reglists(guc, &buf); + if (ret && ret != -ENODATA) + xe_gt_dbg(guc_to_gt(guc), "Capture extraction failed:%d\n", ret); + } while (ret >= 0); + } + + /* Update the state of log buffer err-cap state */ + xe_map_wr(guc_to_xe(guc), &guc->log.bo->vmap, + log_buf_state_offset + offsetof(struct guc_log_buffer_state, read_ptr), u32, + write_offset); + + /* + * Clear the flush_to_file from local first, the local was loaded by above + * xe_map_memcpy_from, then write out the "updated local" through + * xe_map_wr() + */ + log_buf_state_local.flags &= ~GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE; + xe_map_wr(guc_to_xe(guc), &guc->log.bo->vmap, + log_buf_state_offset + offsetof(struct guc_log_buffer_state, flags), u32, + log_buf_state_local.flags); + __guc_capture_flushlog_complete(guc); +} + +/* + * xe_guc_capture_process - Process GuC register captured data + * @guc: The GuC object + * + * When GuC captured data is ready, GuC will send message + * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be + * called to process the data comes with the message. + * + * Returns: None + */ +void xe_guc_capture_process(struct xe_guc *guc) +{ + if (guc->capture) + __guc_capture_process_output(guc); +} + +static struct __guc_capture_parsed_output * +guc_capture_alloc_one_node(struct xe_guc *guc) +{ + struct drm_device *drm = guc_to_drm(guc); + struct __guc_capture_parsed_output *new; + int i; + + new = drmm_kzalloc(drm, sizeof(*new), GFP_KERNEL); + if (!new) + return NULL; + + for (i = 0; i < GUC_STATE_CAPTURE_TYPE_MAX; ++i) { + new->reginfo[i].regs = drmm_kzalloc(drm, guc->capture->max_mmio_per_node * + sizeof(struct guc_mmio_reg), GFP_KERNEL); + if (!new->reginfo[i].regs) { + while (i) + drmm_kfree(drm, new->reginfo[--i].regs); + drmm_kfree(drm, new); + return NULL; + } + } + guc_capture_init_node(guc, new); + + return new; +} + +static void +__guc_capture_create_prealloc_nodes(struct xe_guc *guc) +{ + struct __guc_capture_parsed_output *node = NULL; + int i; + + for (i = 0; i < PREALLOC_NODES_MAX_COUNT; ++i) { + node = guc_capture_alloc_one_node(guc); + if (!node) { + xe_gt_warn(guc_to_gt(guc), "Register capture pre-alloc-cache failure\n"); + /* dont free the priors, use what we got and cleanup at shutdown */ + return; + } + guc_capture_add_node_to_cachelist(guc->capture, node); + } +} + +static int +guc_get_max_reglist_count(struct xe_guc *guc) +{ + int i, j, k, tmp, maxregcount = 0; + + for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; ++i) { + for (j = 0; j < GUC_STATE_CAPTURE_TYPE_MAX; ++j) { + for (k = 0; k < GUC_CAPTURE_LIST_CLASS_MAX; ++k) { + const struct __guc_mmio_reg_descr_group *match; + + if (j == GUC_STATE_CAPTURE_TYPE_GLOBAL && k > 0) + continue; + + tmp = 0; + match = guc_capture_get_one_list(guc->capture->reglists, i, j, k); + if (match) + tmp = match->num_regs; + + match = guc_capture_get_one_list(guc->capture->extlists, i, j, k); + if (match) + tmp += match->num_regs; + + if (tmp > maxregcount) + maxregcount = tmp; + } + } + } + if (!maxregcount) + maxregcount = PREALLOC_NODES_DEFAULT_NUMREGS; + + return maxregcount; +} + +static void +guc_capture_create_prealloc_nodes(struct xe_guc *guc) +{ + /* skip if we've already done the pre-alloc */ + if (guc->capture->max_mmio_per_node) + return; + + guc->capture->max_mmio_per_node = guc_get_max_reglist_count(guc); + __guc_capture_create_prealloc_nodes(guc); +} + +static void +read_reg_to_node(struct xe_hw_engine *hwe, const struct __guc_mmio_reg_descr_group *list, + struct guc_mmio_reg *regs) +{ + int i; + + if (!list || !list->list || list->num_regs == 0) + return; + + if (!regs) + return; + + for (i = 0; i < list->num_regs; i++) { + struct __guc_mmio_reg_descr desc = list->list[i]; + u32 value; + + if (list->type == GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE) { + value = xe_hw_engine_mmio_read32(hwe, desc.reg); + } else { + if (list->type == GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS && + FIELD_GET(GUC_REGSET_STEERING_NEEDED, desc.flags)) { + int group, instance; + + group = FIELD_GET(GUC_REGSET_STEERING_GROUP, desc.flags); + instance = FIELD_GET(GUC_REGSET_STEERING_INSTANCE, desc.flags); + value = xe_gt_mcr_unicast_read(hwe->gt, XE_REG_MCR(desc.reg.addr), + group, instance); + } else { + value = xe_mmio_read32(&hwe->gt->mmio, desc.reg); + } + } + + regs[i].value = value; + regs[i].offset = desc.reg.addr; + regs[i].flags = desc.flags; + regs[i].mask = desc.mask; + } +} + +/** + * xe_engine_manual_capture - Take a manual engine snapshot from engine. + * @hwe: Xe HW Engine. + * @snapshot: The engine snapshot + * + * Take engine snapshot from engine read. + * + * Returns: None + */ +void +xe_engine_manual_capture(struct xe_hw_engine *hwe, struct xe_hw_engine_snapshot *snapshot) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_guc *guc = >->uc.guc; + struct xe_devcoredump *devcoredump = &xe->devcoredump; + enum guc_capture_list_class_type capture_class; + const struct __guc_mmio_reg_descr_group *list; + struct __guc_capture_parsed_output *new; + enum guc_state_capture_type type; + u16 guc_id = 0; + u32 lrca = 0; + + if (IS_SRIOV_VF(xe)) + return; + + new = guc_capture_get_prealloc_node(guc); + if (!new) + return; + + capture_class = xe_hwe_to_guc_capture_class(hwe); + for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { + struct gcap_reg_list_info *reginfo = &new->reginfo[type]; + /* + * regsinfo->regs is allocated based on guc->capture->max_mmio_per_node + * which is based on the descriptor list driving the population so + * should not overflow + */ + + /* Get register list for the type/class */ + list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, + capture_class, false); + if (!list) { + xe_gt_dbg(gt, "Empty GuC capture register descriptor for %s", + hwe->name); + continue; + } + + read_reg_to_node(hwe, list, reginfo->regs); + reginfo->num_regs = list->num_regs; + + /* Capture steering registers for rcs/ccs */ + if (capture_class == GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE) { + list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, + type, capture_class, true); + if (list) { + read_reg_to_node(hwe, list, ®info->regs[reginfo->num_regs]); + reginfo->num_regs += list->num_regs; + } + } + } + + if (devcoredump && devcoredump->captured) { + struct xe_guc_submit_exec_queue_snapshot *ge = devcoredump->snapshot.ge; + + if (ge) { + guc_id = ge->guc.id; + if (ge->lrc[0]) + lrca = ge->lrc[0]->context_desc; + } + } + + new->eng_class = xe_hwe_to_guc_class(hwe); + new->eng_inst = hwe->instance; + new->guc_id = guc_id; + new->lrca = lrca; + new->is_partial = 0; + new->locked = 1; + new->source = XE_ENGINE_CAPTURE_SOURCE_MANUAL; + + guc_capture_add_node_to_outlist(guc->capture, new); + devcoredump->snapshot.matched_node = new; +} + +static struct guc_mmio_reg * +guc_capture_find_reg(struct gcap_reg_list_info *reginfo, u32 addr, u32 flags) +{ + int i; + + if (reginfo && reginfo->num_regs > 0) { + struct guc_mmio_reg *regs = reginfo->regs; + + if (regs) + for (i = 0; i < reginfo->num_regs; i++) + if (regs[i].offset == addr && regs[i].flags == flags) + return ®s[i]; + } + + return NULL; +} + +static void +snapshot_print_by_list_order(struct xe_hw_engine_snapshot *snapshot, struct drm_printer *p, + u32 type, const struct __guc_mmio_reg_descr_group *list) +{ + struct xe_gt *gt = snapshot->hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_devcoredump *devcoredump = &xe->devcoredump; + struct xe_devcoredump_snapshot *devcore_snapshot = &devcoredump->snapshot; + struct gcap_reg_list_info *reginfo = NULL; + u32 i, last_value = 0; + bool low32_ready = false; + + if (!list || !list->list || list->num_regs == 0) + return; + XE_WARN_ON(!devcore_snapshot->matched_node); + + reginfo = &devcore_snapshot->matched_node->reginfo[type]; + + /* + * loop through descriptor first and find the register in the node + * this is more scalable for developer maintenance as it will ensure + * the printout matched the ordering of the static descriptor + * table-of-lists + */ + for (i = 0; i < list->num_regs; i++) { + const struct __guc_mmio_reg_descr *reg_desc = &list->list[i]; + struct guc_mmio_reg *reg; + u32 value; + + reg = guc_capture_find_reg(reginfo, reg_desc->reg.addr, reg_desc->flags); + if (!reg) + continue; + + value = reg->value; + switch (reg_desc->data_type) { + case REG_64BIT_LOW_DW: + last_value = value; + + /* + * A 64 bit register define requires 2 consecutive + * entries in register list, with low dword first + * and hi dword the second, like: + * { XXX_REG_LO(0), REG_64BIT_LOW_DW, 0, 0, NULL}, + * { XXX_REG_HI(0), REG_64BIT_HI_DW, 0, 0, "XXX_REG"}, + * + * Incorrect order will trigger XE_WARN. + * + * Possible double low here, for example: + * { XXX_REG_LO(0), REG_64BIT_LOW_DW, 0, 0, NULL}, + * { XXX_REG_LO(0), REG_64BIT_LOW_DW, 0, 0, NULL}, + */ + XE_WARN_ON(low32_ready); + low32_ready = true; + /* Low 32 bit dword saved, continue for high 32 bit */ + break; + + case REG_64BIT_HI_DW: { + u64 value_qw = ((u64)value << 32) | last_value; + + /* + * Incorrect 64bit register order. Possible missing low. + * for example: + * { XXX_REG(0), REG_32BIT, 0, 0, NULL}, + * { XXX_REG_HI(0), REG_64BIT_HI_DW, 0, 0, NULL}, + */ + XE_WARN_ON(!low32_ready); + low32_ready = false; + + drm_printf(p, "\t%s: 0x%016llx\n", reg_desc->regname, value_qw); + break; + } + + case REG_32BIT: + /* + * Incorrect 64bit register order. Possible missing high. + * for example: + * { XXX_REG_LO(0), REG_64BIT_LOW_DW, 0, 0, NULL}, + * { XXX_REG(0), REG_32BIT, 0, 0, "XXX_REG"}, + */ + XE_WARN_ON(low32_ready); + + if (FIELD_GET(GUC_REGSET_STEERING_NEEDED, reg_desc->flags)) + drm_printf(p, "\t%s[%u]: 0x%08x\n", reg_desc->regname, + reg_desc->dss_id, value); + else + drm_printf(p, "\t%s: 0x%08x\n", reg_desc->regname, value); + + break; + } + } + + /* + * Incorrect 64bit register order. Possible missing high. + * for example: + * { XXX_REG_LO(0), REG_64BIT_LOW_DW, 0, 0, NULL}, + * } // <- Register list end + */ + XE_WARN_ON(low32_ready); +} + +/** + * xe_engine_snapshot_print - Print out a given Xe HW Engine snapshot. + * @snapshot: Xe HW Engine snapshot object. + * @p: drm_printer where it will be printed out. + * + * This function prints out a given Xe HW Engine snapshot object. + */ +void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm_printer *p) +{ + const char *grptype[GUC_STATE_CAPTURE_GROUP_TYPE_MAX] = { + "full-capture", + "partial-capture" + }; + int type; + const struct __guc_mmio_reg_descr_group *list; + enum guc_capture_list_class_type capture_class; + + struct xe_gt *gt; + struct xe_device *xe; + struct xe_devcoredump *devcoredump; + struct xe_devcoredump_snapshot *devcore_snapshot; + + if (!snapshot) + return; + + gt = snapshot->hwe->gt; + xe = gt_to_xe(gt); + devcoredump = &xe->devcoredump; + devcore_snapshot = &devcoredump->snapshot; + + if (!devcore_snapshot->matched_node) + return; + + xe_gt_assert(gt, snapshot->hwe); + + capture_class = xe_hwe_to_guc_capture_class(snapshot->hwe); + + drm_printf(p, "%s (physical), logical instance=%d\n", + snapshot->name ? snapshot->name : "", + snapshot->logical_instance); + drm_printf(p, "\tCapture_source: %s\n", + devcore_snapshot->matched_node->source == XE_ENGINE_CAPTURE_SOURCE_GUC ? + "GuC" : "Manual"); + drm_printf(p, "\tCoverage: %s\n", grptype[devcore_snapshot->matched_node->is_partial]); + drm_printf(p, "\tForcewake: domain 0x%x, ref %d\n", + snapshot->forcewake.domain, snapshot->forcewake.ref); + drm_printf(p, "\tReserved: %s\n", + str_yes_no(snapshot->kernel_reserved)); + + for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { + list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, + capture_class, false); + snapshot_print_by_list_order(snapshot, p, type, list); + } + + if (capture_class == GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE) { + list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, + GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + capture_class, true); + snapshot_print_by_list_order(snapshot, p, GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, + list); + } + + drm_puts(p, "\n"); +} + +/** + * xe_guc_capture_get_matching_and_lock - Matching GuC capture for the queue. + * @q: The exec queue object + * + * Search within the capture outlist for the queue, could be used for check if + * GuC capture is ready for the queue. + * If found, the locked boolean of the node will be flagged. + * + * Returns: found guc-capture node ptr else NULL + */ +struct __guc_capture_parsed_output * +xe_guc_capture_get_matching_and_lock(struct xe_exec_queue *q) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_device *xe; + u16 guc_class = GUC_LAST_ENGINE_CLASS + 1; + struct xe_devcoredump_snapshot *ss; + + if (!q || !q->gt) + return NULL; + + xe = gt_to_xe(q->gt); + + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) + return NULL; + + if (!xe_device_uc_enabled(xe)) + return NULL; + + if (IS_SRIOV_VF(xe)) + return NULL; + + ss = &xe->devcoredump.snapshot; + if (ss->matched_node && ss->matched_node->source == XE_ENGINE_CAPTURE_SOURCE_GUC) + return ss->matched_node; + + /* Find hwe for the queue */ + for_each_hw_engine(hwe, q->gt, id) { + if (hwe != q->hwe) + continue; + guc_class = xe_hwe_to_guc_class(hwe); + break; + } + + if (guc_class <= GUC_LAST_ENGINE_CLASS) { + struct __guc_capture_parsed_output *n, *ntmp; + struct xe_guc *guc = &q->gt->uc.guc; + u16 guc_id = q->guc->id; + u32 lrca = xe_lrc_ggtt_addr(q->lrc[0]); + + /* + * Look for a matching GuC reported error capture node from + * the internal output link-list based on engine, guc id and + * lrca info. + */ + list_for_each_entry_safe(n, ntmp, &guc->capture->outlist, link) { + if (n->eng_class == guc_class && n->eng_inst == hwe->instance && + n->guc_id == guc_id && n->lrca == lrca && + n->source == XE_ENGINE_CAPTURE_SOURCE_GUC) { + n->locked = 1; + return n; + } + } + } + return NULL; +} + +/** + * xe_engine_snapshot_capture_for_queue - Take snapshot of associated engine + * @q: The exec queue object + * + * Take snapshot of associated HW Engine + * + * Returns: None. + */ +void +xe_engine_snapshot_capture_for_queue(struct xe_exec_queue *q) +{ + struct xe_device *xe = gt_to_xe(q->gt); + struct xe_devcoredump *coredump = &xe->devcoredump; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 adj_logical_mask = q->logical_mask; + + if (IS_SRIOV_VF(xe)) + return; + + for_each_hw_engine(hwe, q->gt, id) { + if (hwe->class != q->hwe->class || + !(BIT(hwe->logical_instance) & adj_logical_mask)) { + coredump->snapshot.hwe[id] = NULL; + continue; + } + + if (!coredump->snapshot.hwe[id]) { + coredump->snapshot.hwe[id] = + xe_hw_engine_snapshot_capture(hwe, q); + } else { + struct __guc_capture_parsed_output *new; + + new = xe_guc_capture_get_matching_and_lock(q); + if (new) { + struct xe_guc *guc = &q->gt->uc.guc; + + /* + * If we are in here, it means we found a fresh + * GuC-err-capture node for this engine after + * previously failing to find a match in the + * early part of guc_exec_queue_timedout_job. + * Thus we must free the manually captured node + */ + guc_capture_free_outlist_node(guc->capture, + coredump->snapshot.matched_node); + coredump->snapshot.matched_node = new; + } + } + + break; + } +} + +/* + * xe_guc_capture_put_matched_nodes - Cleanup matched nodes + * @guc: The GuC object + * + * Free matched node and all nodes with the equal guc_id from + * GuC captured outlist + */ +void xe_guc_capture_put_matched_nodes(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_devcoredump *devcoredump = &xe->devcoredump; + struct __guc_capture_parsed_output *n = devcoredump->snapshot.matched_node; + + if (n) { + guc_capture_remove_stale_matches_from_list(guc->capture, n); + guc_capture_free_outlist_node(guc->capture, n); + devcoredump->snapshot.matched_node = NULL; + } +} + +/* + * xe_guc_capture_steered_list_init - Init steering register list + * @guc: The GuC object + * + * Init steering register list for GuC register capture, create pre-alloc node + */ +void xe_guc_capture_steered_list_init(struct xe_guc *guc) +{ + /* + * For certain engine classes, there are slice and subslice + * level registers requiring steering. We allocate and populate + * these based on hw config and add it as an extension list at + * the end of the pre-populated render list. + */ + guc_capture_alloc_steered_lists(guc); + check_guc_capture_size(guc); + guc_capture_create_prealloc_nodes(guc); +} + +/* + * xe_guc_capture_init - Init for GuC register capture + * @guc: The GuC object + * + * Init for GuC register capture, alloc memory for capture data structure. + * + * Returns: 0 if success. + * -ENOMEM if out of memory + */ +int xe_guc_capture_init(struct xe_guc *guc) +{ + guc->capture = drmm_kzalloc(guc_to_drm(guc), sizeof(*guc->capture), GFP_KERNEL); + if (!guc->capture) + return -ENOMEM; + + guc->capture->reglists = guc_capture_get_device_reglist(guc_to_xe(guc)); + + INIT_LIST_HEAD(&guc->capture->outlist); + INIT_LIST_HEAD(&guc->capture->cachelist); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_guc_capture.h b/drivers/gpu/drm/xe/xe_guc_capture.h new file mode 100644 index 000000000..fcd4f1298 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_capture.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021-2024 Intel Corporation + */ + +#ifndef _XE_GUC_CAPTURE_H_ +#define _XE_GUC_CAPTURE_H_ + +#include +#include "abi/guc_capture_abi.h" +#include "abi/guc_scheduler_abi.h" +#include "xe_guc.h" + +struct xe_exec_queue; +struct xe_guc; +struct xe_hw_engine; +struct xe_hw_engine_snapshot; + +static inline enum guc_capture_list_class_type xe_guc_class_to_capture_class(u16 class) +{ + switch (class) { + case GUC_RENDER_CLASS: + case GUC_COMPUTE_CLASS: + return GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE; + case GUC_GSC_OTHER_CLASS: + return GUC_CAPTURE_LIST_CLASS_GSC_OTHER; + case GUC_VIDEO_CLASS: + case GUC_VIDEOENHANCE_CLASS: + case GUC_BLITTER_CLASS: + return class; + case GUC_PAGING_CLASS: + return GUC_CAPTURE_LIST_CLASS_PAGING; + default: + XE_WARN_ON(class); + return GUC_CAPTURE_LIST_CLASS_MAX; + } +} + +static inline enum guc_capture_list_class_type +xe_hwe_to_guc_capture_class(struct xe_hw_engine *hwe) +{ + return xe_guc_class_to_capture_class(xe_hwe_to_guc_class(hwe)); +} + +void xe_guc_capture_process(struct xe_guc *guc); +int xe_guc_capture_getlist(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, void **outptr); +int xe_guc_capture_getlistsize(struct xe_guc *guc, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, size_t *size); +int xe_guc_capture_getnullheader(struct xe_guc *guc, void **outptr, size_t *size); +size_t xe_guc_capture_ads_input_worst_size(struct xe_guc *guc); +const struct __guc_mmio_reg_descr_group * +xe_guc_capture_get_reg_desc_list(struct xe_gt *gt, u32 owner, u32 type, + enum guc_capture_list_class_type capture_class, bool is_ext); +struct __guc_capture_parsed_output *xe_guc_capture_get_matching_and_lock(struct xe_exec_queue *q); +void xe_engine_manual_capture(struct xe_hw_engine *hwe, struct xe_hw_engine_snapshot *snapshot); +void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm_printer *p); +void xe_engine_snapshot_capture_for_queue(struct xe_exec_queue *q); +void xe_guc_capture_steered_list_init(struct xe_guc *guc); +void xe_guc_capture_put_matched_nodes(struct xe_guc *guc); +int xe_guc_capture_init(struct xe_guc *guc); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_capture_types.h b/drivers/gpu/drm/xe/xe_guc_capture_types.h new file mode 100644 index 000000000..058a3f2ea --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_capture_types.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021-2024 Intel Corporation + */ + +#ifndef _XE_GUC_CAPTURE_TYPES_H_ +#define _XE_GUC_CAPTURE_TYPES_H_ + +#include +#include "regs/xe_reg_defs.h" + +struct xe_guc; + +/* data type of the register in register list */ +enum capture_register_data_type { + REG_32BIT = 0, + REG_64BIT_LOW_DW, + REG_64BIT_HI_DW, +}; + +/** + * struct __guc_mmio_reg_descr - GuC mmio register descriptor + * + * xe_guc_capture module uses these structures to define a register + * (offsets, names, flags,...) that are used at the ADS registration + * time as well as during runtime processing and reporting of error- + * capture states generated by GuC just prior to engine reset events. + */ +struct __guc_mmio_reg_descr { + /** @reg: the register */ + struct xe_reg reg; + /** + * @data_type: data type of the register + * Could be 32 bit, low or hi dword of a 64 bit, see enum + * capture_register_data_type + */ + enum capture_register_data_type data_type; + /** @flags: Flags for the register */ + u32 flags; + /** @mask: The mask to apply */ + u32 mask; + /** @dss_id: Cached index for steered registers */ + u32 dss_id; + /** @regname: Name of the register */ + const char *regname; +}; + +/** + * struct __guc_mmio_reg_descr_group - The group of register descriptor + * + * xe_guc_capture module uses these structures to maintain static + * tables (per unique platform) that consists of lists of registers + * (offsets, names, flags,...) that are used at the ADS registration + * time as well as during runtime processing and reporting of error- + * capture states generated by GuC just prior to engine reset events. + */ +struct __guc_mmio_reg_descr_group { + /** @list: The register list */ + const struct __guc_mmio_reg_descr *list; + /** @num_regs: Count of registers in the list */ + u32 num_regs; + /** @owner: PF/VF owner, see enum guc_capture_list_index_type */ + u32 owner; + /** @type: Capture register type, see enum guc_state_capture_type */ + u32 type; + /** @engine: The engine class, see enum guc_capture_list_class_type */ + u32 engine; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c new file mode 100644 index 000000000..fe70c0fd8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -0,0 +1,2326 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_ct.h" + +#include +#include +#include +#include + +#include + +#include + +#include "abi/guc_actions_abi.h" +#include "abi/guc_actions_sriov_abi.h" +#include "abi/guc_klvs_abi.h" +#include "xe_bo.h" +#include "xe_devcoredump.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_monitor.h" +#include "xe_guc.h" +#include "xe_guc_log.h" +#include "xe_guc_pagefault.h" +#include "xe_guc_relay.h" +#include "xe_guc_submit.h" +#include "xe_guc_tlb_inval.h" +#include "xe_map.h" +#include "xe_page_reclaim.h" +#include "xe_pm.h" +#include "xe_sleep.h" +#include "xe_sriov_vf.h" +#include "xe_trace_guc.h" + +static void receive_g2h(struct xe_guc_ct *ct); +static void g2h_worker_func(struct work_struct *w); +static void safe_mode_worker_func(struct work_struct *w); +static void ct_exit_safe_mode(struct xe_guc_ct *ct); +static void guc_ct_change_state(struct xe_guc_ct *ct, + enum xe_guc_ct_state state); + +static struct xe_guc *ct_to_guc(struct xe_guc_ct *ct) +{ + return container_of(ct, struct xe_guc, ct); +} + +static struct xe_gt *ct_to_gt(struct xe_guc_ct *ct) +{ + return container_of(ct, struct xe_gt, uc.guc.ct); +} + +static struct xe_device *ct_to_xe(struct xe_guc_ct *ct) +{ + return gt_to_xe(ct_to_gt(ct)); +} + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +enum { + /* Internal states, not error conditions */ + CT_DEAD_STATE_REARM, /* 0x0001 */ + CT_DEAD_STATE_CAPTURE, /* 0x0002 */ + + /* Error conditions */ + CT_DEAD_SETUP, /* 0x0004 */ + CT_DEAD_H2G_WRITE, /* 0x0008 */ + CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ + CT_DEAD_G2H_READ, /* 0x0020 */ + CT_DEAD_G2H_RECV, /* 0x0040 */ + CT_DEAD_G2H_RELEASE, /* 0x0080 */ + CT_DEAD_DEADLOCK, /* 0x0100 */ + CT_DEAD_PROCESS_FAILED, /* 0x0200 */ + CT_DEAD_FAST_G2H, /* 0x0400 */ + CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ + CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ + CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ + CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ + CT_DEAD_CRASH, /* 0x8000 */ +}; + +static void ct_dead_worker_func(struct work_struct *w); +static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); + +static void ct_dead_fini(struct xe_guc_ct *ct) +{ + cancel_work_sync(&ct->dead.worker); +} + +static void ct_dead_init(struct xe_guc_ct *ct) +{ + spin_lock_init(&ct->dead.lock); + INIT_WORK(&ct->dead.worker, ct_dead_worker_func); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + stack_depot_init(); +#endif +} + +static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot) +{ +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + unsigned long entries[SZ_32]; + unsigned int n; + + n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + /* May be called under spinlock, so avoid sleeping */ + ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); +#endif +} + +static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot) +{ + struct xe_gt *gt = ct_to_gt(ct); +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT); + + if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0)) + xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n", + fence, ct->fast_req[slot].action, buf); + else + xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", + fence, ct->fast_req[slot].action); +#else + xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", + fence, ct->fast_req[slot].action); +#endif +} + +static void fast_req_report(struct xe_guc_ct *ct, u16 fence) +{ + u16 fence_min = U16_MAX, fence_max = 0; + struct xe_gt *gt = ct_to_gt(ct); + unsigned int n; + + lockdep_assert_held(&ct->lock); + + for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { + if (ct->fast_req[n].fence < fence_min) + fence_min = ct->fast_req[n].fence; + if (ct->fast_req[n].fence > fence_max) + fence_max = ct->fast_req[n].fence; + + if (ct->fast_req[n].fence != fence) + continue; + + return fast_req_dump(ct, fence, n); + } + + xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", + fence, fence_min, fence_max, ct->fence_seqno); +} + +static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) +{ + unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); + + fast_req_stack_save(ct, slot); + ct->fast_req[slot].fence = fence; + ct->fast_req[slot].action = action; +} + +#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) + +#else + +static void ct_dead_fini(struct xe_guc_ct *ct) { } +static void ct_dead_init(struct xe_guc_ct *ct) { } + +static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { } +static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } + +#define CT_DEAD(ct, ctb, reason) \ + do { \ + struct guc_ctb *_ctb = (ctb); \ + if (_ctb) \ + _ctb->info.broken = true; \ + } while (0) + +#endif + +/* Used when a CT send wants to block and / or receive data */ +struct g2h_fence { + u32 *response_buffer; + u32 seqno; + /* fields below this point are setup based on the response */ + u32 response_data; + u16 response_len; + u16 error; + u16 hint; + u16 reason; + u32 counter; + bool cancel; + bool retry; + bool wait; + bool fail; + bool done; +}; + +static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) +{ + memset(g2h_fence, 0, sizeof(*g2h_fence)); + g2h_fence->response_buffer = response_buffer; + g2h_fence->seqno = ~0x0; +} + +static void g2h_fence_reinit(struct g2h_fence *g2h_fence) +{ + memset_after(g2h_fence, 0, seqno); +} + +static void g2h_fence_cancel(struct g2h_fence *g2h_fence) +{ + g2h_fence->cancel = true; + g2h_fence->fail = true; + + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ + WRITE_ONCE(g2h_fence->done, true); +} + +static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) +{ + return g2h_fence->seqno == ~0x0; +} + +/** + * DOC: GuC CTB Blob + * + * We allocate single blob to hold both CTB descriptors and buffers: + * + * +--------+-----------------------------------------------+------+ + * | offset | contents | size | + * +========+===============================================+======+ + * | 0x0000 | H2G CTB Descriptor (send) | | + * +--------+-----------------------------------------------+ 4K | + * | 0x0800 | G2H CTB Descriptor (g2h) | | + * +--------+-----------------------------------------------+------+ + * | 0x1000 | H2G CT Buffer (send) | n*4K | + * | | | | + * +--------+-----------------------------------------------+------+ + * | 0x1000 | G2H CT Buffer (g2h) | m*4K | + * | + n*4K | | | + * +--------+-----------------------------------------------+------+ + * + * Size of each ``CT Buffer`` must be multiple of 4K. + * We don't expect too many messages in flight at any time, unless we are + * using the GuC submission. In that case each request requires a minimum + * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this + * enough space to avoid backpressure on the driver. We increase the size + * of the receive buffer (relative to the send) to ensure a G2H response + * CTB has a landing spot. + * + * In addition to submissions, the G2H buffer needs to be able to hold + * enough space for recoverable page fault notifications. The number of + * page faults is interrupt driven and can be as much as the number of + * compute resources available. However, most of the actual work for these + * is in a separate page fault worker thread. Therefore we only need to + * make sure the queue has enough space to handle all of the submissions + * and responses and an extra buffer for incoming page faults. + */ + +#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) +#define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2) +#define CTB_G2H_BUFFER_OFFSET (CTB_DESC_SIZE * 2) +#define CTB_H2G_BUFFER_SIZE (SZ_4K) +#define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32)) +#define CTB_G2H_BUFFER_SIZE (SZ_128K) +#define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32)) +#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) +#define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2) + +/** + * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full + * CT command queue + * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. + * + * Observation is that a 4KiB buffer full of commands takes a little over a + * second to process. Use that to calculate maximum time to process a full CT + * command queue. + * + * Return: Maximum time to process a full CT queue in jiffies. + */ +long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) +{ + BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4K)); + return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; +} + +static size_t guc_h2g_size(void) +{ + return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE; +} + +static size_t guc_g2h_size(void) +{ + return CTB_G2H_BUFFER_OFFSET + CTB_G2H_BUFFER_SIZE; +} + +static void guc_ct_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc_ct *ct = arg; + + ct_dead_fini(ct); + ct_exit_safe_mode(ct); + destroy_workqueue(ct->g2h_wq); + xa_destroy(&ct->fence_lookup); +} + +static void primelockdep(struct xe_guc_ct *ct) +{ + if (!IS_ENABLED(CONFIG_LOCKDEP)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&ct->lock); + fs_reclaim_release(GFP_KERNEL); +} + +int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + int err; + + xe_gt_assert(gt, !(guc_h2g_size() % PAGE_SIZE)); + xe_gt_assert(gt, !(guc_g2h_size() % PAGE_SIZE)); + + err = drmm_mutex_init(&xe->drm, &ct->lock); + if (err) + return err; + + primelockdep(ct); + + ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); + if (!ct->g2h_wq) + return -ENOMEM; + + spin_lock_init(&ct->fast_lock); + xa_init(&ct->fence_lookup); + INIT_WORK(&ct->g2h_worker, g2h_worker_func); + INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); + + ct_dead_init(ct); + init_waitqueue_head(&ct->wq); + init_waitqueue_head(&ct->g2h_fence_wq); + + err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); + if (err) + return err; + + xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); + ct->state = XE_GUC_CT_STATE_DISABLED; + return 0; +} +ALLOW_ERROR_INJECTION(xe_guc_ct_init_noalloc, ERRNO); /* See xe_pci_probe() */ + +static void guc_action_disable_ct(void *arg) +{ + struct xe_guc_ct *ct = arg; + + xe_guc_ct_stop(ct); + guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); +} + +int xe_guc_ct_init(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *bo; + + bo = xe_managed_bo_create_pin_map(xe, tile, guc_h2g_size(), + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ct->ctbs.h2g.bo = bo; + + bo = xe_managed_bo_create_pin_map(xe, tile, guc_g2h_size(), + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ct->ctbs.g2h.bo = bo; + + return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); +} +ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_guc_ct_init_post_hwconfig - Reinitialize the GuC CTB in VRAM + * @ct: the &xe_guc_ct + * + * Allocate a new BO in VRAM and free the previous BO that was allocated + * in system memory (SMEM). Applicable only for DGFX products. + * + * Return: 0 on success, or a negative errno on failure. + */ +int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + struct xe_tile *tile = gt_to_tile(gt); + int ret; + + xe_assert(xe, !xe_guc_ct_enabled(ct)); + + if (IS_DGFX(xe)) { + ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->ctbs.h2g.bo); + if (ret) + return ret; + } + + devm_remove_action(xe->drm.dev, guc_action_disable_ct, ct); + return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); +} + +#define desc_read(xe_, guc_ctb__, field_) \ + xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ + struct guc_ct_buffer_desc, field_) + +#define desc_write(xe_, guc_ctb__, field_, val_) \ + xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ + struct guc_ct_buffer_desc, field_, val_) + +static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, + struct iosys_map *map) +{ + h2g->info.size = CTB_H2G_BUFFER_DWORDS; + h2g->info.resv_space = 0; + h2g->info.tail = 0; + h2g->info.head = 0; + h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, + h2g->info.size) - + h2g->info.resv_space; + h2g->info.broken = false; + + h2g->desc = *map; + xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); + + h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET); +} + +static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, + struct iosys_map *map) +{ + g2h->info.size = CTB_G2H_BUFFER_DWORDS; + g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS; + g2h->info.head = 0; + g2h->info.tail = 0; + g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, + g2h->info.size) - + g2h->info.resv_space; + g2h->info.broken = false; + + g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); + xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); + + g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_G2H_BUFFER_OFFSET); +} + +static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) +{ + struct xe_guc *guc = ct_to_guc(ct); + u32 desc_addr, ctb_addr, size; + int err; + + desc_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo); + ctb_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo) + CTB_H2G_BUFFER_OFFSET; + size = ct->ctbs.h2g.info.size * sizeof(u32); + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, + desc_addr); + if (err) + return err; + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, + ctb_addr); + if (err) + return err; + + return xe_guc_self_cfg32(guc, + GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, + size); +} + +static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) +{ + struct xe_guc *guc = ct_to_guc(ct); + u32 desc_addr, ctb_addr, size; + int err; + + desc_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_DESC_SIZE; + ctb_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_G2H_BUFFER_OFFSET; + size = ct->ctbs.g2h.info.size * sizeof(u32); + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, + desc_addr); + if (err) + return err; + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, + ctb_addr); + if (err) + return err; + + return xe_guc_self_cfg32(guc, + GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, + size); +} + +static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) +{ + u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_ACTION_HOST2GUC_CONTROL_CTB), + FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, + enable ? GUC_CTB_CONTROL_ENABLE : + GUC_CTB_CONTROL_DISABLE), + }; + int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static void guc_ct_change_state(struct xe_guc_ct *ct, + enum xe_guc_ct_state state) +{ + struct xe_gt *gt = ct_to_gt(ct); + struct g2h_fence *g2h_fence; + unsigned long idx; + + mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ + spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ + + xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || + state == XE_GUC_CT_STATE_STOPPED); + + if (ct->g2h_outstanding) + xe_pm_runtime_put(ct_to_xe(ct)); + ct->g2h_outstanding = 0; + + /* + * WRITE_ONCE pairs with READ_ONCEs in xe_guc_ct_initialized and + * xe_guc_ct_enabled. + */ + WRITE_ONCE(ct->state, state); + + xe_gt_dbg(gt, "GuC CT communication channel %s\n", + state == XE_GUC_CT_STATE_STOPPED ? "stopped" : + str_enabled_disabled(state == XE_GUC_CT_STATE_ENABLED)); + + spin_unlock_irq(&ct->fast_lock); + + /* cancel all in-flight send-recv requests */ + xa_for_each(&ct->fence_lookup, idx, g2h_fence) + g2h_fence_cancel(g2h_fence); + + /* make sure guc_ct_send_recv() will see g2h_fence changes */ + smp_mb(); + wake_up_all(&ct->g2h_fence_wq); + + /* + * Lockdep doesn't like this under the fast lock and he destroy only + * needs to be serialized with the send path which ct lock provides. + */ + xa_destroy(&ct->fence_lookup); + + mutex_unlock(&ct->lock); +} + +static bool ct_needs_safe_mode(struct xe_guc_ct *ct) +{ + return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); +} + +static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) +{ + if (!ct_needs_safe_mode(ct)) + return false; + + queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); + return true; +} + +static void safe_mode_worker_func(struct work_struct *w) +{ + struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); + + receive_g2h(ct); + + if (!ct_restart_safe_mode_worker(ct)) + xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); +} + +static void ct_enter_safe_mode(struct xe_guc_ct *ct) +{ + if (ct_restart_safe_mode_worker(ct)) + xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); +} + +static void ct_exit_safe_mode(struct xe_guc_ct *ct) +{ + if (cancel_delayed_work_sync(&ct->safe_mode_worker)) + xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); +} + +static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + int err; + + xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); + + if (needs_register) { + xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, 0, 0, + xe_bo_size(ct->ctbs.h2g.bo)); + xe_map_memset(xe, &ct->ctbs.g2h.bo->vmap, 0, 0, + xe_bo_size(ct->ctbs.g2h.bo)); + guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->ctbs.h2g.bo->vmap); + guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->ctbs.g2h.bo->vmap); + + err = guc_ct_ctb_h2g_register(ct); + if (err) + goto err_out; + + err = guc_ct_ctb_g2h_register(ct); + if (err) + goto err_out; + + err = guc_ct_control_toggle(ct, true); + if (err) + goto err_out; + } else { + ct->ctbs.h2g.info.broken = false; + ct->ctbs.g2h.info.broken = false; + /* Skip everything in H2G buffer */ + xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, CTB_H2G_BUFFER_OFFSET, 0, + CTB_H2G_BUFFER_SIZE); + } + + guc_ct_change_state(ct, XE_GUC_CT_STATE_ENABLED); + + smp_mb(); + wake_up_all(&ct->wq); + + if (ct_needs_safe_mode(ct)) + ct_enter_safe_mode(ct); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) + /* + * The CT has now been reset so the dumper can be re-armed + * after any existing dead state has been dumped. + */ + spin_lock_irq(&ct->dead.lock); + if (ct->dead.reason) { + ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); + queue_work(system_dfl_wq, &ct->dead.worker); + } + spin_unlock_irq(&ct->dead.lock); +#endif + + return 0; + +err_out: + xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); + CT_DEAD(ct, NULL, SETUP); + + return err; +} + +/** + * xe_guc_ct_restart() - Restart GuC CT + * @ct: the &xe_guc_ct + * + * Restart GuC CT to an empty state without issuing a CT register MMIO command. + * + * Return: 0 on success, or a negative errno on failure. + */ +int xe_guc_ct_restart(struct xe_guc_ct *ct) +{ + return __xe_guc_ct_start(ct, false); +} + +/** + * xe_guc_ct_enable() - Enable GuC CT + * @ct: the &xe_guc_ct + * + * Enable GuC CT to an empty state and issue a CT register MMIO command. + * + * Return: 0 on success, or a negative errno on failure. + */ +int xe_guc_ct_enable(struct xe_guc_ct *ct) +{ + return __xe_guc_ct_start(ct, true); +} + +static void stop_g2h_handler(struct xe_guc_ct *ct) +{ + cancel_work_sync(&ct->g2h_worker); +} + +/** + * xe_guc_ct_disable - Set GuC to disabled state + * @ct: the &xe_guc_ct + * + * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected + * in this transition. + */ +void xe_guc_ct_disable(struct xe_guc_ct *ct) +{ + guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); + ct_exit_safe_mode(ct); + stop_g2h_handler(ct); +} + +/** + * xe_guc_ct_flush_and_stop - Flush and stop all processing of G2H / H2G + * @ct: the &xe_guc_ct + */ +void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct) +{ + receive_g2h(ct); + xe_guc_ct_stop(ct); +} + +/** + * xe_guc_ct_stop - Set GuC to stopped state + * @ct: the &xe_guc_ct + * + * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h + */ +void xe_guc_ct_stop(struct xe_guc_ct *ct) +{ + if (!xe_guc_ct_initialized(ct)) + return; + + guc_ct_change_state(ct, XE_GUC_CT_STATE_STOPPED); + stop_g2h_handler(ct); +} + +/** + * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend + * @ct: the &xe_guc_ct + * + * Set GuC CT to disabled state. + */ +void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct) +{ + struct guc_ctb *g2h = &ct->ctbs.g2h; + u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS; + + /* We should be back to guc_ct_ctb_g2h_init() values */ + xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits); + + /* + * Since we're already in runtime suspend path, we shouldn't have pending + * messages. But if there happen to be any, we'd probably want them to be + * thrown as errors for further investigation. + */ + xe_guc_ct_disable(ct); +} + +/** + * xe_guc_ct_runtime_resume() - GuC CT runtime resume + * @ct: the &xe_guc_ct + * + * Restart GuC CT and set it to enabled state. + */ +void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct) +{ + xe_guc_ct_restart(ct); +} + +static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) +{ + struct guc_ctb *h2g = &ct->ctbs.h2g; + + lockdep_assert_held(&ct->lock); + + if (cmd_len > h2g->info.space) { + h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); + + if (h2g->info.head > h2g->info.size) { + struct xe_device *xe = ct_to_xe(ct); + u32 desc_status = desc_read(xe, h2g, status); + + desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); + + xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", + h2g->info.head, h2g->info.size); + CT_DEAD(ct, h2g, H2G_HAS_ROOM); + return false; + } + + h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, + h2g->info.size) - + h2g->info.resv_space; + if (cmd_len > h2g->info.space) + return false; + } + + return true; +} + +static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) +{ + if (!g2h_len) + return true; + + lockdep_assert_held(&ct->fast_lock); + + return ct->ctbs.g2h.info.space > g2h_len; +} + +static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) +{ + lockdep_assert_held(&ct->lock); + + if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) + return -EBUSY; + + return 0; +} + +static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) +{ + lockdep_assert_held(&ct->lock); + ct->ctbs.h2g.info.space -= cmd_len; +} + +static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) +{ + xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); + xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || + (g2h_len && num_g2h)); + + if (g2h_len) { + lockdep_assert_held(&ct->fast_lock); + + if (!ct->g2h_outstanding) + xe_pm_runtime_get_noresume(ct_to_xe(ct)); + + ct->ctbs.g2h.info.space -= g2h_len; + ct->g2h_outstanding += num_g2h; + } +} + +static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) +{ + bool bad = false; + + lockdep_assert_held(&ct->fast_lock); + + bad = ct->ctbs.g2h.info.space + g2h_len > + ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; + bad |= !ct->g2h_outstanding; + + if (bad) { + xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", + ct->ctbs.g2h.info.space, g2h_len, + ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, + ct->ctbs.g2h.info.space + g2h_len, + ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, + ct->g2h_outstanding); + CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); + return; + } + + ct->ctbs.g2h.info.space += g2h_len; + if (!--ct->g2h_outstanding) + xe_pm_runtime_put(ct_to_xe(ct)); +} + +static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) +{ + spin_lock_irq(&ct->fast_lock); + __g2h_release_space(ct, g2h_len); + spin_unlock_irq(&ct->fast_lock); +} + +/* + * The CT protocol accepts a 16 bits fence. This field is fully owned by the + * driver, the GuC will just copy it to the reply message. Since we need to + * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, + * we use one bit of the seqno as an indicator for that and a rolling counter + * for the remaining 15 bits. + */ +#define CT_SEQNO_MASK GENMASK(14, 0) +#define CT_SEQNO_UNTRACKED BIT(15) +static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) +{ + u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; + + if (!is_g2h_fence) + seqno |= CT_SEQNO_UNTRACKED; + + return seqno; +} + +#define MAKE_ACTION(type, __action) \ +({ \ + FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | \ + FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | \ + GUC_HXG_EVENT_MSG_0_DATA0, __action); \ +}) + +static bool vf_action_can_safely_fail(struct xe_device *xe, u32 action) +{ + /* + * When resuming a VF, we can't reliably track whether context + * registration has completed in the GuC state machine. It is harmless + * to resend the request, as it will fail silently if GUC_HXG_TYPE_EVENT + * is used. Additionally, if there is an H2G protocol issue on a VF, + * subsequent H2G messages sent as GUC_HXG_TYPE_FAST_REQUEST will likely + * fail. + */ + return IS_SRIOV_VF(xe) && xe_sriov_vf_migration_supported(xe) && + (action == XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC || + action == XE_GUC_ACTION_REGISTER_CONTEXT); +} + +#define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ + +static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 ct_fence_value, bool want_response) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + struct guc_ctb *h2g = &ct->ctbs.h2g; + u32 cmd[H2G_CT_HEADERS]; + u32 tail = h2g->info.tail; + u32 full_len; + struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, + tail * sizeof(u32)); + + full_len = len + GUC_CTB_HDR_LEN; + + lockdep_assert_held(&ct->lock); + xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + u32 desc_tail = desc_read(xe, h2g, tail); + u32 desc_head = desc_read(xe, h2g, head); + u32 desc_status; + + desc_status = desc_read(xe, h2g, status); + if (desc_status) { + xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); + goto corrupted; + } + + if (tail != desc_tail) { + desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); + xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); + goto corrupted; + } + + if (tail > h2g->info.size) { + desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); + xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", + tail, h2g->info.size); + goto corrupted; + } + + if (desc_head >= h2g->info.size) { + desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); + xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", + desc_head, h2g->info.size); + goto corrupted; + } + } + + /* Command will wrap, zero fill (NOPs), return and check credits again */ + if (tail + full_len > h2g->info.size) { + xe_map_memset(xe, &map, 0, 0, + (h2g->info.size - tail) * sizeof(u32)); + h2g_reserve_space(ct, (h2g->info.size - tail)); + h2g->info.tail = 0; + desc_write(xe, h2g, tail, h2g->info.tail); + + return -EAGAIN; + } + + /* + * dw0: CT header (including fence) + * dw1: HXG header (including action code) + * dw2+: action data + */ + cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | + FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | + FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); + if (want_response) { + cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_REQUEST, action[0]); + } else if (vf_action_can_safely_fail(xe, action[0])) { + cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_EVENT, action[0]); + } else { + fast_req_track(ct, ct_fence_value, + FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); + + cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_FAST_REQUEST, action[0]); + } + + /* H2G header in cmd[1] replaces action[0] so: */ + --len; + ++action; + + /* Write H2G ensuring visible before descriptor update */ + xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); + xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); + xe_device_wmb(xe); + + /* Update local copies */ + h2g->info.tail = (tail + full_len) % h2g->info.size; + h2g_reserve_space(ct, full_len); + + /* Update descriptor */ + desc_write(xe, h2g, tail, h2g->info.tail); + + /* + * desc_read() performs an VRAM read which serializes the CPU and drains + * posted writes on dGPU platforms. Tracepoints evaluate arguments even + * when disabled, so guard the event to avoid adding µs-scale latency to + * the fast H2G submission path when tracing is not active. + */ + if (trace_xe_guc_ctb_h2g_enabled()) + trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, + desc_read(xe, h2g, head), h2g->info.tail); + + return 0; + +corrupted: + CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); + return -EPIPE; +} + +static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, + u32 len, u32 g2h_len, u32 num_g2h, + struct g2h_fence *g2h_fence) +{ + struct xe_gt *gt = ct_to_gt(ct); + u16 seqno; + int ret; + + xe_gt_assert(gt, xe_guc_ct_initialized(ct)); + xe_gt_assert(gt, !g2h_len || !g2h_fence); + xe_gt_assert(gt, !num_g2h || !g2h_fence); + xe_gt_assert(gt, !g2h_len || num_g2h); + xe_gt_assert(gt, g2h_len || !num_g2h); + lockdep_assert_held(&ct->lock); + + if (xe_device_wedged(ct_to_xe(ct))) { + ret = -ENOTRECOVERABLE; + goto out; + } + + if (unlikely(ct->ctbs.h2g.info.broken)) { + ret = -EPIPE; + goto out; + } + + if (ct->state == XE_GUC_CT_STATE_DISABLED) { + ret = -ENODEV; + goto out; + } + + if (ct->state == XE_GUC_CT_STATE_STOPPED || xe_gt_recovery_pending(gt)) { + ret = -ECANCELED; + goto out; + } + + xe_gt_assert(gt, xe_guc_ct_enabled(ct)); + + if (g2h_fence) { + g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; + num_g2h = 1; + + if (g2h_fence_needs_alloc(g2h_fence)) { + g2h_fence->seqno = next_ct_seqno(ct, true); + ret = xa_err(xa_store(&ct->fence_lookup, + g2h_fence->seqno, g2h_fence, + GFP_ATOMIC)); + if (ret) + goto out; + } + + seqno = g2h_fence->seqno; + } else { + seqno = next_ct_seqno(ct, false); + } + + if (g2h_len) + spin_lock_irq(&ct->fast_lock); +retry: + ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); + if (unlikely(ret)) + goto out_unlock; + + ret = h2g_write(ct, action, len, seqno, !!g2h_fence); + if (unlikely(ret)) { + if (ret == -EAGAIN) + goto retry; + goto out_unlock; + } + + __g2h_reserve_space(ct, g2h_len, num_g2h); + xe_guc_notify(ct_to_guc(ct)); +out_unlock: + if (g2h_len) + spin_unlock_irq(&ct->fast_lock); +out: + return ret; +} + +static void kick_reset(struct xe_guc_ct *ct) +{ + xe_gt_reset_async(ct_to_gt(ct)); +} + +static int dequeue_one_g2h(struct xe_guc_ct *ct); + +/* + * wait before retry of sending h2g message + * Return: true if ready for retry, false if the wait timeouted + */ +static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len, + u32 g2h_len, struct g2h_fence *g2h_fence, + unsigned int *sleep_period_ms, + unsigned int *sleep_total_ms) +{ + struct xe_device *xe = ct_to_xe(ct); + + /* + * We wait to try to restore credits for about 1 second before bailing. + * In the case of H2G credits we have no choice but just to wait for the + * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In + * the case of G2H we process any G2H in the channel, hopefully freeing + * credits as we consume the G2H messages. + */ + if (!h2g_has_room(ct, len + GUC_CTB_HDR_LEN)) { + struct guc_ctb *h2g = &ct->ctbs.h2g; + + if (*sleep_total_ms > 1000) + return false; + + trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, + h2g->info.size, + h2g->info.space, + len + GUC_CTB_HDR_LEN); + *sleep_total_ms += xe_sleep_exponential_ms(sleep_period_ms, 64); + } else { + struct guc_ctb *g2h = &ct->ctbs.g2h; + int ret; + + trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, + desc_read(xe, g2h, tail), + g2h->info.size, + g2h->info.space, + g2h_fence ? + GUC_CTB_HXG_MSG_MAX_LEN : + g2h_len); + +#define g2h_avail(ct) \ + (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) + if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || + g2h_avail(ct), HZ)) + return false; +#undef g2h_avail + + ret = dequeue_one_g2h(ct); + if (ret < 0) { + if (ret != -ECANCELED) + xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)\n", + ERR_PTR(ret)); + return false; + } + } + return true; +} + +static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h, + struct g2h_fence *g2h_fence) +{ + struct xe_gt *gt = ct_to_gt(ct); + unsigned int sleep_period_ms = 1; + unsigned int sleep_total_ms = 0; + int ret; + + xe_gt_assert(gt, !g2h_len || !g2h_fence); + lockdep_assert_held(&ct->lock); + xe_device_assert_mem_access(ct_to_xe(ct)); + +try_again: + ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, + g2h_fence); + + if (unlikely(ret == -EBUSY)) { + if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence, + &sleep_period_ms, &sleep_total_ms)) + goto broken; + goto try_again; + } + + return ret; + +broken: + xe_gt_err(gt, "No forward process on H2G, reset required\n"); + CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); + + return -EDEADLK; +} + +static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) +{ + int ret; + + xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); + + mutex_lock(&ct->lock); + ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); + mutex_unlock(&ct->lock); + + return ret; +} + +/** + * xe_guc_ct_send - Send an HXG message to the GuC over CT + * @ct: the &xe_guc_ct + * @action: dword array with the HXG message (can't be NULL) + * @len: length of the HXG message in dwords (can't be 0) + * @g2h_len: G2H response space to reserve in dwords, or 0 + * @num_g2h: number of G2H messages expected, or 0 + * + * Return codes from the non-blocking send helpers are: + * + * * -ENOTRECOVERABLE: the xe device is wedged. Stop submitting new GuC work; the + * request cannot make progress until the device is recovered. + * * -EPIPE: the H2G CTB is marked broken. The channel stays unusable until the + * CT is restarted, which clears the broken flag. + * * -ENODEV: the CT channel is disabled, messages not expected in this state. + * Don't retry until it is enabled again. + * * -ECANCELED: the CT channel is stopped or a GT recovery is pending; the + * message was dropped. Often benign. Cancel-tolerant callers (e.g. TLB + * invalidations, GuC submission) rely on the stop/start flow to recover; + * others should retry once the CT is re-enabled or the reset/recovery + * completes. + * * -EDEADLK: no CTB room and the wait for space timed out. The send helpers + * have already requested an async GT reset before returning this error. + * + * -ENOMEM may also be returned if an internal allocation fails; the blocking + * xe_guc_ct_send_recv() path retries that allocation. -EBUSY and + * -EAGAIN are internal flow-control results handled by the send helpers. + * + * Return: 0 on success, or a negative error code on failure. + */ +int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h) +{ + int ret; + + ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); + if (ret == -EDEADLK) + kick_reset(ct); + + return ret; +} + +int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h) +{ + int ret; + + ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); + if (ret == -EDEADLK) + kick_reset(ct); + + return ret; +} + +int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) +{ + int ret; + + lockdep_assert_held(&ct->lock); + + ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); + if (ret == -EDEADLK) + kick_reset(ct); + + return ret; +} + +/* + * Check if a GT reset is in progress or will occur and if GT reset brought the + * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. + */ +static bool retry_failure(struct xe_guc_ct *ct, int ret) +{ + if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) + return false; + +#define ct_alive(ct) \ + (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ + !ct->ctbs.g2h.info.broken) + if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) + return false; +#undef ct_alive + + return true; +} + +#define GUC_SEND_RETRY_LIMIT 50 +#define GUC_SEND_RETRY_MSLEEP 5 + +static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 *response_buffer, bool no_fail) +{ + struct xe_gt *gt = ct_to_gt(ct); + struct g2h_fence g2h_fence; + unsigned int retries = 0; + int ret = 0; + + /* + * We use a fence to implement blocking sends / receiving response data. + * The seqno of the fence is sent in the H2G, returned in the G2H, and + * an xarray is used as storage media with the seqno being to key. + * Fields in the fence hold success, failure, retry status and the + * response data. Safe to allocate on the stack as the xarray is the + * only reference and it cannot be present after this function exits. + */ +retry: + g2h_fence_init(&g2h_fence, response_buffer); +retry_same_fence: + ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); + if (unlikely(ret == -ENOMEM)) { + /* Retry allocation /w GFP_KERNEL */ + ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, + &g2h_fence, GFP_KERNEL)); + if (ret) + return ret; + + goto retry_same_fence; + } else if (unlikely(ret)) { + if (ret == -EDEADLK) + kick_reset(ct); + + if (no_fail && retry_failure(ct, ret)) + goto retry_same_fence; + + if (!g2h_fence_needs_alloc(&g2h_fence)) + xa_erase(&ct->fence_lookup, g2h_fence.seqno); + + return ret; + } + + /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response + * and g2h_fence_cancel. + */ +wait_again: + ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); + if (!ret) { + LNL_FLUSH_WORK(&ct->g2h_worker); + if (READ_ONCE(g2h_fence.done)) { + xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", + g2h_fence.seqno, action[0]); + ret = 1; + } + } + + /* + * Ensure we serialize with completion side to prevent UAF with fence going out of scope on + * the stack, since we have no clue if it will fire after the timeout before we can erase + * from the xa. Also we have some dependent loads and stores below for which we need the + * correct ordering, and we lack the needed barriers. + */ + mutex_lock(&ct->lock); + if (!ret) { + xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s\n", + g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); + xa_erase(&ct->fence_lookup, g2h_fence.seqno); + mutex_unlock(&ct->lock); + return -ETIME; + } + + if (g2h_fence.wait) { + xe_gt_dbg(gt, "H2G action %#x busy: counter %u\n", + action[0], g2h_fence.counter); + /* we can't leave any response data if we want to wait again */ + g2h_fence_reinit(&g2h_fence); + mutex_unlock(&ct->lock); + goto wait_again; + } + if (g2h_fence.retry) { + xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", + action[0], g2h_fence.reason); + mutex_unlock(&ct->lock); + if (++retries > GUC_SEND_RETRY_LIMIT) { + xe_gt_err(gt, "H2G action %#x reached retry limit=%u, aborting\n", + action[0], GUC_SEND_RETRY_LIMIT); + return -ELOOP; + } + msleep(GUC_SEND_RETRY_MSLEEP * retries); + goto retry; + } + if (g2h_fence.fail) { + if (g2h_fence.cancel) { + xe_gt_dbg(gt, "H2G request %#x canceled!\n", action[0]); + ret = xe_device_wedged(ct_to_xe(ct)) ? -ENOTRECOVERABLE : -ECANCELED; + goto unlock; + } + xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", + action[0], g2h_fence.error, g2h_fence.hint); + ret = -EIO; + } + + if (ret > 0) + ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; + +unlock: + mutex_unlock(&ct->lock); + + return ret; +} + +/** + * xe_guc_ct_send_recv - Send and receive HXG to the GuC + * @ct: the &xe_guc_ct + * @action: the dword array with `HXG Request`_ message (can't be NULL) + * @len: length of the `HXG Request`_ message (in dwords, can't be 0) + * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) + * + * Send a `HXG Request`_ message to the GuC over CT communication channel and + * blocks until GuC replies with a `HXG Response`_ message. + * + * For non-blocking communication with GuC use xe_guc_ct_send(). + * + * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. + * + * Return: response length (in dwords) if &response_buffer was not NULL, or + * DATA0 from `HXG Response`_ if &response_buffer was NULL, or + * a negative error code on failure. + */ +int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 *response_buffer) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); + return guc_ct_send_recv(ct, action, len, response_buffer, false); +} +ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); + +int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, + u32 len, u32 *response_buffer) +{ + return guc_ct_send_recv(ct, action, len, response_buffer, true); +} + +static u32 *msg_to_hxg(u32 *msg) +{ + return msg + GUC_CTB_MSG_MIN_LEN; +} + +static u32 msg_len_to_hxg_len(u32 len) +{ + return len - GUC_CTB_MSG_MIN_LEN; +} + +static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + u32 *hxg = msg_to_hxg(msg); + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + + lockdep_assert_held(&ct->lock); + + switch (action) { + case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: + case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: + case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: + case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: + g2h_release_space(ct, len); + } + + return 0; +} + +static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) +{ + struct xe_gt *gt = ct_to_gt(ct); + + if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) + xe_gt_err(gt, "GuC Crash dump notification\n"); + else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) + xe_gt_err(gt, "GuC Exception notification\n"); + else + xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); + + CT_DEAD(ct, NULL, CRASH); + + kick_reset(ct); + + return 0; +} + +static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_gt *gt = ct_to_gt(ct); + u32 *hxg = msg_to_hxg(msg); + u32 hxg_len = msg_len_to_hxg_len(len); + u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); + u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); + struct g2h_fence *g2h_fence; + + lockdep_assert_held(&ct->lock); + + /* + * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. + * Those messages should never fail, so if we do get an error back it + * means we're likely doing an illegal operation and the GuC is + * rejecting it. We have no way to inform the code that submitted the + * H2G that the message was rejected, so we need to escalate the + * failure to trigger a reset. + */ + if (fence & CT_SEQNO_UNTRACKED) { + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) + xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", + fence, + FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), + FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); + else + xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", + type, fence); + + fast_req_report(ct, fence); + + /* FIXME: W/A race in the GuC, will get in firmware soon */ + if (xe_gt_recovery_pending(gt)) + return 0; + + CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); + + return -EPROTO; + } + + /* don't erase as we still expect a final response with the same fence */ + if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) + g2h_fence = xa_load(&ct->fence_lookup, fence); + else + g2h_fence = xa_erase(&ct->fence_lookup, fence); + + if (unlikely(!g2h_fence)) { + /* Don't tear down channel, as send could've timed out */ + /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ + xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + return 0; + } + + xe_gt_assert(gt, fence == g2h_fence->seqno); + + /* + * reinit as we might have already process this g2h_fence before + * if we received a NO_RESPONSE_BUSY reply + */ + g2h_fence_reinit(g2h_fence); + + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { + g2h_fence->fail = true; + g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); + g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); + } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { + g2h_fence->retry = true; + g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); + } else if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) { + g2h_fence->wait = true; + g2h_fence->counter = FIELD_GET(GUC_HXG_BUSY_MSG_0_COUNTER, hxg[0]); + } else if (g2h_fence->response_buffer) { + g2h_fence->response_len = hxg_len; + memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); + } else { + g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); + } + + /* don't release any space if it was an intermediate message */ + if (!g2h_fence->wait) + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ + WRITE_ONCE(g2h_fence->done, true); + smp_mb(); + + wake_up_all(&ct->g2h_fence_wq); + + return 0; +} + +static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_gt *gt = ct_to_gt(ct); + u32 *hxg = msg_to_hxg(msg); + u32 origin, type; + int ret; + + lockdep_assert_held(&ct->lock); + + origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); + if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { + xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", + origin); + CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); + + return -EPROTO; + } + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); + switch (type) { + case GUC_HXG_TYPE_EVENT: + ret = parse_g2h_event(ct, msg, len); + break; + case GUC_HXG_TYPE_RESPONSE_SUCCESS: + case GUC_HXG_TYPE_RESPONSE_FAILURE: + case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: + ret = parse_g2h_response(ct, msg, len); + break; + default: + xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", + type); + CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); + + ret = -EOPNOTSUPP; + } + + return ret; +} + +static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_guc *guc = ct_to_guc(ct); + struct xe_gt *gt = ct_to_gt(ct); + u32 hxg_len = msg_len_to_hxg_len(len); + u32 *hxg = msg_to_hxg(msg); + u32 action, adj_len; + u32 *payload; + int ret = 0; + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) + return 0; + + action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; + adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; + + switch (action) { + case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: + ret = xe_guc_sched_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: + ret = xe_guc_deregister_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: + ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: + ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, + adj_len); + break; + case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: + /* Selftest only at the moment */ + break; + case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: + ret = xe_guc_error_capture_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: + /* FIXME: Handle this */ + break; + case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: + ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, + adj_len); + break; + case XE_GUC_ACTION_NOTIFY_UNCORRECTABLE_LOCAL_ERROR: + ret = xe_guc_uncorrectable_error_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: + ret = xe_guc_pagefault_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: + ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: + ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); + break; + case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: + ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); + break; + case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: + ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); + break; + case GUC_ACTION_GUC2PF_ADVERSE_EVENT: + ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); + break; + case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: + case XE_GUC_ACTION_NOTIFY_EXCEPTION: + ret = guc_crash_process_msg(ct, action); + break; +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + case XE_GUC_ACTION_TEST_G2G_RECV: + ret = xe_guc_g2g_test_notification(guc, payload, adj_len); + break; +#endif + case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: + ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR: + ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload, + adj_len); + break; + default: + xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); + } + + if (ret) { + xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", + action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); + CT_DEAD(ct, NULL, PROCESS_FAILED); + } + + return 0; +} + +static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + struct guc_ctb *g2h = &ct->ctbs.g2h; + u32 tail, head, len, desc_status; + s32 avail; + u32 action; + u32 *hxg; + + xe_gt_assert(gt, xe_guc_ct_initialized(ct)); + lockdep_assert_held(&ct->fast_lock); + + if (xe_device_wedged(xe)) + return -ENOTRECOVERABLE; + + if (ct->state == XE_GUC_CT_STATE_DISABLED) + return -ENODEV; + + if (ct->state == XE_GUC_CT_STATE_STOPPED) + return -ECANCELED; + + if (g2h->info.broken) + return -EPIPE; + + xe_gt_assert(gt, xe_guc_ct_enabled(ct)); + + desc_status = desc_read(xe, g2h, status); + if (desc_status) { + if (desc_status & GUC_CTB_STATUS_DISABLED) { + /* + * Potentially valid if a CLIENT_RESET request resulted in + * contexts/engines being reset. But should never happen as + * no contexts should be active when CLIENT_RESET is sent. + */ + xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); + desc_status &= ~GUC_CTB_STATUS_DISABLED; + } + + if (desc_status) { + xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); + goto corrupted; + } + } + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + u32 desc_tail = desc_read(xe, g2h, tail); + /* + u32 desc_head = desc_read(xe, g2h, head); + + * info.head and desc_head are updated back-to-back at the end of + * this function and nowhere else. Hence, they cannot be different + * unless two g2h_read calls are running concurrently. Which is not + * possible because it is guarded by ct->fast_lock. And yet, some + * discrete platforms are regularly hitting this error :(. + * + * desc_head rolling backwards shouldn't cause any noticeable + * problems - just a delay in GuC being allowed to proceed past that + * point in the queue. So for now, just disable the error until it + * can be root caused. + * + if (g2h->info.head != desc_head) { + desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); + xe_gt_err(gt, "CT read: head was modified %u != %u\n", + desc_head, g2h->info.head); + goto corrupted; + } + */ + + if (g2h->info.head > g2h->info.size) { + desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); + xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", + g2h->info.head, g2h->info.size); + goto corrupted; + } + + if (desc_tail >= g2h->info.size) { + desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); + xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", + desc_tail, g2h->info.size); + goto corrupted; + } + } + + /* Calculate DW available to read */ + tail = desc_read(xe, g2h, tail); + avail = tail - g2h->info.head; + if (unlikely(avail == 0)) + return 0; + + if (avail < 0) + avail += g2h->info.size; + + /* Read header */ + xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, + sizeof(u32)); + len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; + if (len > avail) { + xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", + avail, len); + goto corrupted; + } + + head = (g2h->info.head + 1) % g2h->info.size; + avail = len - 1; + + /* Read G2H message */ + if (avail + head > g2h->info.size) { + u32 avail_til_wrap = g2h->info.size - head; + + xe_map_memcpy_from(xe, msg + 1, + &g2h->cmds, sizeof(u32) * head, + avail_til_wrap * sizeof(u32)); + xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, + &g2h->cmds, 0, + (avail - avail_til_wrap) * sizeof(u32)); + } else { + xe_map_memcpy_from(xe, msg + 1, + &g2h->cmds, sizeof(u32) * head, + avail * sizeof(u32)); + } + + hxg = msg_to_hxg(msg); + action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + + if (fast_path) { + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) + return 0; + + switch (action) { + case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: + break; /* Process these in fast-path */ + default: + return 0; + } + } + + /* Update local / descriptor header */ + g2h->info.head = (head + avail) % g2h->info.size; + desc_write(xe, g2h, head, g2h->info.head); + + trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, + action, len, g2h->info.head, tail); + + return len; + +corrupted: + CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); + return -EPROTO; +} + +static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_gt *gt = ct_to_gt(ct); + struct xe_guc *guc = ct_to_guc(ct); + u32 hxg_len = msg_len_to_hxg_len(len); + u32 *hxg = msg_to_hxg(msg); + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; + u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; + int ret = 0; + + switch (action) { + case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: + ret = xe_guc_pagefault_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + __g2h_release_space(ct, len); + ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: + __g2h_release_space(ct, len); + ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len); + break; + default: + xe_gt_warn(gt, "NOT_POSSIBLE\n"); + } + + if (ret) { + xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", + action, ERR_PTR(ret)); + CT_DEAD(ct, NULL, FAST_G2H); + } +} + +/** + * xe_guc_ct_fast_path - process critical G2H in the IRQ handler + * @ct: GuC CT object + * + * Anything related to page faults is critical for performance, process these + * critical G2H in the IRQ. This is safe as these handlers either just wake up + * waiters or queue another worker. + */ +void xe_guc_ct_fast_path(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + bool ongoing; + int len; + + ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); + if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) + return; + + spin_lock(&ct->fast_lock); + do { + len = g2h_read(ct, ct->fast_msg, true); + if (len > 0) + g2h_fast_path(ct, ct->fast_msg, len); + } while (len > 0); + spin_unlock(&ct->fast_lock); + + if (ongoing) + xe_pm_runtime_put(xe); +} + +/* Returns less than zero on error, 0 on done, 1 on more available */ +static int dequeue_one_g2h(struct xe_guc_ct *ct) +{ + int len; + int ret; + + lockdep_assert_held(&ct->lock); + + spin_lock_irq(&ct->fast_lock); + len = g2h_read(ct, ct->msg, false); + spin_unlock_irq(&ct->fast_lock); + if (len <= 0) + return len; + + ret = parse_g2h_msg(ct, ct->msg, len); + if (unlikely(ret < 0)) + return ret; + + ret = process_g2h_msg(ct, ct->msg, len); + if (unlikely(ret < 0)) + return ret; + + return 1; +} + +static void receive_g2h(struct xe_guc_ct *ct) +{ + bool ongoing; + int ret; + + /* + * Normal users must always hold mem_access.ref around CT calls. However + * during the runtime pm callbacks we rely on CT to talk to the GuC, but + * at this stage we can't rely on mem_access.ref and even the + * callback_task will be different than current. For such cases we just + * need to ensure we always process the responses from any blocking + * ct_send requests or where we otherwise expect some response when + * initiated from those callbacks (which will need to wait for the below + * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if + * the device has suspended to the point that the CT communication has + * been disabled. + * + * If we are inside the runtime pm callback, we can be the only task + * still issuing CT requests (since that requires having the + * mem_access.ref). It seems like it might in theory be possible to + * receive unsolicited events from the GuC just as we are + * suspending-resuming, but those will currently anyway be lost when + * eventually exiting from suspend, hence no need to wake up the device + * here. If we ever need something stronger than get_if_ongoing() then + * we need to be careful with blocking the pm callbacks from getting CT + * responses, if the worker here is blocked on those callbacks + * completing, creating a deadlock. + */ + ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); + if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) + return; + + do { + mutex_lock(&ct->lock); + ret = dequeue_one_g2h(ct); + mutex_unlock(&ct->lock); + + if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { + xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d\n", ret); + CT_DEAD(ct, NULL, G2H_RECV); + kick_reset(ct); + } + } while (ret == 1); + + if (ongoing) + xe_pm_runtime_put(ct_to_xe(ct)); +} + +static void g2h_worker_func(struct work_struct *w) +{ + struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); + + receive_g2h(ct); +} + +static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, + bool want_ctb) +{ + struct xe_guc_ct_snapshot *snapshot; + + snapshot = kzalloc_obj(*snapshot, atomic ? GFP_ATOMIC : GFP_KERNEL); + if (!snapshot) + return NULL; + + if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && want_ctb) { + snapshot->ctb_size = xe_bo_size(ct->ctbs.h2g.bo) + + xe_bo_size(ct->ctbs.g2h.bo); + snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); + } + + return snapshot; +} + +static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, + struct guc_ctb_snapshot *snapshot) +{ + xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, + sizeof(struct guc_ct_buffer_desc)); + memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); +} + +static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, + struct drm_printer *p) +{ + drm_printf(p, "\tsize: %d\n", snapshot->info.size); + drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); + drm_printf(p, "\thead: %d\n", snapshot->info.head); + drm_printf(p, "\ttail: %d\n", snapshot->info.tail); + drm_printf(p, "\tspace: %d\n", snapshot->info.space); + drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); + drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); + drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); + drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); +} + +static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, + bool want_ctb) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_guc_ct_snapshot *snapshot; + + snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); + if (!snapshot) { + xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); + return NULL; + } + + if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { + snapshot->ct_enabled = true; + snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); + guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); + guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); + } + + if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && snapshot->ctb) { + xe_map_memcpy_from(xe, snapshot->ctb, &ct->ctbs.h2g.bo->vmap, 0, + xe_bo_size(ct->ctbs.h2g.bo)); + xe_map_memcpy_from(xe, snapshot->ctb + xe_bo_size(ct->ctbs.h2g.bo), + &ct->ctbs.g2h.bo->vmap, 0, + xe_bo_size(ct->ctbs.g2h.bo)); + } + + return snapshot; +} + +/** + * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. + * @ct: GuC CT object. + * + * This can be printed out in a later stage like during dev_coredump + * analysis. This is safe to be called during atomic context. + * + * Returns: a GuC CT snapshot object that must be freed by the caller + * by using `xe_guc_ct_snapshot_free`. + */ +struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) +{ + return guc_ct_snapshot_capture(ct, true, true); +} + +/** + * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. + * @snapshot: GuC CT snapshot object. + * @p: drm_printer where it will be printed out. + * + * This function prints out a given GuC CT snapshot object. + */ +void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, + struct drm_printer *p) +{ + if (!snapshot) + return; + + if (snapshot->ct_enabled) { + drm_puts(p, "H2G CTB (all sizes in DW):\n"); + guc_ctb_snapshot_print(&snapshot->h2g, p); + + drm_puts(p, "G2H CTB (all sizes in DW):\n"); + guc_ctb_snapshot_print(&snapshot->g2h, p); + drm_printf(p, "\tg2h outstanding: %d\n", + snapshot->g2h_outstanding); + + if (snapshot->ctb) { + drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); + xe_print_blob_ascii85(p, "[CTB].data", '\n', + snapshot->ctb, 0, snapshot->ctb_size); + } + } else { + drm_puts(p, "CT disabled\n"); + } +} + +/** + * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. + * @snapshot: GuC CT snapshot object. + * + * This function free all the memory that needed to be allocated at capture + * time. + */ +void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) +{ + if (!snapshot) + return; + + kfree(snapshot->ctb); + kfree(snapshot); +} + +/** + * xe_guc_ct_print - GuC CT Print. + * @ct: GuC CT. + * @p: drm_printer where it will be printed out. + * @want_ctb: Should the full CTB content be dumped (vs just the headers) + * + * This function will quickly capture a snapshot of the CT state + * and immediately print it out. + */ +void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) +{ + struct xe_guc_ct_snapshot *snapshot; + + snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); + xe_guc_ct_snapshot_print(snapshot, p); + xe_guc_ct_snapshot_free(snapshot); +} + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) + +#ifdef CONFIG_FUNCTION_ERROR_INJECTION +/* + * This is a helper function which assists the driver in identifying if a fault + * injection test is currently active, allowing it to reduce unnecessary debug + * output. Typically, the function returns zero, but the fault injection + * framework can alter this to return an error. Since faults are injected + * through this function, it's important to ensure the compiler doesn't optimize + * it into an inline function. To avoid such optimization, the 'noinline' + * attribute is applied. Compiler optimizes the static function defined in the + * header file as an inline function. + */ +noinline int xe_is_injection_active(void) { return 0; } +ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO); +#else +int xe_is_injection_active(void) { return 0; } +#endif + +static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) +{ + struct xe_guc_log_snapshot *snapshot_log; + struct xe_guc_ct_snapshot *snapshot_ct; + struct xe_guc *guc = ct_to_guc(ct); + unsigned long flags; + bool have_capture; + + if (ctb) + ctb->info.broken = true; + /* + * Huge dump is getting generated when injecting error for guc CT/MMIO + * functions. So, let us suppress the dump when fault is injected. + */ + if (xe_is_injection_active()) + return; + + /* Ignore further errors after the first dump until a reset */ + if (ct->dead.reported) + return; + + spin_lock_irqsave(&ct->dead.lock, flags); + + /* And only capture one dump at a time */ + have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); + ct->dead.reason |= (1 << reason_code) | + (1 << CT_DEAD_STATE_CAPTURE); + + spin_unlock_irqrestore(&ct->dead.lock, flags); + + if (have_capture) + return; + + snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); + snapshot_ct = xe_guc_ct_snapshot_capture((ct)); + + spin_lock_irqsave(&ct->dead.lock, flags); + + if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { + xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); + xe_guc_log_snapshot_free(snapshot_log); + xe_guc_ct_snapshot_free(snapshot_ct); + } else { + ct->dead.snapshot_log = snapshot_log; + ct->dead.snapshot_ct = snapshot_ct; + } + + spin_unlock_irqrestore(&ct->dead.lock, flags); + + queue_work(system_dfl_wq, &(ct)->dead.worker); +} + +static void ct_dead_print(struct xe_dead_ct *dead) +{ + struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + static int g_count; + struct drm_printer ip = xe_gt_info_printer(gt); + struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); + + if (!dead->reason) { + xe_gt_err(gt, "CTB is dead for no reason!?\n"); + return; + } + + /* Can't generate a genuine core dump at this point, so just do the good bits */ + drm_puts(&lp, "**** Xe Device Coredump ****\n"); + drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); + xe_device_snapshot_print(xe, &lp); + + drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); + drm_printf(&lp, "\tTile: %d\n", gt->tile->id); + + drm_puts(&lp, "**** GuC Log ****\n"); + xe_guc_log_snapshot_print(dead->snapshot_log, &lp); + + drm_puts(&lp, "**** GuC CT ****\n"); + xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); + + drm_puts(&lp, "Done.\n"); +} + +static void ct_dead_worker_func(struct work_struct *w) +{ + struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); + + if (!ct->dead.reported) { + ct->dead.reported = true; + ct_dead_print(&ct->dead); + } + + spin_lock_irq(&ct->dead.lock); + + xe_guc_log_snapshot_free(ct->dead.snapshot_log); + ct->dead.snapshot_log = NULL; + xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); + ct->dead.snapshot_ct = NULL; + + if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { + /* A reset has occurred so re-arm the error reporting */ + ct->dead.reason = 0; + ct->dead.reported = false; + } + + spin_unlock_irq(&ct->dead.lock); +} +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h new file mode 100644 index 000000000..767365a33 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ct.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_CT_H_ +#define _XE_GUC_CT_H_ + +#include "xe_guc_ct_types.h" + +struct drm_printer; +struct xe_device; + +int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct); +int xe_guc_ct_init(struct xe_guc_ct *ct); +int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct); +int xe_guc_ct_enable(struct xe_guc_ct *ct); +int xe_guc_ct_restart(struct xe_guc_ct *ct); +void xe_guc_ct_disable(struct xe_guc_ct *ct); +void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct); +void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct); +void xe_guc_ct_stop(struct xe_guc_ct *ct); +void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct); +void xe_guc_ct_fast_path(struct xe_guc_ct *ct); + +struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct); +void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, struct drm_printer *p); +void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot); +void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb); + +static inline bool xe_guc_ct_initialized(struct xe_guc_ct *ct) +{ + /* READ_ONCE pairs with WRITE_ONCE in guc_ct_change_state. */ + return READ_ONCE(ct->state) != XE_GUC_CT_STATE_NOT_INITIALIZED; +} + +static inline bool xe_guc_ct_enabled(struct xe_guc_ct *ct) +{ + /* READ_ONCE pairs with WRITE_ONCE in guc_ct_change_state. */ + return READ_ONCE(ct->state) == XE_GUC_CT_STATE_ENABLED; +} + +static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct) +{ + if (!xe_guc_ct_enabled(ct)) + return; + + wake_up_all(&ct->wq); + queue_work(ct->g2h_wq, &ct->g2h_worker); + xe_guc_ct_fast_path(ct); +} + +/* Basic CT send / receives */ +int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h); +int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h); +int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 *response_buffer); +static inline int +xe_guc_ct_send_block(struct xe_guc_ct *ct, const u32 *action, u32 len) +{ + return xe_guc_ct_send_recv(ct, action, len, NULL); +} + +/* This is only version of the send CT you can call from a G2H handler */ +int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, + u32 len); + +/* Can't fail because a GT reset is in progress */ +int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, + u32 len, u32 *response_buffer); +static inline int +xe_guc_ct_send_block_no_fail(struct xe_guc_ct *ct, const u32 *action, u32 len) +{ + return xe_guc_ct_send_recv_no_fail(ct, action, len, NULL); +} + +long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct); + +/** + * xe_guc_ct_wake_waiters() - GuC CT wake up waiters + * @ct: GuC CT object + */ +static inline void xe_guc_ct_wake_waiters(struct xe_guc_ct *ct) +{ + wake_up_all(&ct->wq); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h new file mode 100644 index 000000000..f88e588af --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_CT_TYPES_H_ +#define _XE_GUC_CT_TYPES_H_ + +#include +#include +#include +#include +#include +#include + +#include "abi/guc_communication_ctb_abi.h" + +struct xe_bo; + +/** + * struct guc_ctb_info - GuC command transport buffer (CTB) info + */ +struct guc_ctb_info { + /** @size: size of CTB commands (DW) */ + u32 size; + /** @resv_space: reserved space of CTB commands (DW) */ + u32 resv_space; + /** @head: head of CTB commands (DW) */ + u32 head; + /** @tail: tail of CTB commands (DW) */ + u32 tail; + /** @space: space in CTB commands (DW) */ + u32 space; + /** @broken: channel broken */ + bool broken; +}; + +/** + * struct guc_ctb - GuC command transport buffer (CTB) + */ +struct guc_ctb { + /** @bo: Xe BO for CTB */ + struct xe_bo *bo; + /** @desc: dma buffer map for CTB descriptor */ + struct iosys_map desc; + /** @cmds: dma buffer map for CTB commands */ + struct iosys_map cmds; + /** @info: CTB info */ + struct guc_ctb_info info; +}; + +/** + * struct guc_ctb_snapshot - GuC command transport buffer (CTB) snapshot + */ +struct guc_ctb_snapshot { + /** @desc: snapshot of the CTB descriptor */ + struct guc_ct_buffer_desc desc; + /** @info: snapshot of the CTB info */ + struct guc_ctb_info info; +}; + +/** + * struct xe_guc_ct_snapshot - GuC command transport (CT) snapshot + */ +struct xe_guc_ct_snapshot { + /** @ct_enabled: CT enabled info at capture time. */ + bool ct_enabled; + /** @g2h_outstanding: G2H outstanding info at the capture time */ + u32 g2h_outstanding; + /** @g2h: G2H CTB snapshot */ + struct guc_ctb_snapshot g2h; + /** @h2g: H2G CTB snapshot */ + struct guc_ctb_snapshot h2g; + /** @ctb_size: size of the snapshot of the CTB */ + size_t ctb_size; + /** @ctb: snapshot of the entire CTB */ + void *ctb; +}; + +/** + * enum xe_guc_ct_state - CT state + * @XE_GUC_CT_STATE_NOT_INITIALIZED: CT not initialized, messages not expected in this state + * @XE_GUC_CT_STATE_DISABLED: CT disabled, messages not expected in this state + * @XE_GUC_CT_STATE_STOPPED: CT stopped, drop messages without errors + * @XE_GUC_CT_STATE_ENABLED: CT enabled, messages sent / received in this state + */ +enum xe_guc_ct_state { + XE_GUC_CT_STATE_NOT_INITIALIZED = 0, + XE_GUC_CT_STATE_DISABLED, + XE_GUC_CT_STATE_STOPPED, + XE_GUC_CT_STATE_ENABLED, +}; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +/** struct xe_dead_ct - Information for debugging a dead CT */ +struct xe_dead_ct { + /** @lock: protects memory allocation/free operations, and @reason updates */ + spinlock_t lock; + /** @reason: bit mask of CT_DEAD_* reason codes */ + unsigned int reason; + /** @reported: for preventing multiple dumps per error sequence */ + bool reported; + /** @worker: worker thread to get out of interrupt context before dumping */ + struct work_struct worker; + /** @snapshot_ct: copy of CT state and CTB content at point of error */ + struct xe_guc_ct_snapshot *snapshot_ct; + /** @snapshot_log: copy of GuC log at point of error */ + struct xe_guc_log_snapshot *snapshot_log; +}; + +/** struct xe_fast_req_fence - Used to track FAST_REQ messages by fence to match error responses */ +struct xe_fast_req_fence { + /** @fence: sequence number sent in H2G and return in G2H error */ + u16 fence; + /** @action: H2G action code */ + u16 action; +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + /** @stack: call stack from when the H2G was sent */ + depot_stack_handle_t stack; +#endif +}; +#endif + +/** + * struct xe_guc_ct - GuC command transport (CT) layer + * + * Includes a pair of CT buffers for bi-directional communication and tracking + * for the H2G and G2H requests sent and received through the buffers. + */ +struct xe_guc_ct { + /** @lock: protects everything in CT layer */ + struct mutex lock; + /** @fast_lock: protects G2H channel and credits */ + spinlock_t fast_lock; + /** @ctbs: buffers for sending and receiving commands */ + struct { + /** @ctbs.h2g: Host to GuC (H2G, send) channel */ + struct guc_ctb h2g; + /** @ctbs.g2h: GuC to Host (G2H, receive) channel */ + struct guc_ctb g2h; + } ctbs; + /** @g2h_outstanding: number of outstanding G2H */ + u32 g2h_outstanding; + /** @g2h_worker: worker to process G2H messages */ + struct work_struct g2h_worker; + /** @safe_mode_worker: worker to check G2H messages with IRQ disabled */ + struct delayed_work safe_mode_worker; + /** @state: CT state */ + enum xe_guc_ct_state state; + /** @fence_seqno: G2H fence seqno - 16 bits used by CT */ + u32 fence_seqno; + /** @fence_lookup: G2H fence lookup */ + struct xarray fence_lookup; + /** @wq: wait queue used for reliable CT sends and freeing G2H credits */ + wait_queue_head_t wq; + /** @g2h_fence_wq: wait queue used for G2H fencing */ + wait_queue_head_t g2h_fence_wq; + /** @g2h_wq: used to process G2H */ + struct workqueue_struct *g2h_wq; + /** @msg: Message buffer */ + u32 msg[GUC_CTB_MSG_MAX_LEN]; + /** @fast_msg: Message buffer */ + u32 fast_msg[GUC_CTB_MSG_MAX_LEN]; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) + /** @dead: information for debugging dead CTs */ + struct xe_dead_ct dead; + /** @fast_req: history of FAST_REQ messages for matching with G2H error responses */ + struct xe_fast_req_fence fast_req[SZ_32]; +#endif +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_db_mgr.c b/drivers/gpu/drm/xe/xe_guc_db_mgr.c new file mode 100644 index 000000000..6767e8076 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_db_mgr.c @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include + +#include + +#include "regs/xe_guc_regs.h" + +#include "xe_assert.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_db_mgr.h" +#include "xe_guc_types.h" + +/** + * DOC: GuC Doorbells + * + * The GFX doorbell solution provides a mechanism for submission of workload + * to the graphics hardware by a ring3 application without the penalty of + * ring transition for each workload submission. + * + * In SR-IOV mode, the doorbells are treated as shared resource and PF must + * be able to provision exclusive range of IDs across VFs, which may want to + * use this feature. + */ + +static struct xe_guc *dbm_to_guc(struct xe_guc_db_mgr *dbm) +{ + return container_of(dbm, struct xe_guc, dbm); +} + +static struct xe_gt *dbm_to_gt(struct xe_guc_db_mgr *dbm) +{ + return guc_to_gt(dbm_to_guc(dbm)); +} + +static struct xe_device *dbm_to_xe(struct xe_guc_db_mgr *dbm) +{ + return gt_to_xe(dbm_to_gt(dbm)); +} + +#define dbm_assert(_dbm, _cond) xe_gt_assert(dbm_to_gt(_dbm), _cond) +#define dbm_mutex(_dbm) (&dbm_to_guc(_dbm)->submission_state.lock) + +static void dbm_print_locked(struct xe_guc_db_mgr *dbm, struct drm_printer *p, int indent); + +static void __fini_dbm(struct drm_device *drm, void *arg) +{ + struct xe_guc_db_mgr *dbm = arg; + unsigned int weight; + + mutex_lock(dbm_mutex(dbm)); + + weight = bitmap_weight(dbm->bitmap, dbm->count); + if (weight) { + struct drm_printer p = xe_gt_info_printer(dbm_to_gt(dbm)); + + xe_gt_err(dbm_to_gt(dbm), "GuC doorbells manager unclean (%u/%u)\n", + weight, dbm->count); + dbm_print_locked(dbm, &p, 1); + } + + bitmap_free(dbm->bitmap); + dbm->bitmap = NULL; + dbm->count = 0; + + mutex_unlock(dbm_mutex(dbm)); +} + +/** + * xe_guc_db_mgr_init() - Initialize GuC Doorbells Manager. + * @dbm: the &xe_guc_db_mgr to initialize + * @count: number of doorbells to manage + * + * The bare-metal or PF driver can pass ~0 as &count to indicate that all + * doorbells supported by the hardware are available for use. + * + * Only VF's drivers will have to provide explicit number of doorbells IDs + * that they can use. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_db_mgr_init(struct xe_guc_db_mgr *dbm, unsigned int count) +{ + int ret; + + if (count == ~0) + count = GUC_NUM_DOORBELLS; + + dbm_assert(dbm, !dbm->bitmap); + dbm_assert(dbm, count <= GUC_NUM_DOORBELLS); + + if (!count) + goto done; + + dbm->bitmap = bitmap_zalloc(count, GFP_KERNEL); + if (!dbm->bitmap) + return -ENOMEM; + dbm->count = count; + + ret = drmm_add_action_or_reset(&dbm_to_xe(dbm)->drm, __fini_dbm, dbm); + if (ret) + return ret; +done: + xe_gt_dbg(dbm_to_gt(dbm), "using %u doorbell%s\n", + dbm->count, str_plural(dbm->count)); + return 0; +} + +static int dbm_reserve_chunk_locked(struct xe_guc_db_mgr *dbm, + unsigned int count, unsigned int spare) +{ + unsigned int used; + int index; + + dbm_assert(dbm, count); + dbm_assert(dbm, count <= GUC_NUM_DOORBELLS); + dbm_assert(dbm, dbm->count <= GUC_NUM_DOORBELLS); + lockdep_assert_held(dbm_mutex(dbm)); + + if (!dbm->count) + return -ENODATA; + + if (spare) { + used = bitmap_weight(dbm->bitmap, dbm->count); + if (used + count + spare > dbm->count) + return -EDQUOT; + } + + index = bitmap_find_next_zero_area(dbm->bitmap, dbm->count, 0, count, 0); + if (index >= dbm->count) + return -ENOSPC; + + bitmap_set(dbm->bitmap, index, count); + + return index; +} + +static void dbm_release_chunk_locked(struct xe_guc_db_mgr *dbm, + unsigned int start, unsigned int count) +{ + dbm_assert(dbm, count); + dbm_assert(dbm, count <= GUC_NUM_DOORBELLS); + dbm_assert(dbm, dbm->count); + dbm_assert(dbm, dbm->count <= GUC_NUM_DOORBELLS); + lockdep_assert_held(dbm_mutex(dbm)); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + unsigned int n; + + for (n = 0; n < count; n++) + dbm_assert(dbm, test_bit(start + n, dbm->bitmap)); + } + bitmap_clear(dbm->bitmap, start, count); +} + +/** + * xe_guc_db_mgr_reserve_id_locked() - Reserve a single GuC Doorbell ID. + * @dbm: the &xe_guc_db_mgr + * + * This function expects that submission lock is already taken. + * + * Return: ID of the allocated GuC doorbell or a negative error code on failure. + */ +int xe_guc_db_mgr_reserve_id_locked(struct xe_guc_db_mgr *dbm) +{ + return dbm_reserve_chunk_locked(dbm, 1, 0); +} + +/** + * xe_guc_db_mgr_release_id_locked() - Release a single GuC Doorbell ID. + * @dbm: the &xe_guc_db_mgr + * @id: the GuC Doorbell ID to release + * + * This function expects that submission lock is already taken. + */ +void xe_guc_db_mgr_release_id_locked(struct xe_guc_db_mgr *dbm, unsigned int id) +{ + return dbm_release_chunk_locked(dbm, id, 1); +} + +/** + * xe_guc_db_mgr_reserve_range() - Reserve a range of GuC Doorbell IDs. + * @dbm: the &xe_guc_db_mgr + * @count: number of GuC doorbell IDs to reserve + * @spare: number of GuC doorbell IDs to keep available + * + * This function is dedicated for the for use by the PF which expects that + * allocated range for the VF will be contiguous and that there will be at + * least &spare IDs still available for the PF use after this reservation. + * + * Return: starting ID of the allocated GuC doorbell ID range or + * a negative error code on failure. + */ +int xe_guc_db_mgr_reserve_range(struct xe_guc_db_mgr *dbm, + unsigned int count, unsigned int spare) +{ + int ret; + + mutex_lock(dbm_mutex(dbm)); + ret = dbm_reserve_chunk_locked(dbm, count, spare); + mutex_unlock(dbm_mutex(dbm)); + + return ret; +} + +/** + * xe_guc_db_mgr_release_range() - Release a range of Doorbell IDs. + * @dbm: the &xe_guc_db_mgr + * @start: the starting ID of GuC doorbell ID range to release + * @count: number of GuC doorbell IDs to release + */ +void xe_guc_db_mgr_release_range(struct xe_guc_db_mgr *dbm, + unsigned int start, unsigned int count) +{ + mutex_lock(dbm_mutex(dbm)); + dbm_release_chunk_locked(dbm, start, count); + mutex_unlock(dbm_mutex(dbm)); +} + +static void dbm_print_locked(struct xe_guc_db_mgr *dbm, struct drm_printer *p, int indent) +{ + unsigned int rs, re; + unsigned int total; + + drm_printf_indent(p, indent, "count: %u\n", dbm->count); + if (!dbm->bitmap) + return; + + total = 0; + for_each_clear_bitrange(rs, re, dbm->bitmap, dbm->count) { + drm_printf_indent(p, indent, "available range: %u..%u (%u)\n", + rs, re - 1, re - rs); + total += re - rs; + } + drm_printf_indent(p, indent, "available total: %u\n", total); + + total = 0; + for_each_set_bitrange(rs, re, dbm->bitmap, dbm->count) { + drm_printf_indent(p, indent, "reserved range: %u..%u (%u)\n", + rs, re - 1, re - rs); + total += re - rs; + } + drm_printf_indent(p, indent, "reserved total: %u\n", total); +} + +/** + * xe_guc_db_mgr_print() - Print status of GuC Doorbells Manager. + * @dbm: the &xe_guc_db_mgr to print + * @p: the &drm_printer to print to + * @indent: tab indentation level + */ +void xe_guc_db_mgr_print(struct xe_guc_db_mgr *dbm, + struct drm_printer *p, int indent) +{ + mutex_lock(dbm_mutex(dbm)); + dbm_print_locked(dbm, p, indent); + mutex_unlock(dbm_mutex(dbm)); +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_guc_db_mgr_test.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_db_mgr.h b/drivers/gpu/drm/xe/xe_guc_db_mgr.h new file mode 100644 index 000000000..c250fa0ca --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_db_mgr.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GUC_DB_MGR_H_ +#define _XE_GUC_DB_MGR_H_ + +struct drm_printer; +struct xe_guc_db_mgr; + +int xe_guc_db_mgr_init(struct xe_guc_db_mgr *dbm, unsigned int count); + +int xe_guc_db_mgr_reserve_id_locked(struct xe_guc_db_mgr *dbm); +void xe_guc_db_mgr_release_id_locked(struct xe_guc_db_mgr *dbm, unsigned int id); + +int xe_guc_db_mgr_reserve_range(struct xe_guc_db_mgr *dbm, unsigned int count, unsigned int spare); +void xe_guc_db_mgr_release_range(struct xe_guc_db_mgr *dbm, unsigned int start, unsigned int count); + +void xe_guc_db_mgr_print(struct xe_guc_db_mgr *dbm, struct drm_printer *p, int indent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.c b/drivers/gpu/drm/xe/xe_guc_debugfs.c new file mode 100644 index 000000000..2f2311968 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_debugfs.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_debugfs.h" + +#include +#include + +#include "xe_device_types.h" +#include "xe_gt_types.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_log.h" +#include "xe_guc_pc.h" +#include "xe_pm.h" + +/* + * guc_debugfs_show - A show callback for struct drm_info_list + * @m: the &seq_file + * @data: data used by the drm debugfs helpers + * + * This callback can be used in struct drm_info_list to describe debugfs + * files that are &xe_guc specific in similar way how we handle &xe_gt + * specific files using &xe_gt_debugfs_simple_show. + * + * It is assumed that those debugfs files will be created on directory entry + * which grandparent struct dentry d_inode->i_private points to &xe_gt. + * + * /sys/kernel/debug/dri/0/ + * ├── gt0 # dent->d_parent->d_parent (d_inode->i_private == gt) + * │   ├── uc # dent->d_parent + * │   │   ├── guc_info # dent + * │   │   ├── guc_... + * + * This function assumes that &m->private will be set to the &struct + * drm_info_node corresponding to the instance of the info on a given &struct + * drm_minor (see struct drm_info_list.show for details). + * + * This function also assumes that struct drm_info_list.data will point to the + * function code that will actually print a file content:: + * + * int (*print)(struct xe_guc *, struct drm_printer *) + * + * Example:: + * + * int foo(struct xe_guc *guc, struct drm_printer *p) + * { + * drm_printf(p, "enabled %d\n", guc->submission_state.enabled); + * return 0; + * } + * + * static const struct drm_info_list bar[] = { + * { name = "foo", .show = guc_debugfs_show, .data = foo }, + * }; + * + * parent = debugfs_create_dir("uc", gtdir); + * drm_debugfs_create_files(bar, ARRAY_SIZE(bar), parent, minor); + * + * Return: 0 on success or a negative error code on failure. + */ +static int guc_debugfs_show(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + struct drm_info_node *node = m->private; + struct dentry *parent = node->dent->d_parent; + struct dentry *grandparent = parent->d_parent; + struct xe_gt *gt = grandparent->d_inode->i_private; + struct xe_device *xe = gt_to_xe(gt); + int (*print)(struct xe_guc *, struct drm_printer *) = node->info_ent->data; + + guard(xe_pm_runtime)(xe); + return print(>->uc.guc, &p); +} + +static int guc_log(struct xe_guc *guc, struct drm_printer *p) +{ + xe_guc_log_print(&guc->log, p); + return 0; +} + +static int guc_log_lfd(struct xe_guc *guc, struct drm_printer *p) +{ + xe_guc_log_print_lfd(&guc->log, p); + return 0; +} + +static int guc_log_dmesg(struct xe_guc *guc, struct drm_printer *p) +{ + xe_guc_log_print_dmesg(&guc->log); + return 0; +} + +static int guc_ctb(struct xe_guc *guc, struct drm_printer *p) +{ + xe_guc_ct_print(&guc->ct, p, true); + return 0; +} + +static int guc_pc(struct xe_guc *guc, struct drm_printer *p) +{ + xe_guc_pc_print(&guc->pc, p); + return 0; +} + +/* + * only for GuC debugfs files which can be safely used on the VF as well: + * - without access to the GuC privileged registers + * - without access to the PF specific GuC objects + */ +static const struct drm_info_list vf_safe_debugfs_list[] = { + { "guc_info", .show = guc_debugfs_show, .data = xe_guc_print_info }, + { "guc_ctb", .show = guc_debugfs_show, .data = guc_ctb }, +}; + +/* For GuC debugfs files that require the SLPC support */ +static const struct drm_info_list slpc_debugfs_list[] = { + { "guc_pc", .show = guc_debugfs_show, .data = guc_pc }, +}; + +/* everything else should be added here */ +static const struct drm_info_list pf_only_debugfs_list[] = { + { "guc_log", .show = guc_debugfs_show, .data = guc_log }, + { "guc_log_lfd", .show = guc_debugfs_show, .data = guc_log_lfd }, + { "guc_log_dmesg", .show = guc_debugfs_show, .data = guc_log_dmesg }, +}; + +void xe_guc_debugfs_register(struct xe_guc *guc, struct dentry *parent) +{ + struct xe_device *xe = guc_to_xe(guc); + struct drm_minor *minor = xe->drm.primary; + + drm_debugfs_create_files(vf_safe_debugfs_list, + ARRAY_SIZE(vf_safe_debugfs_list), + parent, minor); + + if (!IS_SRIOV_VF(xe)) { + drm_debugfs_create_files(pf_only_debugfs_list, + ARRAY_SIZE(pf_only_debugfs_list), + parent, minor); + + if (!xe->info.skip_guc_pc) + drm_debugfs_create_files(slpc_debugfs_list, + ARRAY_SIZE(slpc_debugfs_list), + parent, minor); + } +} diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.h b/drivers/gpu/drm/xe/xe_guc_debugfs.h new file mode 100644 index 000000000..4756dff26 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_DEBUGFS_H_ +#define _XE_GUC_DEBUGFS_H_ + +struct dentry; +struct xe_guc; + +void xe_guc_debugfs_register(struct xe_guc *guc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_engine_activity.c b/drivers/gpu/drm/xe/xe_guc_engine_activity.c new file mode 100644 index 000000000..a782be57c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_engine_activity.c @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "abi/guc_actions_abi.h" +#include "regs/xe_gt_regs.h" + +#include "xe_bo.h" +#include "xe_force_wake.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_engine_activity.h" +#include "xe_guc_ct.h" +#include "xe_hw_engine.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_trace_guc.h" + +#define TOTAL_QUANTA 0x8000 + +static struct iosys_map engine_activity_map(struct xe_guc *guc, struct xe_hw_engine *hwe, + unsigned int index) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct engine_activity_buffer *buffer; + u16 guc_class = xe_hwe_to_guc_class(hwe); + u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); + size_t offset; + + if (engine_activity->num_functions) { + buffer = &engine_activity->function_buffer; + offset = sizeof(struct guc_engine_activity_data) * index; + } else { + buffer = &engine_activity->device_buffer; + offset = 0; + } + + offset += offsetof(struct guc_engine_activity_data, + engine_activity[guc_class][guc_logical_instance]); + + return IOSYS_MAP_INIT_OFFSET(&buffer->activity_bo->vmap, offset); +} + +static struct iosys_map engine_metadata_map(struct xe_guc *guc, + unsigned int index) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct engine_activity_buffer *buffer; + size_t offset; + + if (engine_activity->num_functions) { + buffer = &engine_activity->function_buffer; + offset = sizeof(struct guc_engine_activity_metadata) * index; + } else { + buffer = &engine_activity->device_buffer; + offset = 0; + } + + return IOSYS_MAP_INIT_OFFSET(&buffer->metadata_bo->vmap, offset); +} + +static int allocate_engine_activity_group(struct xe_guc *guc) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct xe_device *xe = guc_to_xe(guc); + u32 num_activity_group; + + /* + * An additional activity group is allocated for PF + */ + num_activity_group = IS_SRIOV_PF(xe) ? xe_sriov_pf_get_totalvfs(xe) + 1 : 1; + + engine_activity->eag = drmm_kcalloc(&xe->drm, num_activity_group, + sizeof(struct engine_activity_group), GFP_KERNEL); + + if (!engine_activity->eag) + return -ENOMEM; + + engine_activity->num_activity_group = num_activity_group; + + return 0; +} + +static int allocate_engine_activity_buffers(struct xe_guc *guc, + struct engine_activity_buffer *buffer, + int count) +{ + u32 metadata_size = sizeof(struct guc_engine_activity_metadata) * count; + u32 size = sizeof(struct guc_engine_activity_data) * count; + struct xe_gt *gt = guc_to_gt(guc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *bo, *metadata_bo; + + metadata_bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(metadata_size), + ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE, + false); + + if (IS_ERR(metadata_bo)) + return PTR_ERR(metadata_bo); + + bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(size), + ttm_bo_type_kernel, XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE, false); + + if (IS_ERR(bo)) { + xe_bo_unpin_map_no_vm(metadata_bo); + return PTR_ERR(bo); + } + + buffer->metadata_bo = metadata_bo; + buffer->activity_bo = bo; + return 0; +} + +static void free_engine_activity_buffers(struct engine_activity_buffer *buffer) +{ + xe_bo_unpin_map_no_vm(buffer->metadata_bo); + xe_bo_unpin_map_no_vm(buffer->activity_bo); +} + +static bool is_engine_activity_supported(struct xe_guc *guc) +{ + struct xe_uc_fw_version *version = &guc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]; + struct xe_uc_fw_version required = { .major = 1, .minor = 14, .patch = 1 }; + struct xe_gt *gt = guc_to_gt(guc); + + if (IS_SRIOV_VF(gt_to_xe(gt))) { + xe_gt_info(gt, "engine activity stats not supported on VFs\n"); + return false; + } + + /* engine activity stats is supported from GuC interface version (1.14.1) */ + if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER_STRUCT(required)) { + xe_gt_info(gt, + "engine activity stats unsupported in GuC interface v%u.%u.%u, need v%u.%u.%u or higher\n", + version->major, version->minor, version->patch, required.major, + required.minor, required.patch); + return false; + } + + return true; +} + +static struct engine_activity *hw_engine_to_engine_activity(struct xe_hw_engine *hwe, + unsigned int index) +{ + struct xe_guc *guc = &hwe->gt->uc.guc; + struct engine_activity_group *eag = &guc->engine_activity.eag[index]; + u16 guc_class = xe_hwe_to_guc_class(hwe); + u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); + + return &eag->engine[guc_class][guc_logical_instance]; +} + +static u64 cpu_ns_to_guc_tsc_tick(ktime_t ns, u32 freq) +{ + return mul_u64_u32_div(ns, freq, NSEC_PER_SEC); +} + +#define read_engine_activity_record(xe_, map_, field_) \ + xe_map_rd_field(xe_, map_, 0, struct guc_engine_activity, field_) + +#define read_metadata_record(xe_, map_, field_) \ + xe_map_rd_field(xe_, map_, 0, struct guc_engine_activity_metadata, field_) + +static u64 get_engine_active_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, + unsigned int index) +{ + struct engine_activity *ea = hw_engine_to_engine_activity(hwe, index); + struct guc_engine_activity *cached_activity = &ea->activity; + struct guc_engine_activity_metadata *cached_metadata = &ea->metadata; + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct iosys_map activity_map, metadata_map; + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + u32 last_update_tick, global_change_num; + u64 active_ticks, gpm_ts; + u16 change_num; + + activity_map = engine_activity_map(guc, hwe, index); + metadata_map = engine_metadata_map(guc, index); + global_change_num = read_metadata_record(xe, &metadata_map, global_change_num); + + /* GuC has not initialized activity data yet, return 0 */ + if (!global_change_num) + goto update; + + if (global_change_num == cached_metadata->global_change_num) + goto update; + + cached_metadata->global_change_num = global_change_num; + change_num = read_engine_activity_record(xe, &activity_map, change_num); + + if (!change_num || change_num == cached_activity->change_num) + goto update; + + /* read engine activity values */ + last_update_tick = read_engine_activity_record(xe, &activity_map, last_update_tick); + active_ticks = read_engine_activity_record(xe, &activity_map, active_ticks); + + /* activity calculations */ + ea->running = !!last_update_tick; + ea->total += active_ticks - cached_activity->active_ticks; + ea->active = 0; + + /* cache the counter */ + cached_activity->change_num = change_num; + cached_activity->last_update_tick = last_update_tick; + cached_activity->active_ticks = active_ticks; + +update: + if (ea->running) { + gpm_ts = xe_mmio_read64_2x32(>->mmio, MISC_STATUS_0) >> + engine_activity->gpm_timestamp_shift; + ea->active = lower_32_bits(gpm_ts) - cached_activity->last_update_tick; + } + + trace_xe_guc_engine_activity(xe, ea, hwe->name, hwe->instance); + + return ea->total + ea->active; +} + +static u64 get_engine_total_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, unsigned int index) +{ + struct engine_activity *ea = hw_engine_to_engine_activity(hwe, index); + struct guc_engine_activity_metadata *cached_metadata = &ea->metadata; + struct guc_engine_activity *cached_activity = &ea->activity; + struct iosys_map activity_map, metadata_map; + struct xe_device *xe = guc_to_xe(guc); + ktime_t now, cpu_delta; + u64 numerator; + u16 quanta_ratio; + + activity_map = engine_activity_map(guc, hwe, index); + metadata_map = engine_metadata_map(guc, index); + + if (!cached_metadata->guc_tsc_frequency_hz) + cached_metadata->guc_tsc_frequency_hz = read_metadata_record(xe, &metadata_map, + guc_tsc_frequency_hz); + + quanta_ratio = read_engine_activity_record(xe, &activity_map, quanta_ratio); + cached_activity->quanta_ratio = quanta_ratio; + + /* Total ticks calculations */ + now = ktime_get(); + cpu_delta = now - ea->last_cpu_ts; + ea->last_cpu_ts = now; + numerator = (ea->quanta_remainder_ns + cpu_delta) * cached_activity->quanta_ratio; + ea->quanta_ns += numerator / TOTAL_QUANTA; + ea->quanta_remainder_ns = numerator % TOTAL_QUANTA; + ea->quanta = cpu_ns_to_guc_tsc_tick(ea->quanta_ns, cached_metadata->guc_tsc_frequency_hz); + + trace_xe_guc_engine_activity(xe, ea, hwe->name, hwe->instance); + + return ea->quanta; +} + +static int enable_engine_activity_stats(struct xe_guc *guc) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct engine_activity_buffer *buffer = &engine_activity->device_buffer; + u32 action[] = { + XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER, + xe_bo_ggtt_addr(buffer->metadata_bo), + 0, + xe_bo_ggtt_addr(buffer->activity_bo), + 0, + }; + + /* Blocking here to ensure the buffers are ready before reading them */ + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +static int enable_function_engine_activity_stats(struct xe_guc *guc, bool enable) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + u32 metadata_ggtt_addr = 0, ggtt_addr = 0, num_functions = 0; + struct engine_activity_buffer *buffer = &engine_activity->function_buffer; + u32 action[6]; + int len = 0; + + if (enable) { + metadata_ggtt_addr = xe_bo_ggtt_addr(buffer->metadata_bo); + ggtt_addr = xe_bo_ggtt_addr(buffer->activity_bo); + num_functions = engine_activity->num_functions; + } + + action[len++] = XE_GUC_ACTION_SET_FUNCTION_ENGINE_ACTIVITY_BUFFER; + action[len++] = num_functions; + action[len++] = metadata_ggtt_addr; + action[len++] = 0; + action[len++] = ggtt_addr; + action[len++] = 0; + + /* Blocking here to ensure the buffers are ready before reading them */ + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +static void engine_activity_set_cpu_ts(struct xe_guc *guc, unsigned int index) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct engine_activity_group *eag = &engine_activity->eag[index]; + int i, j; + + xe_gt_assert(guc_to_gt(guc), index < engine_activity->num_activity_group); + + for (i = 0; i < GUC_MAX_ENGINE_CLASSES; i++) + for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; j++) + eag->engine[i][j].last_cpu_ts = ktime_get(); +} + +static u32 gpm_timestamp_shift(struct xe_gt *gt) +{ + u32 reg; + + reg = xe_mmio_read32(>->mmio, RPM_CONFIG0); + + return 3 - REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg); +} + +static bool is_function_valid(struct xe_guc *guc, unsigned int fn_id) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + + if (!IS_SRIOV_PF(xe) && fn_id) + return false; + + if (engine_activity->num_functions && fn_id >= engine_activity->num_functions) + return false; + + return true; +} + +static int engine_activity_disable_function_stats(struct xe_guc *guc) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct engine_activity_buffer *buffer = &engine_activity->function_buffer; + int ret; + + if (!engine_activity->num_functions) + return 0; + + ret = enable_function_engine_activity_stats(guc, false); + if (ret) + return ret; + + free_engine_activity_buffers(buffer); + engine_activity->num_functions = 0; + + return 0; +} + +static int engine_activity_enable_function_stats(struct xe_guc *guc, int num_vfs) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct engine_activity_buffer *buffer = &engine_activity->function_buffer; + int ret, i; + + if (!num_vfs) + return 0; + + /* This includes 1 PF and num_vfs */ + engine_activity->num_functions = num_vfs + 1; + + ret = allocate_engine_activity_buffers(guc, buffer, engine_activity->num_functions); + if (ret) + return ret; + + ret = enable_function_engine_activity_stats(guc, true); + if (ret) { + free_engine_activity_buffers(buffer); + engine_activity->num_functions = 0; + return ret; + } + + /* skip PF as it was already setup */ + for (i = 1; i < engine_activity->num_functions; i++) + engine_activity_set_cpu_ts(guc, i); + + return 0; +} + +/** + * xe_guc_engine_activity_active_ticks - Get engine active ticks + * @guc: The GuC object + * @hwe: The hw_engine object + * @fn_id: function id to report on + * + * Return: accumulated ticks @hwe was active since engine activity stats were enabled. + */ +u64 xe_guc_engine_activity_active_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, + unsigned int fn_id) +{ + if (!xe_guc_engine_activity_supported(guc)) + return 0; + + if (!is_function_valid(guc, fn_id)) + return 0; + + return get_engine_active_ticks(guc, hwe, fn_id); +} + +/** + * xe_guc_engine_activity_total_ticks - Get engine total ticks + * @guc: The GuC object + * @hwe: The hw_engine object + * @fn_id: function id to report on + * + * Return: accumulated quanta of ticks allocated for the engine + */ +u64 xe_guc_engine_activity_total_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, + unsigned int fn_id) +{ + if (!xe_guc_engine_activity_supported(guc)) + return 0; + + if (!is_function_valid(guc, fn_id)) + return 0; + + return get_engine_total_ticks(guc, hwe, fn_id); +} + +/** + * xe_guc_engine_activity_supported - Check support for engine activity stats + * @guc: The GuC object + * + * Engine activity stats is supported from GuC interface version (1.14.1) + * + * Return: true if engine activity stats supported, false otherwise + */ +bool xe_guc_engine_activity_supported(struct xe_guc *guc) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + + return engine_activity->supported; +} + +/** + * xe_guc_engine_activity_function_stats - Enable/Disable per-function engine activity stats + * @guc: The GuC object + * @num_vfs: number of vfs + * @enable: true to enable, false otherwise + * + * Return: 0 on success, negative error code otherwise + */ +int xe_guc_engine_activity_function_stats(struct xe_guc *guc, int num_vfs, bool enable) +{ + if (!xe_guc_engine_activity_supported(guc)) + return 0; + + if (enable) + return engine_activity_enable_function_stats(guc, num_vfs); + + return engine_activity_disable_function_stats(guc); +} + +/** + * xe_guc_engine_activity_enable_stats - Enable engine activity stats + * @guc: The GuC object + * + * Enable engine activity stats and set initial timestamps + */ +void xe_guc_engine_activity_enable_stats(struct xe_guc *guc) +{ + int ret; + + if (!xe_guc_engine_activity_supported(guc)) + return; + + ret = enable_engine_activity_stats(guc); + if (ret) + xe_gt_err(guc_to_gt(guc), "failed to enable activity stats: %pe\n", ERR_PTR(ret)); + else + engine_activity_set_cpu_ts(guc, 0); +} + +static void engine_activity_fini(void *arg) +{ + struct xe_guc_engine_activity *engine_activity = arg; + struct engine_activity_buffer *buffer = &engine_activity->device_buffer; + + free_engine_activity_buffers(buffer); +} + +/** + * xe_guc_engine_activity_init - Initialize the engine activity data + * @guc: The GuC object + * + * Return: 0 on success, negative error code otherwise. + */ +int xe_guc_engine_activity_init(struct xe_guc *guc) +{ + struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + engine_activity->supported = is_engine_activity_supported(guc); + if (!engine_activity->supported) + return 0; + + ret = allocate_engine_activity_group(guc); + if (ret) { + xe_gt_err(gt, "failed to allocate engine activity group (%pe)\n", ERR_PTR(ret)); + return ret; + } + + ret = allocate_engine_activity_buffers(guc, &engine_activity->device_buffer, 1); + if (ret) { + xe_gt_err(gt, "failed to allocate engine activity buffers (%pe)\n", ERR_PTR(ret)); + return ret; + } + + engine_activity->gpm_timestamp_shift = gpm_timestamp_shift(gt); + + return devm_add_action_or_reset(gt_to_xe(gt)->drm.dev, engine_activity_fini, + engine_activity); +} diff --git a/drivers/gpu/drm/xe/xe_guc_engine_activity.h b/drivers/gpu/drm/xe/xe_guc_engine_activity.h new file mode 100644 index 000000000..b32926c2d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_engine_activity.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GUC_ENGINE_ACTIVITY_H_ +#define _XE_GUC_ENGINE_ACTIVITY_H_ + +#include + +struct xe_hw_engine; +struct xe_guc; + +int xe_guc_engine_activity_init(struct xe_guc *guc); +bool xe_guc_engine_activity_supported(struct xe_guc *guc); +void xe_guc_engine_activity_enable_stats(struct xe_guc *guc); +int xe_guc_engine_activity_function_stats(struct xe_guc *guc, int num_vfs, bool enable); +u64 xe_guc_engine_activity_active_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, + unsigned int fn_id); +u64 xe_guc_engine_activity_total_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, + unsigned int fn_id); +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_engine_activity_types.h b/drivers/gpu/drm/xe/xe_guc_engine_activity_types.h new file mode 100644 index 000000000..48f69ddef --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_engine_activity_types.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GUC_ENGINE_ACTIVITY_TYPES_H_ +#define _XE_GUC_ENGINE_ACTIVITY_TYPES_H_ + +#include + +#include "xe_guc_fwif.h" +/** + * struct engine_activity - Engine specific activity data + * + * Contains engine specific activity data and snapshot of the + * structures from GuC + */ +struct engine_activity { + /** @active: current activity */ + u64 active; + + /** @last_cpu_ts: cpu timestamp in nsec of previous sample */ + u64 last_cpu_ts; + + /** @quanta: total quanta used on HW */ + u64 quanta; + + /** @quanta_ns: total quanta_ns used on HW */ + u64 quanta_ns; + + /** + * @quanta_remainder_ns: remainder when the CPU time is scaled as + * per the quanta_ratio. This remainder is used in subsequent + * quanta calculations. + */ + u64 quanta_remainder_ns; + + /** @total: total engine activity */ + u64 total; + + /** @running: true if engine is running some work */ + bool running; + + /** @metadata: snapshot of engine activity metadata */ + struct guc_engine_activity_metadata metadata; + + /** @activity: snapshot of engine activity counter */ + struct guc_engine_activity activity; +}; + +/** + * struct engine_activity_group - Activity data for all engines + */ +struct engine_activity_group { + /** @engine: engine specific activity data */ + struct engine_activity engine[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; +}; + +/** + * struct engine_activity_buffer - engine activity buffers + * + * This contains the buffers allocated for metadata and activity data + */ +struct engine_activity_buffer { + /** @activity_bo: object allocated to hold activity data */ + struct xe_bo *activity_bo; + + /** @metadata_bo: object allocated to hold activity metadata */ + struct xe_bo *metadata_bo; +}; + +/** + * struct xe_guc_engine_activity - Data used by engine activity implementation + */ +struct xe_guc_engine_activity { + /** @gpm_timestamp_shift: Right shift value for the gpm timestamp */ + u32 gpm_timestamp_shift; + + /** @num_activity_group: number of activity groups */ + u32 num_activity_group; + + /** @num_functions: number of functions */ + u32 num_functions; + + /** @supported: indicates support for engine activity stats */ + bool supported; + + /** + * @eag: holds the device level engine activity data in native mode. + * In SRIOV mode, points to an array with entries which holds the engine + * activity data for PF and VF's + */ + struct engine_activity_group *eag; + + /** @device_buffer: buffer object for global engine activity */ + struct engine_activity_buffer device_buffer; + + /** @function_buffer: buffer object for per-function engine activity */ + struct engine_activity_buffer function_buffer; +}; +#endif + diff --git a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h new file mode 100644 index 000000000..acdc24d1a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_EXEC_QUEUE_TYPES_H_ +#define _XE_GUC_EXEC_QUEUE_TYPES_H_ + +#include +#include + +#include "xe_gpu_scheduler_types.h" +#include "xe_hw_fence_types.h" + +struct dma_fence; +struct xe_exec_queue; + +/** + * struct xe_guc_exec_queue - GuC specific state for an xe_exec_queue + */ +struct xe_guc_exec_queue { + /** @q: Backpointer to parent xe_exec_queue */ + struct xe_exec_queue *q; + /** @rcu: For safe freeing of exported dma fences */ + struct rcu_head rcu; + /** @sched: GPU scheduler for this xe_exec_queue */ + struct xe_gpu_scheduler sched; + /** + * @name: Scheduler timeline name, kept with @sched until RCU free. + */ + char name[MAX_FENCE_NAME_LEN]; + /** @entity: Scheduler entity for this xe_exec_queue */ + struct xe_sched_entity entity; + /** + * @static_msgs: Static messages for this xe_exec_queue, used when + * a message needs to sent through the GPU scheduler but memory + * allocations are not allowed. + */ +#define MAX_STATIC_MSG_TYPE 3 + struct xe_sched_msg static_msgs[MAX_STATIC_MSG_TYPE]; + /** @destroy_async: do final destroy async from this worker */ + struct work_struct destroy_async; + /** @resume_time: time of last resume */ + u64 resume_time; + /** @state: GuC specific state for this xe_exec_queue */ + atomic_t state; + /** @wqi_head: work queue item tail */ + u32 wqi_head; + /** @wqi_tail: work queue item tail */ + u32 wqi_tail; + /** @id: GuC id for this exec_queue */ + u16 id; + /** @suspend_wait: wait queue used to wait on pending suspends */ + wait_queue_head_t suspend_wait; + /** @suspend_pending: a suspend of the exec_queue is pending */ + bool suspend_pending; + /** + * @suspend_count: Reference count of active suspend requests. The + * exec_queue remains suspended while this is non-zero, allowing + * multiple concurrent callers to independently hold a suspend without + * prematurely re-enabling the queue. Protected by @sched.msg_lock. + */ + int suspend_count; + /** + * @needs_cleanup: Needs a cleanup message during VF post migration + * recovery. + */ + bool needs_cleanup; + /** + * @needs_suspend: Needs a suspend message during VF post migration + * recovery. + */ + bool needs_suspend; + /** + * @needs_resume: Needs a resume message during VF post migration + * recovery. + */ + bool needs_resume; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h new file mode 100644 index 000000000..971b850f2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_FWIF_H_ +#define _XE_GUC_FWIF_H_ + +#include + +#include "abi/guc_capture_abi.h" +#include "abi/guc_klvs_abi.h" +#include "abi/guc_scheduler_abi.h" +#include "xe_hw_engine_types.h" + +#define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET 4 +#define G2H_LEN_DW_DEREGISTER_CONTEXT 3 +#define G2H_LEN_DW_TLB_INVALIDATE 3 +#define G2H_LEN_DW_G2G_NOTIFY_MIN 3 +#define G2H_LEN_DW_MULTI_QUEUE_CONTEXT 3 +#define G2H_LEN_DW_PAGE_RECLAMATION 3 + +/* 32-bit KLV structure as used by policy updates and others */ +struct guc_klv_generic_dw_t { + u32 kl; + u32 value; +} __packed; + +/* Format of the UPDATE_CONTEXT_POLICIES H2G data packet */ +struct guc_update_exec_queue_policy_header { + u32 action; + u32 guc_id; +} __packed; + +struct guc_update_exec_queue_policy { + struct guc_update_exec_queue_policy_header header; + struct guc_klv_generic_dw_t klv[GUC_CONTEXT_POLICIES_KLV_NUM_IDS]; +} __packed; + +/* GUC_CTL_* - Parameters for loading the GuC */ +#define GUC_CTL_LOG_PARAMS 0 +#define GUC_LOG_VALID BIT(0) +#define GUC_LOG_NOTIFY_ON_HALF_FULL BIT(1) +#define GUC_LOG_CAPTURE_ALLOC_UNITS BIT(2) +#define GUC_LOG_LOG_ALLOC_UNITS BIT(3) +#define GUC_LOG_CRASH_DUMP REG_GENMASK(5, 4) +#define GUC_LOG_EVENT_DATA REG_GENMASK(9, 6) +#define GUC_LOG_STATE_CAPTURE REG_GENMASK(11, 10) +#define GUC_LOG_BUF_ADDR REG_GENMASK(31, 12) + +#define GUC_CTL_WA 1 +#define GUC_WA_GAM_CREDITS BIT(10) +#define GUC_WA_DUAL_QUEUE BIT(11) +#define GUC_WA_RCS_RESET_BEFORE_RC6 BIT(13) +#define GUC_WA_CONTEXT_ISOLATION BIT(15) +#define GUC_WA_PRE_PARSER BIT(14) +#define GUC_WA_HOLD_CCS_SWITCHOUT BIT(17) +#define GUC_WA_POLLCS BIT(18) +#define GUC_WA_RENDER_RST_RC6_EXIT BIT(19) +#define GUC_WA_RCS_REGS_IN_CCS_REGS_LIST BIT(21) +#define GUC_WA_ENABLE_TSC_CHECK_ON_RC6 BIT(22) +#define GUC_WA_SAVE_RESTORE_MCFG_REG_AT_MC6 BIT(25) + +#define GUC_CTL_FEATURE 2 +#define GUC_CTL_ENABLE_SLPC BIT(2) +#define GUC_CTL_ENABLE_LITE_RESTORE BIT(4) +#define GUC_CTL_ENABLE_PSMI_LOGGING BIT(7) +#define GUC_CTL_MAIN_GAMCTRL_QUEUES BIT(9) +#define GUC_CTL_DISABLE_SCHEDULER BIT(14) +#define GUC_CTL_ENABLE_L2FLUSH_OPT BIT(15) +#define GUC_CTL_DISABLE_MULTI_QUEUE BIT(24) + +#define GUC_CTL_DEBUG 3 +#define GUC_LOG_VERBOSITY REG_GENMASK(1, 0) +#define GUC_LOG_VERBOSITY_MAX 3 +#define GUC_LOG_DESTINATION REG_GENMASK(5, 4) +#define GUC_LOG_DISABLED BIT(6) +#define GUC_PROFILE_ENABLED BIT(7) + +#define GUC_CTL_ADS 4 +#define GUC_ADS_ADDR REG_GENMASK(21, 1) + +#define GUC_CTL_DEVID 5 + +#define GUC_CTL_MAX_DWORDS 14 + +/* Scheduling policy settings */ + +#define GLOBAL_POLICY_MAX_NUM_WI 15 + +/* Don't reset an engine upon preemption failure */ +#define GLOBAL_POLICY_DISABLE_ENGINE_RESET BIT(0) + +#define GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US 500000 + +struct guc_policies { + u32 submission_queue_depth[GUC_MAX_ENGINE_CLASSES]; + /* + * In micro seconds. How much time to allow before DPC processing is + * called back via interrupt (to prevent DPC queue drain starving). + * Typically 1000s of micro seconds (example only, not granularity). + */ + u32 dpc_promote_time; + + /* Must be set to take these new values. */ + u32 is_valid; + + /* + * Max number of WIs to process per call. A large value may keep CS + * idle. + */ + u32 max_num_work_items; + + u32 global_flags; + u32 reserved[4]; +} __packed; + +/* Generic GT SysInfo data types */ +#define GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED 0 +#define GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK 1 +#define GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI 2 +#define GUC_GENERIC_GT_SYSINFO_MAX 16 + +/* HW info */ +struct guc_gt_system_info { + u8 mapping_table[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 engine_enabled_masks[GUC_MAX_ENGINE_CLASSES]; + u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; +} __packed; + +/* GuC Additional Data Struct */ +struct guc_ads { + struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 reserved0; + u32 scheduler_policies; + u32 gt_system_info; + u32 reserved1; + u32 control_data; + u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; + u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; + u32 private_data; + u32 um_init_data; + u32 capture_instance[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES]; + u32 capture_class[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES]; + u32 capture_global[GUC_CAPTURE_LIST_INDEX_MAX]; + u32 wa_klv_addr_lo; + u32 wa_klv_addr_hi; + u32 wa_klv_size; + u32 reserved[11]; +} __packed; + +/* Engine usage stats */ +struct guc_engine_usage_record { + u32 current_context_index; + u32 last_switch_in_stamp; + u32 reserved0; + u32 total_runtime; + u32 reserved1[4]; +} __packed; + +struct guc_engine_usage { + struct guc_engine_usage_record engines[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; +} __packed; + +/* Engine Activity stats */ +struct guc_engine_activity { + u16 change_num; + u16 quanta_ratio; + u32 last_update_tick; + u64 active_ticks; +} __packed; + +struct guc_engine_activity_data { + struct guc_engine_activity engine_activity[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; +} __packed; + +struct guc_engine_activity_metadata { + u32 guc_tsc_frequency_hz; + u32 lag_latency_usec; + u32 global_change_num; + u32 reserved; +} __packed; + +/* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ +enum xe_guc_recv_message { + XE_GUC_RECV_MSG_CRASH_DUMP_POSTED = BIT(1), + XE_GUC_RECV_MSG_EXCEPTION = BIT(30), +}; + +/* Page fault structures */ +struct access_counter_desc { + u32 dw0; +#define ACCESS_COUNTER_TYPE BIT(0) +#define ACCESS_COUNTER_SUBG_LO GENMASK(31, 1) + + u32 dw1; +#define ACCESS_COUNTER_SUBG_HI BIT(0) +#define ACCESS_COUNTER_RSVD0 GENMASK(2, 1) +#define ACCESS_COUNTER_ENG_INSTANCE GENMASK(8, 3) +#define ACCESS_COUNTER_ENG_CLASS GENMASK(11, 9) +#define ACCESS_COUNTER_ASID GENMASK(31, 12) + + u32 dw2; +#define ACCESS_COUNTER_VFID GENMASK(5, 0) +#define ACCESS_COUNTER_RSVD1 GENMASK(7, 6) +#define ACCESS_COUNTER_GRANULARITY GENMASK(10, 8) +#define ACCESS_COUNTER_RSVD2 GENMASK(16, 11) +#define ACCESS_COUNTER_VIRTUAL_ADDR_RANGE_LO GENMASK(31, 17) + + u32 dw3; +#define ACCESS_COUNTER_VIRTUAL_ADDR_RANGE_HI GENMASK(31, 0) +} __packed; + +enum guc_um_queue_type { + GUC_UM_HW_QUEUE_PAGE_FAULT = 0, + GUC_UM_HW_QUEUE_PAGE_FAULT_RESPONSE, + GUC_UM_HW_QUEUE_ACCESS_COUNTER, + GUC_UM_HW_QUEUE_MAX +}; + +struct guc_um_queue_params { + u64 base_dpa; + u32 base_ggtt_address; + u32 size_in_bytes; + u32 rsvd[4]; +} __packed; + +struct guc_um_init_params { + u64 page_response_timeout_in_us; + u32 rsvd[6]; + struct guc_um_queue_params queue_params[GUC_UM_HW_QUEUE_MAX]; +} __packed; + +enum xe_guc_fault_reply_type { + PFR_ACCESS = 0, + PFR_ENGINE, + PFR_VFID, + PFR_ALL, + PFR_INVALID +}; + +enum xe_guc_response_desc_type { + TLB_INVALIDATION_DESC = 0, + FAULT_RESPONSE_DESC +}; + +struct xe_guc_pagefault_desc { + u32 dw0; +#define PFD_FAULT_LEVEL GENMASK(2, 0) +#define PFD_SRC_ID GENMASK(10, 3) +#define PFD_RSVD_0 GENMASK(17, 11) +#define XE2_PFD_TRVA_FAULT BIT(18) +#define PFD_ENG_INSTANCE GENMASK(24, 19) +#define PFD_ENG_CLASS GENMASK(27, 25) +#define PFD_PDATA_LO GENMASK(31, 28) + + u32 dw1; +#define PFD_PDATA_HI GENMASK(11, 0) +#define PFD_PDATA_HI_SHIFT 4 +#define PFD_ASID GENMASK(31, 12) + + u32 dw2; +#define PFD_ACCESS_TYPE GENMASK(1, 0) +#define PFD_FAULT_TYPE GENMASK(3, 2) +#define PFD_VFID GENMASK(9, 4) +#define PFD_RSVD_1 BIT(10) +#define PFD_PREFETCH BIT(11) /* Only valid on Xe3+, reserved on prior platforms */ +#define PFD_VIRTUAL_ADDR_LO GENMASK(31, 12) +#define PFD_VIRTUAL_ADDR_LO_SHIFT 12 + + u32 dw3; +#define PFD_VIRTUAL_ADDR_HI GENMASK(31, 0) +#define PFD_VIRTUAL_ADDR_HI_SHIFT 32 +} __packed; + +struct xe_guc_pagefault_reply { + u32 dw0; +#define PFR_VALID BIT(0) +#define PFR_SUCCESS BIT(1) +#define PFR_REPLY GENMASK(4, 2) +#define PFR_RSVD_0 GENMASK(9, 5) +#define PFR_DESC_TYPE GENMASK(11, 10) +#define PFR_ASID GENMASK(31, 12) + + u32 dw1; +#define PFR_VFID GENMASK(5, 0) +#define PFR_PREFETCH BIT(6) /* Only valid on Xe3+, reserved on prior platforms */ +#define PFR_ENG_INSTANCE GENMASK(12, 7) +#define PFR_ENG_CLASS GENMASK(15, 13) +#define PFR_PDATA GENMASK(31, 16) + + u32 dw2; +#define PFR_RSVD_2 GENMASK(31, 0) +} __packed; + +struct xe_guc_acc_desc { + u32 dw0; +#define ACC_TYPE BIT(0) +#define ACC_TRIGGER 0 +#define ACC_NOTIFY 1 +#define ACC_SUBG_LO GENMASK(31, 1) + + u32 dw1; +#define ACC_SUBG_HI BIT(0) +#define ACC_RSVD0 GENMASK(2, 1) +#define ACC_ENG_INSTANCE GENMASK(8, 3) +#define ACC_ENG_CLASS GENMASK(11, 9) +#define ACC_ASID GENMASK(31, 12) + + u32 dw2; +#define ACC_VFID GENMASK(5, 0) +#define ACC_RSVD1 GENMASK(7, 6) +#define ACC_GRANULARITY GENMASK(10, 8) +#define ACC_RSVD2 GENMASK(16, 11) +#define ACC_VIRTUAL_ADDR_RANGE_LO GENMASK(31, 17) + + u32 dw3; +#define ACC_VIRTUAL_ADDR_RANGE_HI GENMASK(31, 0) +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c new file mode 100644 index 000000000..b300901db --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_hwconfig.h" + +#include +#include + +#include "abi/guc_actions_abi.h" +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_gt_types.h" +#include "xe_guc.h" +#include "xe_map.h" + +static int send_get_hwconfig(struct xe_guc *guc, u64 ggtt_addr, u32 size) +{ + u32 action[] = { + XE_GUC_ACTION_GET_HWCONFIG, + lower_32_bits(ggtt_addr), + upper_32_bits(ggtt_addr), + size, + }; + + return xe_guc_mmio_send(guc, action, ARRAY_SIZE(action)); +} + +static int guc_hwconfig_size(struct xe_guc *guc, u32 *size) +{ + int ret = send_get_hwconfig(guc, 0, 0); + + if (ret < 0) + return ret; + + *size = ret; + return 0; +} + +static int guc_hwconfig_copy(struct xe_guc *guc) +{ + int ret = send_get_hwconfig(guc, xe_bo_ggtt_addr(guc->hwconfig.bo), + guc->hwconfig.size); + + if (ret < 0) + return ret; + + return 0; +} + +int xe_guc_hwconfig_init(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *bo; + u32 size; + int err; + + /* Initialization already done */ + if (guc->hwconfig.bo) + return 0; + + /* + * All hwconfig the same across GTs so only GT0 needs to be configured + */ + if (gt->info.id != XE_GT0) + return 0; + + /* ADL_P, DG2+ supports hwconfig table */ + if (GRAPHICS_VERx100(xe) < 1255 && xe->info.platform != XE_ALDERLAKE_P) + return 0; + + err = guc_hwconfig_size(guc, &size); + if (err) + return err; + if (!size) + return -EINVAL; + + bo = xe_managed_bo_create_pin_map(xe, tile, PAGE_ALIGN(size), + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + guc->hwconfig.bo = bo; + guc->hwconfig.size = size; + + return guc_hwconfig_copy(guc); +} + +u32 xe_guc_hwconfig_size(struct xe_guc *guc) +{ + return !guc->hwconfig.bo ? 0 : guc->hwconfig.size; +} + +void xe_guc_hwconfig_copy(struct xe_guc *guc, void *dst) +{ + struct xe_device *xe = guc_to_xe(guc); + + XE_WARN_ON(!guc->hwconfig.bo); + + xe_map_memcpy_from(xe, dst, &guc->hwconfig.bo->vmap, 0, + guc->hwconfig.size); +} + +void xe_guc_hwconfig_dump(struct xe_guc *guc, struct drm_printer *p) +{ + size_t size = xe_guc_hwconfig_size(guc); + u32 *hwconfig; + u64 num_dw; + u32 extra_bytes; + int i = 0; + + if (size == 0) { + drm_printf(p, "No hwconfig available\n"); + return; + } + + num_dw = div_u64_rem(size, sizeof(u32), &extra_bytes); + + hwconfig = kzalloc(size, GFP_KERNEL); + if (!hwconfig) { + drm_printf(p, "Error: could not allocate hwconfig memory\n"); + return; + } + + xe_guc_hwconfig_copy(guc, hwconfig); + + /* An entry requires at least three dwords for key, length, value */ + while (i + 3 <= num_dw) { + u32 attribute = hwconfig[i++]; + u32 len_dw = hwconfig[i++]; + + if (i + len_dw > num_dw) { + drm_printf(p, "Error: Attribute %u is %u dwords, but only %llu remain\n", + attribute, len_dw, num_dw - i); + len_dw = num_dw - i; + } + + /* + * If it's a single dword (as most hwconfig attributes are), + * then it's probably a number that makes sense to display + * in decimal form. In the rare cases where it's more than + * one dword, just print it in hex form and let the user + * figure out how to interpret it. + */ + if (len_dw == 1) + drm_printf(p, "[%2u] = %u\n", attribute, hwconfig[i]); + else + drm_printf(p, "[%2u] = { %*ph }\n", attribute, + (int)(len_dw * sizeof(u32)), &hwconfig[i]); + i += len_dw; + } + + if (i < num_dw || extra_bytes) + drm_printf(p, "Error: %llu extra bytes at end of hwconfig\n", + (num_dw - i) * sizeof(u32) + extra_bytes); + + kfree(hwconfig); +} + +/* + * Lookup a specific 32-bit attribute value in the GuC's hwconfig table. + */ +int xe_guc_hwconfig_lookup_u32(struct xe_guc *guc, u32 attribute, u32 *val) +{ + size_t size = xe_guc_hwconfig_size(guc); + u64 num_dw = div_u64(size, sizeof(u32)); + u32 *hwconfig; + bool found = false; + int i = 0; + + if (num_dw == 0) + return -EINVAL; + + hwconfig = kzalloc(size, GFP_KERNEL); + if (!hwconfig) + return -ENOMEM; + + xe_guc_hwconfig_copy(guc, hwconfig); + + /* An entry requires at least three dwords for key, length, value */ + while (i + 3 <= num_dw) { + u32 key = hwconfig[i++]; + u32 len_dw = hwconfig[i++]; + + if (key != attribute) { + i += len_dw; + continue; + } + + *val = hwconfig[i]; + found = true; + break; + } + + kfree(hwconfig); + + return found ? 0 : -ENOENT; +} diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.h b/drivers/gpu/drm/xe/xe_guc_hwconfig.h new file mode 100644 index 000000000..ab4e50382 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_HWCONFIG_H_ +#define _XE_GUC_HWCONFIG_H_ + +#include + +struct drm_printer; +struct xe_guc; + +int xe_guc_hwconfig_init(struct xe_guc *guc); +u32 xe_guc_hwconfig_size(struct xe_guc *guc); +void xe_guc_hwconfig_copy(struct xe_guc *guc, void *dst); +void xe_guc_hwconfig_dump(struct xe_guc *guc, struct drm_printer *p); +int xe_guc_hwconfig_lookup_u32(struct xe_guc *guc, u32 attribute, u32 *val); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_hxg_helpers.h b/drivers/gpu/drm/xe/xe_guc_hxg_helpers.h new file mode 100644 index 000000000..aeeb573c6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_hxg_helpers.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GUC_HXG_HELPERS_H_ +#define _XE_GUC_HXG_HELPERS_H_ + +#include +#include + +#include "abi/guc_messages_abi.h" + +/** + * hxg_sizeof - Queries size of the object or type (in HXG units). + * @T: the object or type + * + * Force a compilation error if actual size is not aligned to HXG unit (u32). + * + * Return: size in dwords (u32). + */ +#define hxg_sizeof(T) (sizeof(T) / sizeof(u32) + BUILD_BUG_ON_ZERO(sizeof(T) % sizeof(u32))) + +static inline const char *guc_hxg_type_to_string(unsigned int type) +{ + switch (type) { + case GUC_HXG_TYPE_REQUEST: + return "request"; + case GUC_HXG_TYPE_FAST_REQUEST: + return "fast-request"; + case GUC_HXG_TYPE_EVENT: + return "event"; + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: + return "busy"; + case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + return "retry"; + case GUC_HXG_TYPE_RESPONSE_FAILURE: + return "failure"; + case GUC_HXG_TYPE_RESPONSE_SUCCESS: + return "response"; + default: + return ""; + } +} + +static inline bool guc_hxg_type_is_action(unsigned int type) +{ + switch (type) { + case GUC_HXG_TYPE_REQUEST: + case GUC_HXG_TYPE_FAST_REQUEST: + case GUC_HXG_TYPE_EVENT: + return true; + default: + return false; + } +} + +static inline bool guc_hxg_type_is_reply(unsigned int type) +{ + switch (type) { + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: + case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + case GUC_HXG_TYPE_RESPONSE_FAILURE: + case GUC_HXG_TYPE_RESPONSE_SUCCESS: + return true; + default: + return false; + } +} + +static inline u32 guc_hxg_msg_encode_success(u32 *msg, u32 data0) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_SUCCESS) | + FIELD_PREP(GUC_HXG_RESPONSE_MSG_0_DATA0, data0); + + return GUC_HXG_RESPONSE_MSG_MIN_LEN; +} + +static inline u32 guc_hxg_msg_encode_failure(u32 *msg, u32 error, u32 hint) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_FAILURE) | + FIELD_PREP(GUC_HXG_FAILURE_MSG_0_HINT, hint) | + FIELD_PREP(GUC_HXG_FAILURE_MSG_0_ERROR, error); + + return GUC_HXG_FAILURE_MSG_LEN; +} + +static inline u32 guc_hxg_msg_encode_busy(u32 *msg, u32 counter) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_NO_RESPONSE_BUSY) | + FIELD_PREP(GUC_HXG_BUSY_MSG_0_COUNTER, counter); + + return GUC_HXG_BUSY_MSG_LEN; +} + +static inline u32 guc_hxg_msg_encode_retry(u32 *msg, u32 reason) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_NO_RESPONSE_RETRY) | + FIELD_PREP(GUC_HXG_RETRY_MSG_0_REASON, reason); + + return GUC_HXG_RETRY_MSG_LEN; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_id_mgr.c b/drivers/gpu/drm/xe/xe_guc_id_mgr.c new file mode 100644 index 000000000..e845425d6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_id_mgr.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include +#include + +#include + +#include "xe_assert.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_id_mgr.h" +#include "xe_guc_types.h" + +static struct xe_guc *idm_to_guc(struct xe_guc_id_mgr *idm) +{ + return container_of(idm, struct xe_guc, submission_state.idm); +} + +static struct xe_gt *idm_to_gt(struct xe_guc_id_mgr *idm) +{ + return guc_to_gt(idm_to_guc(idm)); +} + +static struct xe_device *idm_to_xe(struct xe_guc_id_mgr *idm) +{ + return gt_to_xe(idm_to_gt(idm)); +} + +#define idm_assert(idm, cond) xe_gt_assert(idm_to_gt(idm), cond) +#define idm_mutex(idm) (&idm_to_guc(idm)->submission_state.lock) + +static void idm_print_locked(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent); + +static void __fini_idm(struct drm_device *drm, void *arg) +{ + struct xe_guc_id_mgr *idm = arg; + + mutex_lock(idm_mutex(idm)); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + unsigned int weight = bitmap_weight(idm->bitmap, idm->total); + + if (weight) { + struct drm_printer p = xe_gt_info_printer(idm_to_gt(idm)); + + xe_gt_err(idm_to_gt(idm), "GUC ID manager unclean (%u/%u)\n", + weight, idm->total); + idm_print_locked(idm, &p, 1); + } + } + + bitmap_free(idm->bitmap); + idm->bitmap = NULL; + idm->total = 0; + idm->used = 0; + + mutex_unlock(idm_mutex(idm)); +} + +/** + * xe_guc_id_mgr_init() - Initialize GuC context ID Manager. + * @idm: the &xe_guc_id_mgr to initialize + * @limit: number of IDs to manage + * + * The bare-metal or PF driver can pass ~0 as &limit to indicate that all + * context IDs supported by the GuC firmware are available for use. + * + * Only VF drivers will have to provide explicit number of context IDs + * that they can use. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_id_mgr_init(struct xe_guc_id_mgr *idm, unsigned int limit) +{ + int ret; + + idm_assert(idm, !idm->bitmap); + idm_assert(idm, !idm->total); + idm_assert(idm, !idm->used); + + if (limit == ~0) + limit = GUC_ID_MAX; + else if (limit > GUC_ID_MAX) + return -ERANGE; + else if (!limit) + return -EINVAL; + + idm->bitmap = bitmap_zalloc(limit, GFP_KERNEL); + if (!idm->bitmap) + return -ENOMEM; + idm->total = limit; + + ret = drmm_add_action_or_reset(&idm_to_xe(idm)->drm, __fini_idm, idm); + if (ret) + return ret; + + xe_gt_dbg(idm_to_gt(idm), "using %u GuC ID%s\n", + idm->total, str_plural(idm->total)); + return 0; +} + +static unsigned int find_last_zero_area(unsigned long *bitmap, + unsigned int total, + unsigned int count) +{ + unsigned int found = total; + unsigned int rs, re, range; + + for_each_clear_bitrange(rs, re, bitmap, total) { + range = re - rs; + if (range < count) + continue; + found = rs + (range - count); + } + return found; +} + +static int idm_reserve_chunk_locked(struct xe_guc_id_mgr *idm, + unsigned int count, unsigned int retain) +{ + int id; + + idm_assert(idm, count); + lockdep_assert_held(idm_mutex(idm)); + + if (!idm->total) + return -ENODATA; + + if (retain) { + /* + * For IDs reservations (used on PF for VFs) we want to make + * sure there will be at least 'retain' available for the PF + */ + if (idm->used + count + retain > idm->total) + return -EDQUOT; + /* + * ... and we want to reserve highest IDs close to the end. + */ + id = find_last_zero_area(idm->bitmap, idm->total, count); + } else { + /* + * For regular IDs reservations (used by submission code) + * we start searching from the lower range of IDs. + */ + id = bitmap_find_next_zero_area(idm->bitmap, idm->total, 0, count, 0); + } + if (id >= idm->total) + return -ENOSPC; + + bitmap_set(idm->bitmap, id, count); + idm->used += count; + + return id; +} + +static void idm_release_chunk_locked(struct xe_guc_id_mgr *idm, + unsigned int start, unsigned int count) +{ + idm_assert(idm, count); + idm_assert(idm, count <= idm->used); + idm_assert(idm, start < idm->total); + idm_assert(idm, start + count - 1 < idm->total); + lockdep_assert_held(idm_mutex(idm)); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + unsigned int n; + + for (n = 0; n < count; n++) + idm_assert(idm, test_bit(start + n, idm->bitmap)); + } + bitmap_clear(idm->bitmap, start, count); + idm->used -= count; +} + +/** + * xe_guc_id_mgr_reserve_locked() - Reserve one or more GuC context IDs. + * @idm: the &xe_guc_id_mgr + * @count: number of IDs to allocate (can't be 0) + * + * This function is dedicated for the use by the GuC submission code, + * where submission lock is already taken. + * + * Return: ID of allocated GuC context or a negative error code on failure. + */ +int xe_guc_id_mgr_reserve_locked(struct xe_guc_id_mgr *idm, unsigned int count) +{ + return idm_reserve_chunk_locked(idm, count, 0); +} + +/** + * xe_guc_id_mgr_release_locked() - Release one or more GuC context IDs. + * @idm: the &xe_guc_id_mgr + * @id: the GuC context ID to release + * @count: number of IDs to release (can't be 0) + * + * This function is dedicated for the use by the GuC submission code, + * where submission lock is already taken. + */ +void xe_guc_id_mgr_release_locked(struct xe_guc_id_mgr *idm, unsigned int id, + unsigned int count) +{ + return idm_release_chunk_locked(idm, id, count); +} + +/** + * xe_guc_id_mgr_reserve() - Reserve a range of GuC context IDs. + * @idm: the &xe_guc_id_mgr + * @count: number of GuC context IDs to reserve (can't be 0) + * @retain: number of GuC context IDs to keep available (can't be 0) + * + * This function is dedicated for the use by the PF driver which expects that + * reserved range of IDs will be contiguous and that there will be at least + * &retain IDs still available for the PF after this reservation. + * + * Return: starting ID of the allocated GuC context ID range or + * a negative error code on failure. + */ +int xe_guc_id_mgr_reserve(struct xe_guc_id_mgr *idm, + unsigned int count, unsigned int retain) +{ + int ret; + + idm_assert(idm, count); + idm_assert(idm, retain); + + mutex_lock(idm_mutex(idm)); + ret = idm_reserve_chunk_locked(idm, count, retain); + mutex_unlock(idm_mutex(idm)); + + return ret; +} + +/** + * xe_guc_id_mgr_release() - Release a range of GuC context IDs. + * @idm: the &xe_guc_id_mgr + * @start: the starting ID of GuC context range to release + * @count: number of GuC context IDs to release + */ +void xe_guc_id_mgr_release(struct xe_guc_id_mgr *idm, + unsigned int start, unsigned int count) +{ + mutex_lock(idm_mutex(idm)); + idm_release_chunk_locked(idm, start, count); + mutex_unlock(idm_mutex(idm)); +} + +static void idm_print_locked(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent) +{ + unsigned int rs, re; + + lockdep_assert_held(idm_mutex(idm)); + + drm_printf_indent(p, indent, "total %u\n", idm->total); + if (!idm->bitmap) + return; + + drm_printf_indent(p, indent, "used %u\n", idm->used); + for_each_set_bitrange(rs, re, idm->bitmap, idm->total) + drm_printf_indent(p, indent, "range %u..%u (%u)\n", rs, re - 1, re - rs); +} + +/** + * xe_guc_id_mgr_print() - Print status of GuC ID Manager. + * @idm: the &xe_guc_id_mgr to print + * @p: the &drm_printer to print to + * @indent: tab indentation level + */ +void xe_guc_id_mgr_print(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent) +{ + mutex_lock(idm_mutex(idm)); + idm_print_locked(idm, p, indent); + mutex_unlock(idm_mutex(idm)); +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_guc_id_mgr_test.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_id_mgr.h b/drivers/gpu/drm/xe/xe_guc_id_mgr.h new file mode 100644 index 000000000..368f8c80e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_id_mgr.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GUC_ID_MGR_H_ +#define _XE_GUC_ID_MGR_H_ + +struct drm_printer; +struct xe_guc_id_mgr; + +int xe_guc_id_mgr_init(struct xe_guc_id_mgr *idm, unsigned int count); + +int xe_guc_id_mgr_reserve_locked(struct xe_guc_id_mgr *idm, unsigned int count); +void xe_guc_id_mgr_release_locked(struct xe_guc_id_mgr *idm, unsigned int id, unsigned int count); + +int xe_guc_id_mgr_reserve(struct xe_guc_id_mgr *idm, unsigned int count, unsigned int retain); +void xe_guc_id_mgr_release(struct xe_guc_id_mgr *idm, unsigned int start, unsigned int count); + +void xe_guc_id_mgr_print(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c new file mode 100644 index 000000000..dc8612d76 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include +#include +#include + +#include "abi/guc_klvs_abi.h" +#include "abi/xe_driver_klvs_abi.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_klv_thresholds_set.h" + +#define make_u64(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) + +static bool is_group_key(u16 key) +{ + KUNIT_STATIC_STUB_REDIRECT(is_group_key, key); + return false; +} + +static bool is_reserved_key(u16 key) +{ + return in_range(key, GUC_KLV_RESERVED_RANGE_START, GUC_KLV_RESERVED_RANGE_LEN); +} + +/** + * xe_guc_klv_key_to_string - Convert KLV key into friendly name. + * @key: the `GuC KLV`_ key + * + * Return: name of the KLV key. + */ +const char *xe_guc_klv_key_to_string(u16 key) +{ + switch (key) { + /* GuC Global Config KLVs */ + case GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY: + return "group_scheduling_available"; + case GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_KEY: + return "num_paging_engine_instances"; + /* VGT POLICY keys */ + case GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY: + return "sched_if_idle"; + case GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY: + return "sample_period"; + case GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_KEY: + return "engine_group_config"; + case GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY: + return "reset_engine"; + /* VF CFG keys */ + case GUC_KLV_VF_CFG_GGTT_START_KEY: + return "ggtt_start"; + case GUC_KLV_VF_CFG_GGTT_SIZE_KEY: + return "ggtt_size"; + case GUC_KLV_VF_CFG_LMEM_SIZE_KEY: + return "lmem_size"; + case GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY: + return "num_contexts"; + case GUC_KLV_VF_CFG_TILE_MASK_KEY: + return "tile_mask"; + case GUC_KLV_VF_CFG_NUM_DOORBELLS_KEY: + return "num_doorbells"; + case GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY: + return "exec_quantum"; + case GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY: + return "preempt_timeout"; + case GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID_KEY: + return "begin_db_id"; + case GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_KEY: + return "begin_ctx_id"; + case GUC_KLV_VF_CFG_SCHED_PRIORITY_KEY: + return "sched_priority"; + case GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY: + return "sched_groups_exec_quantum"; + case GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY: + return "sched_groups_preempt_timeout"; + + /* VF CFG threshold keys */ +#define define_threshold_key_to_string_case(TAG, NAME, ...) \ + \ + case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG): \ + return #NAME; + + /* private: auto-generated case statements */ + MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_threshold_key_to_string_case) +#undef define_threshold_key_to_string_case + + /* driver KLVs */ + case MIGRATION_KLV_DEVICE_DEVID_KEY: + return "migration_devid"; + case MIGRATION_KLV_DEVICE_REVID_KEY: + return "migration_revid"; + + default: + if (is_reserved_key(key)) + return "(reserved)"; + return "(unknown)"; + } +} + +/** + * xe_guc_klv_print_one() - Print single `GuC KLV`_. + * @key: KLV key + * @len: KLV length (in u32 dwords) of the KLV @value + * @value: KLV value (as array of @len u32 dwords) + * @p: the &drm_printer + * + * The buffer may contain more than one KLV. + */ +void xe_guc_klv_print_one(u16 key, u16 len, const u32 *value, struct drm_printer *p) +{ + const char *name = xe_guc_klv_key_to_string(key); + + if (is_group_key(key)) { + struct drm_printer gp = drm_line_printer(p, name, 0); + + drm_printf(p, "{ key %#06x : group %u dwords } # %s\n", + key, len, name); + + /* print group recursively */ + xe_guc_klv_print(value, len, &gp); + return; + } + + switch (len) { + case 0: + drm_printf(p, "{ key %#06x : no value } # %s\n", key, name); + break; + case 1: + drm_printf(p, "{ key %#06x : 32b value %u } # %s\n", + key, value[0], name); + break; + case 2: + drm_printf(p, "{ key %#06x : 64b value %#llx } # %s\n", + key, make_u64(value[1], value[0]), name); + break; + default: + drm_printf(p, "{ key %#06x : %zu bytes %*ph } # %s\n", + key, len * sizeof(u32), (int)(len * sizeof(u32)), + value, name); + break; + } +} + +/** + * xe_guc_klv_print - Print content of the buffer with `GuC KLV`_. + * @klvs: the buffer with KLVs + * @num_dwords: number of dwords (u32) available in the buffer + * @p: the &drm_printer + * + * The buffer may contain more than one KLV. + */ +void xe_guc_klv_print(const u32 *klvs, u32 num_dwords, struct drm_printer *p) +{ + while (num_dwords >= GUC_KLV_LEN_MIN) { + u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]); + u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + + klvs += GUC_KLV_LEN_MIN; + num_dwords -= GUC_KLV_LEN_MIN; + + if (num_dwords < len) { + drm_printf(p, "{ key %#06x : truncated %zu of %zu bytes %*ph } # %s\n", + key, num_dwords * sizeof(u32), len * sizeof(u32), + (int)(num_dwords * sizeof(u32)), klvs, + xe_guc_klv_key_to_string(key)); + return; + } + + xe_guc_klv_print_one(key, len, klvs, p); + + klvs += len; + num_dwords -= len; + } + + /* we don't expect any leftovers, fix if KLV header is ever changed */ + BUILD_BUG_ON(GUC_KLV_LEN_MIN > 1); +} + +/** + * xe_guc_klv_count - Count KLVs present in the buffer. + * @klvs: the buffer with KLVs + * @num_dwords: number of dwords (u32) in the buffer + * + * Return: number of recognized KLVs or + * a negative error code if KLV buffer is truncated. + */ +int xe_guc_klv_count(const u32 *klvs, u32 num_dwords) +{ + int num_klvs = 0; + + while (num_dwords >= GUC_KLV_LEN_MIN) { + u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + + if (num_dwords < len + GUC_KLV_LEN_MIN) + break; + + klvs += GUC_KLV_LEN_MIN + len; + num_dwords -= GUC_KLV_LEN_MIN + len; + num_klvs++; + } + + return num_dwords ? -ENODATA : num_klvs; +} + +static size_t to_num_bytes(u16 dwords) +{ + return dwords * sizeof(u32); +} + +static u16 to_num_dwords(size_t size) +{ + return round_up(size, sizeof(u32)) / sizeof(u32); +} + +/** + * xe_guc_klv_encode_u32() - Encode 32-bit value as KLV. + * @klvs: the buffer where to place KLV + * @avail: number of dwords (u32) available in the buffer + * @key: key to be used + * @value: value to be encoded + * + * Return: pointer to the buffer location past the encoded KLV or + * an ERR_PTR if there was no space to encode the KLV. + */ +u32 *xe_guc_klv_encode_u32(u32 *klvs, u32 avail, u16 key, u32 value) +{ + u16 len = to_num_dwords(sizeof(u32)); + + if (IS_ERR(klvs)) + return klvs; + + if (avail < GUC_KLV_LEN_MIN + len) + return ERR_PTR(-ENOSPC); + + *klvs++ = PREP_GUC_KLV(key, len); + *klvs++ = value; + return klvs; +} + +/** + * xe_guc_klv_encode_u64() - Encode 64-bit value as KLV. + * @klvs: the buffer where to place KLV + * @avail: number of dwords (u32) available in the buffer + * @key: key to be used + * @value: value to be encoded + * + * Return: pointer to the buffer location past the encoded KLV or + * an ERR_PTR if there was no space to encode the KLV. + */ +u32 *xe_guc_klv_encode_u64(u32 *klvs, u32 avail, u16 key, u64 value) +{ + u16 len = to_num_dwords(sizeof(u64)); + + if (IS_ERR(klvs)) + return klvs; + + if (avail < GUC_KLV_LEN_MIN + len) + return ERR_PTR(-ENOSPC); + + *klvs++ = PREP_GUC_KLV(key, len); + *klvs++ = lower_32_bits(value); + *klvs++ = upper_32_bits(value); + return klvs; +} + +/** + * xe_guc_klv_encode_string() - Encode string as KLV. + * @klvs: the buffer where to place KLV + * @avail: number of dwords (u32) available in the buffer + * @key: key to be used + * @s: string to be encoded + * + * Return: pointer to the buffer location past the encoded KLV or + * an ERR_PTR if there was no space to encode the KLV. + */ +u32 *xe_guc_klv_encode_string(u32 *klvs, u32 avail, u16 key, const char *s) +{ + size_t longest = to_num_bytes(FIELD_MAX(GUC_KLV_0_LEN)); + size_t size = strnlen(s, longest) + 1; /* \0 */ + u16 len = to_num_dwords(size); + + if (IS_ERR(klvs)) + return klvs; + + if (size > longest) + return ERR_PTR(-E2BIG); + + if (avail < GUC_KLV_LEN_MIN + len) + return ERR_PTR(-ENOSPC); + + *klvs++ = PREP_GUC_KLV(key, len); + strscpy_pad((void *)klvs, s, to_num_bytes(len)); + return klvs + len; +} + +/** + * xe_guc_klv_encode_object() - Encode object using custom encoder as single KLV. + * @klvs: the buffer where to place KLV + * @avail: number of dwords (u32) available in the buffer + * @key: key to be used + * @obj: opaque object pointer + * @encoder: function pointer to the custom encoder + * + * Return: pointer to the buffer location past the encoded KLV or + * an ERR_PTR if there was no space to encode the KLV. + */ +u32 *xe_guc_klv_encode_object(u32 *klvs, u32 avail, u16 key, const void *obj, + u32 *(*encoder)(u32 *klvs, u32 avail, const void *obj)) +{ + u32 *end; + + if (IS_ERR(klvs)) + return klvs; + + if (avail < GUC_KLV_LEN_MIN) + return ERR_PTR(-ENOSPC); + + if (avail > GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN)) + avail = GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN); + + end = encoder(klvs + GUC_KLV_LEN_MIN, avail - GUC_KLV_LEN_MIN, obj); + if (IS_ERR(end)) + return end; + + if (WARN_ON(end < klvs + GUC_KLV_LEN_MIN)) + return ERR_PTR(-EPIPE); + + if (WARN_ON(end > klvs + avail)) + return ERR_PTR(-EFBIG); + + *klvs = PREP_GUC_KLV(key, end - (klvs + GUC_KLV_LEN_MIN)); + return end; +} + +/** + * xe_guc_klv_parser() - Parse and decode stream of KLVs. + * @klvs: the buffer with KLVs + * @num_dwords: number of dwords (u32) available in the buffer + * @obj: opaque pointer to be used by the @decoder function + * @decoder: pointer to the decoder function + * + * Return: The sum of all results returned by the decoder or + * an -errno on decoder or buffer failure. + */ +int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj, + int (*decoder)(void *obj, u16 key, u16 len, const u32 *value)) +{ + int total = 0; + int ret; + + while (num_dwords >= GUC_KLV_LEN_MIN) { + u16 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]); + u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + + klvs += GUC_KLV_LEN_MIN; + num_dwords -= GUC_KLV_LEN_MIN; + + if (num_dwords < len) + return -ENODATA; + + ret = decoder(obj, key, len, klvs); + if (ret < 0) + return ret; + total += ret; + + klvs += len; + num_dwords -= len; + } + + return total; +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_guc_klv_helpers_kunit.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h new file mode 100644 index 000000000..cf3b29adc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GUC_KLV_HELPERS_H_ +#define _XE_GUC_KLV_HELPERS_H_ + +#include +#include + +struct drm_printer; + +const char *xe_guc_klv_key_to_string(u16 key); + +void xe_guc_klv_print_one(u16 key, u16 len, const u32 *value, struct drm_printer *p); +void xe_guc_klv_print(const u32 *klvs, u32 num_dwords, struct drm_printer *p); +int xe_guc_klv_count(const u32 *klvs, u32 num_dwords); + +u32 *xe_guc_klv_encode_u32(u32 *klvs, u32 avail, u16 key, u32 value); +u32 *xe_guc_klv_encode_u64(u32 *klvs, u32 avail, u16 key, u64 value); +u32 *xe_guc_klv_encode_string(u32 *klvs, u32 avail, u16 key, const char *s); +u32 *xe_guc_klv_encode_object(u32 *klvs, u32 avail, u16 key, const void *obj, + u32 *(*encoder)(u32 *klvs, u32 avail, const void *obj)); + +int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj, + int (*decoder)(void *obj, u16 key, u16 len, const u32 *value)); + +/** + * PREP_GUC_KLV - Prepare KLV header value based on provided key and len. + * @key: KLV key + * @len: KLV length + * + * Return: value of the KLV header (u32). + */ +#define PREP_GUC_KLV(key, len) \ + (FIELD_PREP(GUC_KLV_0_KEY, (key)) | \ + FIELD_PREP(GUC_KLV_0_LEN, (len))) + +/** + * PREP_GUC_KLV_CONST - Prepare KLV header value based on const key and len. + * @key: const KLV key + * @len: const KLV length + * + * Return: value of the KLV header (u32). + */ +#define PREP_GUC_KLV_CONST(key, len) \ + (FIELD_PREP_CONST(GUC_KLV_0_KEY, (key)) | \ + FIELD_PREP_CONST(GUC_KLV_0_LEN, (len))) + +/** + * MAKE_GUC_KLV_KEY - Prepare KLV KEY name based on unique KLV definition tag. + * @TAG: unique tag of the KLV definition + */ +#define MAKE_GUC_KLV_KEY(TAG) CONCATENATE(CONCATENATE(GUC_KLV_, TAG), _KEY) + +/** + * MAKE_GUC_KLV_LEN - Prepare KLV LEN name based on unique KLV definition tag. + * @TAG: unique tag of the KLV definition + */ +#define MAKE_GUC_KLV_LEN(TAG) CONCATENATE(CONCATENATE(GUC_KLV_, TAG), _LEN) + +/** + * PREP_GUC_KLV_TAG - Prepare KLV header value based on unique KLV definition tag. + * @TAG: unique tag of the KLV definition + * + * Combine separate KEY and LEN definitions of the KLV identified by the TAG. + * + * Return: value of the KLV header (u32). + */ +#define PREP_GUC_KLV_TAG(TAG) \ + PREP_GUC_KLV_CONST(MAKE_GUC_KLV_KEY(TAG), MAKE_GUC_KLV_LEN(TAG)) + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set.h b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set.h new file mode 100644 index 000000000..da10cf038 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GUC_KLV_THRESHOLDS_SET_H_ +#define _XE_GUC_KLV_THRESHOLDS_SET_H_ + +#include "abi/guc_klvs_abi.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_klv_thresholds_set_types.h" + +/** + * MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY - Prepare the name of the KLV key constant. + * @TAG: unique tag of the GuC threshold KLV key. + */ +#define MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG) \ + MAKE_GUC_KLV_KEY(CONCATENATE(VF_CFG_THRESHOLD_, TAG)) + +/** + * MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN - Prepare the name of the KLV length constant. + * @TAG: unique tag of the GuC threshold KLV key. + */ +#define MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG) \ + MAKE_GUC_KLV_LEN(CONCATENATE(VF_CFG_THRESHOLD_, TAG)) + +/** + * xe_guc_klv_threshold_key_to_index - Find index of the tracked GuC threshold. + * @key: GuC threshold KLV key. + * + * This translation is automatically generated using &MAKE_XE_GUC_KLV_THRESHOLDS_SET. + * Return: index of the GuC threshold KLV or -1 if not found. + */ +static inline int xe_guc_klv_threshold_key_to_index(u32 key) +{ + switch (key) { +#define define_xe_guc_klv_threshold_key_to_index_case(TAG, ...) \ + \ + case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG): \ + return MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG); + + /* private: auto-generated case statements */ + MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_xe_guc_klv_threshold_key_to_index_case) + } + return -1; +#undef define_xe_guc_klv_threshold_key_to_index_case +} + +/** + * xe_guc_klv_threshold_index_to_key - Get tracked GuC threshold KLV key. + * @index: GuC threshold KLV index. + * + * This translation is automatically generated using &MAKE_XE_GUC_KLV_THRESHOLDS_SET. + * Return: key of the GuC threshold KLV or 0 on malformed index. + */ +static inline u32 xe_guc_klv_threshold_index_to_key(enum xe_guc_klv_threshold_index index) +{ + switch (index) { +#define define_xe_guc_klv_threshold_index_to_key_case(TAG, ...) \ + \ + case MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG): \ + return MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG); + + /* private: auto-generated case statements */ + MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_xe_guc_klv_threshold_index_to_key_case) + } + return 0; /* unreachable */ +#undef define_xe_guc_klv_threshold_index_to_key_case +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h new file mode 100644 index 000000000..45ab5a3b5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GUC_KLV_THRESHOLDS_SET_TYPES_H_ +#define _XE_GUC_KLV_THRESHOLDS_SET_TYPES_H_ + +#include "xe_args.h" + +/** + * MAKE_XE_GUC_KLV_THRESHOLDS_SET - Generate various GuC thresholds definitions. + * @define: name of the inner macro to expand. + * + * The GuC firmware is able to monitor VF's adverse activity and will notify the + * PF driver once any threshold is exceeded. + * + * This super macro allows various conversions between the GuC adverse event + * threshold KLV definitions and the driver code without repeating similar code + * or risking missing some cases. + * + * For each GuC threshold definition, the inner macro &define will be provided + * with the &TAG, that corresponds to the GuC threshold KLV key name defined by + * ABI and the associated &NAME, that may be used in code or debugfs/sysfs:: + * + * define(TAG, NAME) + * + * If required, KLVs can be labeled with GuC firmware version that added them:: + * + * define(TAG, NAME, MAJOR, MINOR) + * define(TAG, NAME, MAJOR, MINOR, PATCH) + */ +#define MAKE_XE_GUC_KLV_THRESHOLDS_SET(define) \ + define(CAT_ERR, cat_error_count) \ + define(ENGINE_RESET, engine_reset_count) \ + define(PAGE_FAULT, page_fault_count) \ + define(H2G_STORM, guc_time_us) \ + define(IRQ_STORM, irq_time_us) \ + define(DOORBELL_STORM, doorbell_time_us) \ + define(MULTI_LRC_COUNT, multi_lrc_count, 70, 53)\ + /* end */ + +/** + * XE_GUC_KLV_NUM_THRESHOLDS - Number of GuC thresholds KLVs. + * + * Calculated automatically using &MAKE_XE_GUC_KLV_THRESHOLDS_SET. + */ +#define XE_GUC_KLV_NUM_THRESHOLDS \ + (CALL_ARGS(COUNT_ARGS, MAKE_XE_GUC_KLV_THRESHOLDS_SET(ARGS_SEP_COMMA)) - 1) + +/** + * MAKE_XE_GUC_KLV_THRESHOLD_INDEX - Create enumerator name. + * @TAG: unique TAG of the enum xe_guc_klv_threshold_index. + */ +#define MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG) \ + CONCATENATE(XE_GUC_KLV_THRESHOLD_INDEX_, TAG) + +/** + * enum xe_guc_klv_threshold_index - Index of the tracked GuC threshold. + * + * This enum is automatically generated using &MAKE_XE_GUC_KLV_THRESHOLDS_SET. + * All these generated enumerators will only be used by the also generated code. + */ +enum xe_guc_klv_threshold_index { +#define define_xe_guc_klv_threshold_index_enum(TAG, ...) \ + \ + MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG), + + /* private: auto-generated enum definitions */ + MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_xe_guc_klv_threshold_index_enum) +#undef define_xe_guc_klv_threshold_index_enum +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c new file mode 100644 index 000000000..538d4df0f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_log.h" + +#include + +#include +#include + +#include "abi/guc_lfd_abi.h" +#include "regs/xe_guc_regs.h" +#include "xe_bo.h" +#include "xe_devcoredump.h" +#include "xe_force_wake.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_module.h" + +#define GUC_LOG_CHUNK_SIZE SZ_2M + +/* Magic keys define */ +#define GUC_LFD_DRIVER_KEY_STREAMING 0x8086AAAA474C5346 +#define GUC_LFD_LOG_BUFFER_MARKER_2 0xDEADFEED +#define GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2 0x8086DEAD +#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2 0xBEEFFEED +#define GUC_LFD_LOG_BUFFER_MARKER_1V2 0xCABBA9E6 +#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2 0xCABBA9F7 +#define GUC_LFD_DATA_HEADER_MAGIC 0x8086 + +/* LFD supported LIC type range */ +#define GUC_LIC_TYPE_FIRST GUC_LIC_TYPE_GUC_SW_VERSION +#define GUC_LIC_TYPE_LAST GUC_LIC_TYPE_BUILD_PLATFORM_ID +#define GUC_LFD_TYPE_FW_RANGE_FIRST GUC_LFD_TYPE_FW_VERSION +#define GUC_LFD_TYPE_FW_RANGE_LAST GUC_LFD_TYPE_BUILD_PLATFORM_ID + +#define GUC_LOG_BUFFER_STATE_HEADER_LENGTH 4096 +#define GUC_LOG_BUFFER_INIT_CONFIG 3 + +struct guc_log_buffer_entry_list { + u32 offset; + u32 rd_ptr; + u32 wr_ptr; + u32 wrap_offset; + u32 buf_size; +}; + +struct guc_lic_save { + u32 version; + /* + * Array of init config KLV values. + * Range from GUC_LOG_LIC_TYPE_FIRST to GUC_LOG_LIC_TYPE_LAST + */ + u32 values[GUC_LIC_TYPE_LAST - GUC_LIC_TYPE_FIRST + 1]; + struct guc_log_buffer_entry_list entry[GUC_LOG_BUFFER_INIT_CONFIG]; +}; + +static struct guc_log_buffer_entry_markers { + u32 key[2]; +} const entry_markers[GUC_LOG_BUFFER_INIT_CONFIG + 1] = { + {{ + GUC_LFD_LOG_BUFFER_MARKER_1V2, + GUC_LFD_LOG_BUFFER_MARKER_2 + }}, + {{ + GUC_LFD_LOG_BUFFER_MARKER_1V2, + GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2 + }}, + {{ + GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2, + GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2 + }}, + {{ + GUC_LIC_MAGIC, + (FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MAJOR, GUC_LIC_VERSION_MAJOR) | + FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MINOR, GUC_LIC_VERSION_MINOR)) + }} +}; + +static struct guc_log_lic_lfd_map { + u32 lic; + u32 lfd; +} const lic_lfd_type_map[] = { + {GUC_LIC_TYPE_GUC_SW_VERSION, GUC_LFD_TYPE_FW_VERSION}, + {GUC_LIC_TYPE_GUC_DEVICE_ID, GUC_LFD_TYPE_GUC_DEVICE_ID}, + {GUC_LIC_TYPE_TSC_FREQUENCY, GUC_LFD_TYPE_TSC_FREQUENCY}, + {GUC_LIC_TYPE_GMD_ID, GUC_LFD_TYPE_GMD_ID}, + {GUC_LIC_TYPE_BUILD_PLATFORM_ID, GUC_LFD_TYPE_BUILD_PLATFORM_ID} +}; + +static struct xe_guc * +log_to_guc(struct xe_guc_log *log) +{ + return container_of(log, struct xe_guc, log); +} + +static struct xe_gt * +log_to_gt(struct xe_guc_log *log) +{ + return container_of(log, struct xe_gt, uc.guc.log); +} + +static struct xe_device * +log_to_xe(struct xe_guc_log *log) +{ + return gt_to_xe(log_to_gt(log)); +} + +static struct xe_guc_log_snapshot *xe_guc_log_snapshot_alloc(struct xe_guc_log *log, bool atomic) +{ + struct xe_guc_log_snapshot *snapshot; + size_t remain; + int i; + + snapshot = kzalloc_obj(*snapshot, atomic ? GFP_ATOMIC : GFP_KERNEL); + if (!snapshot) + return NULL; + + /* + * NB: kmalloc has a hard limit well below the maximum GuC log buffer size. + * Also, can't use vmalloc as might be called from atomic context. So need + * to break the buffer up into smaller chunks that can be allocated. + */ + snapshot->size = xe_bo_size(log->bo); + snapshot->num_chunks = DIV_ROUND_UP(snapshot->size, GUC_LOG_CHUNK_SIZE); + + snapshot->copy = kzalloc_objs(*snapshot->copy, snapshot->num_chunks, + atomic ? GFP_ATOMIC : GFP_KERNEL); + if (!snapshot->copy) + goto fail_snap; + + remain = snapshot->size; + for (i = 0; i < snapshot->num_chunks; i++) { + size_t size = min(GUC_LOG_CHUNK_SIZE, remain); + + snapshot->copy[i] = kmalloc(size, atomic ? GFP_ATOMIC : GFP_KERNEL); + if (!snapshot->copy[i]) + goto fail_copy; + remain -= size; + } + + return snapshot; + +fail_copy: + for (i = 0; i < snapshot->num_chunks; i++) + kfree(snapshot->copy[i]); + kfree(snapshot->copy); +fail_snap: + kfree(snapshot); + return NULL; +} + +/** + * xe_guc_log_snapshot_free - free a previously captured GuC log snapshot + * @snapshot: GuC log snapshot structure + * + * Return: pointer to a newly allocated snapshot object or null if out of memory. Caller is + * responsible for calling xe_guc_log_snapshot_free when done with the snapshot. + */ +void xe_guc_log_snapshot_free(struct xe_guc_log_snapshot *snapshot) +{ + int i; + + if (!snapshot) + return; + + if (snapshot->copy) { + for (i = 0; i < snapshot->num_chunks; i++) + kfree(snapshot->copy[i]); + kfree(snapshot->copy); + } + + kfree(snapshot); +} + +/** + * xe_guc_log_snapshot_capture - create a new snapshot copy the GuC log for later dumping + * @log: GuC log structure + * @atomic: is the call inside an atomic section of some kind? + * + * Return: pointer to a newly allocated snapshot object or null if out of memory. Caller is + * responsible for calling xe_guc_log_snapshot_free when done with the snapshot. + */ +struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, bool atomic) +{ + struct xe_guc_log_snapshot *snapshot; + struct xe_device *xe = log_to_xe(log); + struct xe_guc *guc = log_to_guc(log); + struct xe_gt *gt = log_to_gt(log); + size_t remain; + int i; + + if (!log->bo) + return NULL; + + snapshot = xe_guc_log_snapshot_alloc(log, atomic); + if (!snapshot) + return NULL; + + remain = snapshot->size; + for (i = 0; i < snapshot->num_chunks; i++) { + size_t size = min(GUC_LOG_CHUNK_SIZE, remain); + + xe_map_memcpy_from(xe, snapshot->copy[i], &log->bo->vmap, + i * GUC_LOG_CHUNK_SIZE, size); + remain -= size; + } + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + snapshot->stamp = ~0ULL; + else + snapshot->stamp = xe_mmio_read64_2x32(>->mmio, GUC_PMTIMESTAMP_LO); + + snapshot->ktime = ktime_get_boottime_ns(); + snapshot->level = log->level; + snapshot->ver_found = guc->fw.versions.found[XE_UC_FW_VER_RELEASE]; + snapshot->ver_want = guc->fw.versions.wanted; + snapshot->path = guc->fw.path; + + return snapshot; +} + +/** + * xe_guc_log_snapshot_print - dump a previously saved copy of the GuC log to some useful location + * @snapshot: a snapshot of the GuC log + * @p: the printer object to output to + */ +void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p) +{ + size_t remain; + int i; + + if (!snapshot) { + drm_printf(p, "GuC log snapshot not allocated!\n"); + return; + } + + drm_printf(p, "GuC firmware: %s\n", snapshot->path); + drm_printf(p, "GuC version: %u.%u.%u (wanted %u.%u.%u)\n", + snapshot->ver_found.major, snapshot->ver_found.minor, snapshot->ver_found.patch, + snapshot->ver_want.major, snapshot->ver_want.minor, snapshot->ver_want.patch); + drm_printf(p, "Kernel timestamp: 0x%08llX [%llu]\n", snapshot->ktime, snapshot->ktime); + drm_printf(p, "GuC timestamp: 0x%08llX [%llu]\n", snapshot->stamp, snapshot->stamp); + drm_printf(p, "Log level: %u\n", snapshot->level); + + drm_printf(p, "[LOG].length: 0x%zx\n", snapshot->size); + remain = snapshot->size; + for (i = 0; i < snapshot->num_chunks; i++) { + size_t size = min(GUC_LOG_CHUNK_SIZE, remain); + const char *prefix = i ? NULL : "[LOG].data"; + char suffix = i == snapshot->num_chunks - 1 ? '\n' : 0; + + xe_print_blob_ascii85(p, prefix, suffix, snapshot->copy[i], 0, size); + remain -= size; + } +} + +static inline void lfd_output_binary(struct drm_printer *p, char *buf, int buf_size) +{ + seq_write(p->arg, buf, buf_size); +} + +static inline int xe_guc_log_add_lfd_header(struct guc_lfd_data *lfd) +{ + lfd->header = FIELD_PREP_CONST(GUC_LFD_DATA_HEADER_MASK_MAGIC, GUC_LFD_DATA_HEADER_MAGIC); + return offsetof(struct guc_lfd_data, data); +} + +static int xe_guc_log_add_typed_payload(struct drm_printer *p, u32 type, + u32 data_len, void *data) +{ + struct guc_lfd_data lfd; + int len; + + len = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, type); + /* make length DW aligned */ + lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32)); + lfd_output_binary(p, (char *)&lfd, len); + + lfd_output_binary(p, data, data_len); + len += lfd.data_count * sizeof(u32); + + return len; +} + +static inline int lic_type_to_index(u32 lic_type) +{ + XE_WARN_ON(lic_type < GUC_LIC_TYPE_FIRST || lic_type > GUC_LIC_TYPE_LAST); + + return lic_type - GUC_LIC_TYPE_FIRST; +} + +static inline int lfd_type_to_index(u32 lfd_type) +{ + int i, lic_type = 0; + + XE_WARN_ON(lfd_type < GUC_LFD_TYPE_FW_RANGE_FIRST || lfd_type > GUC_LFD_TYPE_FW_RANGE_LAST); + + for (i = 0; i < ARRAY_SIZE(lic_lfd_type_map); i++) + if (lic_lfd_type_map[i].lfd == lfd_type) + lic_type = lic_lfd_type_map[i].lic; + + /* If not found, lic_type_to_index will warning invalid type */ + return lic_type_to_index(lic_type); +} + +static int xe_guc_log_add_klv(struct drm_printer *p, u32 lfd_type, + struct guc_lic_save *config) +{ + int klv_index = lfd_type_to_index(lfd_type); + + return xe_guc_log_add_typed_payload(p, lfd_type, sizeof(u32), &config->values[klv_index]); +} + +static int xe_guc_log_add_os_id(struct drm_printer *p, u32 id) +{ + struct guc_lfd_data_os_info os_id; + struct guc_lfd_data lfd; + int len, info_len, section_len; + char *version; + u32 blank = 0; + + len = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_OS_ID); + + os_id.os_id = id; + section_len = offsetof(struct guc_lfd_data_os_info, build_version); + + version = init_utsname()->release; + info_len = strlen(version); + + /* make length DW aligned */ + lfd.data_count = DIV_ROUND_UP(section_len + info_len, sizeof(u32)); + lfd_output_binary(p, (char *)&lfd, len); + lfd_output_binary(p, (char *)&os_id, section_len); + lfd_output_binary(p, version, info_len); + + /* Padding with 0 */ + section_len = lfd.data_count * sizeof(u32) - section_len - info_len; + if (section_len) + lfd_output_binary(p, (char *)&blank, section_len); + + len += lfd.data_count * sizeof(u32); + return len; +} + +static void xe_guc_log_loop_log_init(struct guc_lic *init, struct guc_lic_save *config) +{ + struct guc_klv_generic_dw_t *p = (void *)init->data; + int i; + + for (i = 0; i < init->data_count;) { + int klv_len = FIELD_GET(GUC_KLV_0_LEN, p->kl) + 1; + int key = FIELD_GET(GUC_KLV_0_KEY, p->kl); + + if (key < GUC_LIC_TYPE_FIRST || key > GUC_LIC_TYPE_LAST) { + XE_WARN_ON(key < GUC_LIC_TYPE_FIRST || key > GUC_LIC_TYPE_LAST); + break; + } + config->values[lic_type_to_index(key)] = p->value; + i += klv_len + 1; /* Whole KLV structure length in dwords */ + p = (void *)((u32 *)p + klv_len); + } +} + +static int find_marker(u32 mark0, u32 mark1) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(entry_markers); i++) + if (mark0 == entry_markers[i].key[0] && mark1 == entry_markers[i].key[1]) + return i; + + return ARRAY_SIZE(entry_markers); +} + +static void xe_guc_log_load_lic(void *guc_log, struct guc_lic_save *config) +{ + u32 offset = GUC_LOG_BUFFER_STATE_HEADER_LENGTH; + struct guc_log_buffer_state *p = guc_log; + + config->version = p->version; + while (p->marker[0]) { + int index; + + index = find_marker(p->marker[0], p->marker[1]); + + if (index < ARRAY_SIZE(entry_markers)) { + if (index == GUC_LOG_BUFFER_INIT_CONFIG) { + /* Load log init config */ + xe_guc_log_loop_log_init((void *)p, config); + + /* LIC structure is the last */ + return; + } + config->entry[index].offset = offset; + config->entry[index].rd_ptr = p->read_ptr; + config->entry[index].wr_ptr = p->write_ptr; + config->entry[index].wrap_offset = p->wrap_offset; + config->entry[index].buf_size = p->size; + } + offset += p->size; + p++; + } +} + +static int +xe_guc_log_output_lfd_init(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + struct guc_lic_save *config) +{ + int type, len; + size_t size = 0; + + /* FW required types */ + for (type = GUC_LFD_TYPE_FW_RANGE_FIRST; type <= GUC_LFD_TYPE_FW_RANGE_LAST; type++) + size += xe_guc_log_add_klv(p, type, config); + + /* KMD required type(s) */ + len = xe_guc_log_add_os_id(p, GUC_LFD_OS_TYPE_OSID_LIN); + size += len; + + return size; +} + +static void +xe_guc_log_print_chunks(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + u32 from, u32 to) +{ + int chunk_from = from % GUC_LOG_CHUNK_SIZE; + int chunk_id = from / GUC_LOG_CHUNK_SIZE; + int to_chunk_id = to / GUC_LOG_CHUNK_SIZE; + int chunk_to = to % GUC_LOG_CHUNK_SIZE; + int pos = from; + + do { + size_t size = (to_chunk_id == chunk_id ? chunk_to : GUC_LOG_CHUNK_SIZE) - + chunk_from; + + lfd_output_binary(p, snapshot->copy[chunk_id] + chunk_from, size); + pos += size; + chunk_id++; + chunk_from = 0; + } while (pos < to); +} + +static inline int +xe_guc_log_add_log_event(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + struct guc_lic_save *config) +{ + size_t size; + u32 data_len, section_len; + struct guc_lfd_data lfd; + struct guc_log_buffer_entry_list *entry; + struct guc_lfd_data_log_events_buf events_buf; + + entry = &config->entry[GUC_LOG_TYPE_EVENT_DATA]; + + /* Skip empty log */ + if (entry->rd_ptr == entry->wr_ptr) + return 0; + + size = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_LOG_EVENTS_BUFFER); + events_buf.log_events_format_version = config->version; + + /* Adjust to log_format_buf */ + section_len = offsetof(struct guc_lfd_data_log_events_buf, log_event); + data_len = section_len; + + /* Calculate data length */ + data_len += entry->rd_ptr < entry->wr_ptr ? (entry->wr_ptr - entry->rd_ptr) : + (entry->wr_ptr + entry->wrap_offset - entry->rd_ptr); + /* make length u32 aligned */ + lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32)); + + /* Output GUC_LFD_TYPE_LOG_EVENTS_BUFFER header */ + lfd_output_binary(p, (char *)&lfd, size); + lfd_output_binary(p, (char *)&events_buf, section_len); + + /* Output data from guc log chunks directly */ + if (entry->rd_ptr < entry->wr_ptr) { + xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr, + entry->offset + entry->wr_ptr); + } else { + /* 1st, print from rd to wrap offset */ + xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr, + entry->offset + entry->wrap_offset); + + /* 2nd, print from buf start to wr */ + xe_guc_log_print_chunks(p, snapshot, entry->offset, entry->offset + entry->wr_ptr); + } + return size; +} + +static int +xe_guc_log_add_crash_dump(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + struct guc_lic_save *config) +{ + struct guc_log_buffer_entry_list *entry; + int chunk_from, chunk_id; + int from, to, i; + size_t size = 0; + u32 *buf32; + + entry = &config->entry[GUC_LOG_TYPE_CRASH_DUMP]; + + /* Skip zero sized crash dump */ + if (!entry->buf_size) + return 0; + + /* Check if crash dump section are all zero */ + from = entry->offset; + to = entry->offset + entry->buf_size; + chunk_from = from % GUC_LOG_CHUNK_SIZE; + chunk_id = from / GUC_LOG_CHUNK_SIZE; + buf32 = snapshot->copy[chunk_id] + chunk_from; + + for (i = 0; i < entry->buf_size / sizeof(u32); i++) + if (buf32[i]) + break; + + /* Buffer has non-zero data? */ + if (i < entry->buf_size / sizeof(u32)) { + struct guc_lfd_data lfd; + + size = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_FW_CRASH_DUMP); + /* Calculate data length */ + lfd.data_count = DIV_ROUND_UP(entry->buf_size, sizeof(u32)); + /* Output GUC_LFD_TYPE_FW_CRASH_DUMP header */ + lfd_output_binary(p, (char *)&lfd, size); + + /* rd/wr ptr is not used for crash dump */ + xe_guc_log_print_chunks(p, snapshot, from, to); + } + return size; +} + +static void +xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p) +{ + struct guc_lfd_file_header header; + struct guc_lic_save config; + size_t size; + + if (!snapshot || !snapshot->size) + return; + + header.magic = GUC_LFD_DRIVER_KEY_STREAMING; + header.version = FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR, + GUC_LFD_FORMAT_VERSION_MINOR) | + FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR, + GUC_LFD_FORMAT_VERSION_MAJOR); + + /* Output LFD file header */ + lfd_output_binary(p, (char *)&header, + offsetof(struct guc_lfd_file_header, stream)); + + /* Output LFD stream */ + xe_guc_log_load_lic(snapshot->copy[0], &config); + size = xe_guc_log_output_lfd_init(p, snapshot, &config); + if (!size) + return; + + xe_guc_log_add_log_event(p, snapshot, &config); + xe_guc_log_add_crash_dump(p, snapshot, &config); +} + +/** + * xe_guc_log_print_dmesg - dump a copy of the GuC log to dmesg + * @log: GuC log structure + */ +void xe_guc_log_print_dmesg(struct xe_guc_log *log) +{ + struct xe_gt *gt = log_to_gt(log); + static int g_count; + struct drm_printer ip = xe_gt_info_printer(gt); + struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); + + drm_printf(&lp, "Dumping GuC log for %ps...\n", __builtin_return_address(0)); + + xe_guc_log_print(log, &lp); + + drm_printf(&lp, "Done.\n"); +} + +/** + * xe_guc_log_print - dump a copy of the GuC log to some useful location + * @log: GuC log structure + * @p: the printer object to output to + */ +void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p) +{ + struct xe_guc_log_snapshot *snapshot; + + drm_printf(p, "**** GuC Log ****\n"); + + snapshot = xe_guc_log_snapshot_capture(log, false); + drm_printf(p, "CS reference clock: %u\n", log_to_gt(log)->info.reference_clock); + xe_guc_log_snapshot_print(snapshot, p); + xe_guc_log_snapshot_free(snapshot); +} + +/** + * xe_guc_log_print_lfd - dump a copy of the GuC log in LFD format + * @log: GuC log structure + * @p: the printer object to output to + */ +void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p) +{ + struct xe_guc_log_snapshot *snapshot; + + snapshot = xe_guc_log_snapshot_capture(log, false); + xe_guc_log_snapshot_print_lfd(snapshot, p); + xe_guc_log_snapshot_free(snapshot); +} + +int xe_guc_log_init(struct xe_guc_log *log) +{ + struct xe_device *xe = log_to_xe(log); + struct xe_tile *tile = gt_to_tile(log_to_gt(log)); + struct xe_bo *bo; + + bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + xe_map_memset(xe, &bo->vmap, 0, 0, xe_bo_size(bo)); + log->bo = bo; + log->level = xe_modparam.guc_log_level; + + return 0; +} + +ALLOW_ERROR_INJECTION(xe_guc_log_init, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_guc_check_log_buf_overflow - Check if log buffer overflowed + * @log: The log object. + * @type: The log buffer type + * @full_cnt: The count of buffer full + * + * This function will check count of buffer full against previous, mismatch + * indicate overflowed. + * Update the sampled_overflow counter, if the 4 bit counter overflowed, add + * up 16 to correct the value. + * + * Return: True if overflowed. + */ +bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_type type, + unsigned int full_cnt) +{ + unsigned int prev_full_cnt = log->stats[type].sampled_overflow; + bool overflow = false; + + if (full_cnt != prev_full_cnt) { + overflow = true; + + log->stats[type].overflow = full_cnt; + log->stats[type].sampled_overflow += full_cnt - prev_full_cnt; + + if (full_cnt < prev_full_cnt) { + /* buffer_full_cnt is a 4 bit counter */ + log->stats[type].sampled_overflow += 16; + } + xe_gt_notice(log_to_gt(log), "log buffer overflow\n"); + } + + return overflow; +} diff --git a/drivers/gpu/drm/xe/xe_guc_log.h b/drivers/gpu/drm/xe/xe_guc_log.h new file mode 100644 index 000000000..4649a2607 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_log.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_LOG_H_ +#define _XE_GUC_LOG_H_ + +#include "xe_guc_log_types.h" +#include "abi/guc_log_abi.h" + +struct drm_printer; +struct xe_device; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_16M +#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_1M +#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_2M +#elif IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_8M +#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_1M +#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_1M +#else +#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_64K +#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_16K +#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_1M +#endif + +#define GUC_LOG_SIZE (SZ_4K + \ + XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE + \ + XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE + \ + XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) + +#define XE_GUC_LOG_EVENT_DATA_OFFSET SZ_4K +#define XE_GUC_LOG_CRASH_DUMP_OFFSET (XE_GUC_LOG_EVENT_DATA_OFFSET + \ + XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE) +#define XE_GUC_LOG_STATE_CAPTURE_OFFSET (XE_GUC_LOG_CRASH_DUMP_OFFSET + \ + XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) + +/* + * While we're using plain log level in i915, GuC controls are much more... + * "elaborate"? We have a couple of bits for verbosity, separate bit for actual + * log enabling, and separate bit for default logging - which "conveniently" + * ignores the enable bit. + */ +#define GUC_LOG_LEVEL_DISABLED 0 +#define GUC_LOG_LEVEL_NON_VERBOSE 1 +#define GUC_LOG_LEVEL_IS_ENABLED(x) ((x) > GUC_LOG_LEVEL_DISABLED) +#define GUC_LOG_LEVEL_IS_VERBOSE(x) ((x) > GUC_LOG_LEVEL_NON_VERBOSE) +#define GUC_LOG_LEVEL_TO_VERBOSITY(x) ({ \ + typeof(x) _x = (x); \ + GUC_LOG_LEVEL_IS_VERBOSE(_x) ? _x - 2 : 0; \ +}) +#define GUC_VERBOSITY_TO_LOG_LEVEL(x) ((x) + 2) +#define GUC_LOG_LEVEL_MAX GUC_VERBOSITY_TO_LOG_LEVEL(GUC_LOG_VERBOSITY_MAX) + +int xe_guc_log_init(struct xe_guc_log *log); +void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p); +void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p); +void xe_guc_log_print_dmesg(struct xe_guc_log *log); +struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, bool atomic); +void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p); +void xe_guc_log_snapshot_free(struct xe_guc_log_snapshot *snapshot); + +static inline u32 +xe_guc_log_get_level(struct xe_guc_log *log) +{ + return log->level; +} + +bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, + enum guc_log_type type, + unsigned int full_cnt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_log_types.h b/drivers/gpu/drm/xe/xe_guc_log_types.h new file mode 100644 index 000000000..02851b924 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_log_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_LOG_TYPES_H_ +#define _XE_GUC_LOG_TYPES_H_ + +#include +#include "abi/guc_log_abi.h" + +#include "xe_uc_fw_types.h" + +struct xe_bo; + +/** + * struct xe_guc_log_snapshot: + * Capture of the GuC log plus various state useful for decoding the log + */ +struct xe_guc_log_snapshot { + /** @size: Size in bytes of the @copy allocation */ + size_t size; + /** @copy: Host memory copy of the log buffer for later dumping, split into chunks */ + void **copy; + /** @num_chunks: Number of chunks within @copy */ + int num_chunks; + /** @ktime: Kernel time the snapshot was taken */ + u64 ktime; + /** @stamp: GuC timestamp at which the snapshot was taken */ + u64 stamp; + /** @level: GuC log verbosity level */ + u32 level; + /** @ver_found: GuC firmware version */ + struct xe_uc_fw_version ver_found; + /** @ver_want: GuC firmware version that driver expected */ + struct xe_uc_fw_version ver_want; + /** @path: Path of GuC firmware blob */ + const char *path; +}; + +/** + * struct xe_guc_log - GuC log + */ +struct xe_guc_log { + /** @level: GuC log level */ + u32 level; + /** @bo: Xe BO for GuC log */ + struct xe_bo *bo; + /** @stats: logging related stats */ + struct { + u32 sampled_overflow; + u32 overflow; + u32 flush; + } stats[GUC_LOG_BUFFER_TYPE_MAX]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_pagefault.c b/drivers/gpu/drm/xe/xe_guc_pagefault.c new file mode 100644 index 000000000..607e32392 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pagefault.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "abi/guc_actions_abi.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_pagefault.h" +#include "xe_pagefault.h" +#include "xe_pagefault_types.h" + +static void guc_ack_fault(struct xe_pagefault *pf, int err) +{ + u32 vfid = FIELD_GET(PFD_VFID, pf->producer.msg[2]); + u32 prefetch = FIELD_GET(PFD_PREFETCH, pf->producer.msg[2]); + u32 engine_instance = FIELD_GET(PFD_ENG_INSTANCE, pf->producer.msg[0]); + u32 engine_class = FIELD_GET(PFD_ENG_CLASS, pf->producer.msg[0]); + u32 pdata = FIELD_GET(PFD_PDATA_LO, pf->producer.msg[0]) | + (FIELD_GET(PFD_PDATA_HI, pf->producer.msg[1]) << + PFD_PDATA_HI_SHIFT); + u32 asid = FIELD_GET(PFD_ASID, pf->producer.msg[1]); + u32 action[] = { + XE_GUC_ACTION_PAGE_FAULT_RES_DESC, + + FIELD_PREP(PFR_VALID, 1) | + FIELD_PREP(PFR_SUCCESS, !!err) | + FIELD_PREP(PFR_REPLY, PFR_ACCESS) | + FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) | + FIELD_PREP(PFR_ASID, asid), + + FIELD_PREP(PFR_VFID, vfid) | + FIELD_PREP(PFR_PREFETCH, err ? prefetch : 0) | + FIELD_PREP(PFR_ENG_INSTANCE, engine_instance) | + FIELD_PREP(PFR_ENG_CLASS, engine_class) | + FIELD_PREP(PFR_PDATA, pdata), + }; + struct xe_guc *guc = pf->producer.private; + + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); +} + +static const struct xe_pagefault_ops guc_pagefault_ops = { + .ack_fault = guc_ack_fault, +}; + +/** + * xe_guc_pagefault_handler() - G2H page fault handler + * @guc: GuC object + * @msg: G2H message + * @len: Length of G2H message + * + * Parse GuC to host (G2H) message into a struct xe_pagefault and forward onto + * the Xe page fault layer. + * + * Return: 0 on success, errno on failure + */ +int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_pagefault pf; + int i; + +#define GUC_PF_MSG_LEN_DW \ + (sizeof(struct xe_guc_pagefault_desc) / sizeof(u32)) + + BUILD_BUG_ON(GUC_PF_MSG_LEN_DW > XE_PAGEFAULT_PRODUCER_MSG_LEN_DW); + + if (len != GUC_PF_MSG_LEN_DW) + return -EPROTO; + + pf.gt = guc_to_gt(guc); + + /* + * XXX: These values happen to match the enum in xe_pagefault_types.h. + * If that changes, we’ll need to remap them here. + */ + pf.consumer.page_addr = ((u64)FIELD_GET(PFD_VIRTUAL_ADDR_HI, msg[3]) + << PFD_VIRTUAL_ADDR_HI_SHIFT) | + (FIELD_GET(PFD_VIRTUAL_ADDR_LO, msg[2]) << + PFD_VIRTUAL_ADDR_LO_SHIFT); + pf.consumer.asid = FIELD_GET(PFD_ASID, msg[1]); + pf.consumer.access_type = FIELD_GET(PFD_ACCESS_TYPE, msg[2]) | + (FIELD_GET(PFD_PREFETCH, msg[2]) ? XE_PAGEFAULT_ACCESS_PREFETCH : 0); + if (FIELD_GET(XE2_PFD_TRVA_FAULT, msg[0])) + pf.consumer.fault_type_level = XE_PAGEFAULT_TYPE_LEVEL_NACK; + else + pf.consumer.fault_type_level = + FIELD_PREP(XE_PAGEFAULT_LEVEL_MASK, + FIELD_GET(PFD_FAULT_LEVEL, msg[0])) | + FIELD_PREP(XE_PAGEFAULT_TYPE_MASK, + FIELD_GET(PFD_FAULT_TYPE, msg[2])); + pf.consumer.engine_class = FIELD_GET(PFD_ENG_CLASS, msg[0]); + pf.consumer.engine_instance = FIELD_GET(PFD_ENG_INSTANCE, msg[0]); + + pf.producer.private = guc; + pf.producer.ops = &guc_pagefault_ops; + for (i = 0; i < GUC_PF_MSG_LEN_DW; ++i) + pf.producer.msg[i] = msg[i]; + +#undef GUC_PF_MSG_LEN_DW + + return xe_pagefault_handler(guc_to_xe(guc), &pf); +} diff --git a/drivers/gpu/drm/xe/xe_guc_pagefault.h b/drivers/gpu/drm/xe/xe_guc_pagefault.h new file mode 100644 index 000000000..3bd599e72 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pagefault.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GUC_PAGEFAULT_H_ +#define _XE_GUC_PAGEFAULT_H_ + +#include + +struct xe_guc; + +int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c new file mode 100644 index 000000000..7cf8f4858 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -0,0 +1,1448 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_pc.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "abi/guc_actions_slpc_abi.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_idle.h" +#include "xe_gt_printk.h" +#include "xe_gt_throttle.h" +#include "xe_gt_types.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_pcode.h" +#include "xe_pm.h" +#include "xe_sriov.h" +#include "xe_wa.h" + +#define MCHBAR_MIRROR_BASE_SNB 0x140000 + +#define RP_STATE_CAP XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5998) +#define RP0_MASK REG_GENMASK(7, 0) +#define RP1_MASK REG_GENMASK(15, 8) +#define RPN_MASK REG_GENMASK(23, 16) + +#define FREQ_INFO_REC XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5ef0) +#define RPE_MASK REG_GENMASK(15, 8) +#define RPA_MASK REG_GENMASK(31, 16) + +#define GT_PERF_STATUS XE_REG(0x1381b4) +#define CAGF_MASK REG_GENMASK(19, 11) + +#define GT_FREQUENCY_MULTIPLIER 50 +#define GT_FREQUENCY_SCALER 3 + +#define LNL_MERT_FREQ_CAP 800 +#define BMG_MERT_FREQ_CAP 2133 +#define BMG_MIN_FREQ 1200 +#define BMG_MERT_FLUSH_FREQ_CAP 2600 + +#define SLPC_RESET_TIMEOUT_MS 5 /* roughly 5ms, but no need for precision */ +#define SLPC_RESET_EXTENDED_TIMEOUT_MS 1000 /* To be used only at pc_start */ +#define SLPC_ACT_FREQ_TIMEOUT_MS 100 + +/** + * DOC: GuC Power Conservation (PC) + * + * GuC Power Conservation (PC) supports multiple features for the most + * efficient and performing use of the GT when GuC submission is enabled, + * including frequency management, Render-C states management, and various + * algorithms for power balancing. + * + * Single Loop Power Conservation (SLPC) is the name given to the suite of + * connected power conservation features in the GuC firmware. The firmware + * exposes a programming interface to the host for the control of SLPC. + * + * Frequency management: + * --------------------- + * + * Xe driver enables SLPC with all of its defaults features and frequency + * selection, which varies per platform. + * + * Power profiles add another level of control to SLPC. When power saving + * profile is chosen, SLPC will use conservative thresholds to ramp frequency, + * thus saving power. Base profile is default and ensures balanced performance + * for any workload. + * + * Render-C States: + * ---------------- + * + * Render-C states is also a GuC PC feature that is now enabled in Xe for + * all platforms. + * + * Implementation details: + * ----------------------- + * The implementation for GuC Power Management features is split as follows: + * + * xe_guc_rc: Logic for handling GuC RC + * xe_gt_idle: Host side logic for RC6 and Coarse Power gating (CPG) + * xe_guc_pc: Logic for all other SLPC related features + * + * There is some cross interaction between these where host C6 will need to be + * enabled when we plan to skip GuC RC. Also, the GuC RC mode is currently + * overridden through 0x3003 which is an SLPC H2G call. + */ + +static struct xe_guc *pc_to_guc(struct xe_guc_pc *pc) +{ + return container_of(pc, struct xe_guc, pc); +} + +static struct xe_guc_ct *pc_to_ct(struct xe_guc_pc *pc) +{ + return &pc_to_guc(pc)->ct; +} + +static struct xe_gt *pc_to_gt(struct xe_guc_pc *pc) +{ + return guc_to_gt(pc_to_guc(pc)); +} + +static struct xe_device *pc_to_xe(struct xe_guc_pc *pc) +{ + return guc_to_xe(pc_to_guc(pc)); +} + +static struct iosys_map *pc_to_maps(struct xe_guc_pc *pc) +{ + return &pc->bo->vmap; +} + +#define slpc_shared_data_read(pc_, field_) \ + xe_map_rd_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \ + struct slpc_shared_data, field_) + +#define slpc_shared_data_write(pc_, field_, val_) \ + xe_map_wr_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \ + struct slpc_shared_data, field_, val_) + +#define SLPC_EVENT(id, count) \ + (FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID, id) | \ + FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, count)) + +static int wait_for_pc_state(struct xe_guc_pc *pc, + enum slpc_global_state target_state, + int timeout_ms) +{ + enum slpc_global_state state; + + xe_device_assert_mem_access(pc_to_xe(pc)); + + return poll_timeout_us(state = slpc_shared_data_read(pc, header.global_state), + state == target_state, + 20, timeout_ms * USEC_PER_MSEC, false); +} + +static int wait_for_flush_complete(struct xe_guc_pc *pc) +{ + const unsigned long timeout = msecs_to_jiffies(30); + + if (!wait_var_event_timeout(&pc->flush_freq_limit, + !atomic_read(&pc->flush_freq_limit), + timeout)) + return -ETIMEDOUT; + + return 0; +} + +static int wait_for_act_freq_max_limit(struct xe_guc_pc *pc, u32 max_limit) +{ + u32 freq; + + return poll_timeout_us(freq = xe_guc_pc_get_act_freq(pc), + freq <= max_limit, + 20, SLPC_ACT_FREQ_TIMEOUT_MS * USEC_PER_MSEC, false); +} + +static int pc_action_reset(struct xe_guc_pc *pc) +{ + struct xe_guc_ct *ct = pc_to_ct(pc); + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_RESET, 2), + xe_bo_ggtt_addr(pc->bo), + 0, + }; + int ret; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret && !(xe_device_wedged(pc_to_xe(pc)) && ret == -ECANCELED)) + xe_gt_err(pc_to_gt(pc), "GuC PC reset failed: %pe\n", + ERR_PTR(ret)); + + return ret; +} + +static int pc_action_query_task_state(struct xe_guc_pc *pc) +{ + struct xe_guc_ct *ct = pc_to_ct(pc); + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_QUERY_TASK_STATE, 2), + xe_bo_ggtt_addr(pc->bo), + 0, + }; + int ret; + + if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, + SLPC_RESET_TIMEOUT_MS)) + return -EAGAIN; + + /* Blocking here to ensure the results are ready before reading them */ + ret = xe_guc_ct_send_block(ct, action, ARRAY_SIZE(action)); + if (ret && !(xe_device_wedged(pc_to_xe(pc)) && ret == -ECANCELED)) + xe_gt_err(pc_to_gt(pc), "GuC PC query task state failed: %pe\n", + ERR_PTR(ret)); + + return ret; +} + +static int pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value) +{ + struct xe_guc_ct *ct = pc_to_ct(pc); + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_PARAMETER_SET, 2), + id, + value, + }; + int ret; + + if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, + SLPC_RESET_TIMEOUT_MS)) + return -EAGAIN; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret && !(xe_device_wedged(pc_to_xe(pc)) && ret == -ECANCELED)) + xe_gt_err(pc_to_gt(pc), "GuC PC set param[%u]=%u failed: %pe\n", + id, value, ERR_PTR(ret)); + + return ret; +} + +static int pc_action_unset_param(struct xe_guc_pc *pc, u8 id) +{ + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 1), + id, + }; + struct xe_guc_ct *ct = &pc_to_guc(pc)->ct; + int ret; + + if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, + SLPC_RESET_TIMEOUT_MS)) + return -EAGAIN; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret && !(xe_device_wedged(pc_to_xe(pc)) && ret == -ECANCELED)) + xe_gt_err(pc_to_gt(pc), "GuC PC unset param failed: %pe", + ERR_PTR(ret)); + + return ret; +} + +/** + * xe_guc_pc_action_set_param() - Set value of SLPC param + * @pc: Xe_GuC_PC instance + * @id: Param id + * @value: Value to set + * + * This function can be used to set any SLPC param. + * + * Return: 0 on Success + */ +int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value) +{ + xe_device_assert_mem_access(pc_to_xe(pc)); + return pc_action_set_param(pc, id, value); +} + +/** + * xe_guc_pc_action_unset_param() - Revert to default value + * @pc: Xe_GuC_PC instance + * @id: Param id + * + * This function can be used revert any SLPC param to its default value. + * + * Return: 0 on Success + */ +int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id) +{ + xe_device_assert_mem_access(pc_to_xe(pc)); + return pc_action_unset_param(pc, id); +} + +static u32 decode_freq(u32 raw) +{ + return DIV_ROUND_CLOSEST(raw * GT_FREQUENCY_MULTIPLIER, + GT_FREQUENCY_SCALER); +} + +static u32 encode_freq(u32 freq) +{ + return DIV_ROUND_CLOSEST(freq * GT_FREQUENCY_SCALER, + GT_FREQUENCY_MULTIPLIER); +} + +static u32 pc_get_min_freq(struct xe_guc_pc *pc) +{ + u32 freq; + + freq = FIELD_GET(SLPC_MIN_UNSLICE_FREQ_MASK, + slpc_shared_data_read(pc, task_state_data.freq)); + + return decode_freq(freq); +} + +static void pc_set_manual_rp_ctrl(struct xe_guc_pc *pc, bool enable) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 state = enable ? RPSWCTL_ENABLE : RPSWCTL_DISABLE; + + /* Allow/Disallow punit to process software freq requests */ + xe_mmio_write32(>->mmio, RP_CONTROL, state); +} + +static void pc_set_cur_freq(struct xe_guc_pc *pc, u32 freq) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 rpnswreq; + + pc_set_manual_rp_ctrl(pc, true); + + /* Req freq is in units of 16.66 Mhz */ + rpnswreq = REG_FIELD_PREP(REQ_RATIO_MASK, encode_freq(freq)); + xe_mmio_write32(>->mmio, RPNSWREQ, rpnswreq); + + /* Sleep for a small time to allow pcode to respond */ + usleep_range(100, 300); + + pc_set_manual_rp_ctrl(pc, false); +} + +static int pc_set_min_freq(struct xe_guc_pc *pc, u32 freq) +{ + /* + * Let's only check for the rpn-rp0 range. If max < min, + * min becomes a fixed request. + */ + if (freq < pc->rpn_freq || freq > pc->rp0_freq) + return -EINVAL; + + /* + * GuC policy is to elevate minimum frequency to the efficient levels + * Our goal is to have the admin choices respected. + */ + pc_action_set_param(pc, SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY, + freq < xe_guc_pc_get_rpe_freq(pc)); + + return pc_action_set_param(pc, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ, + freq); +} + +static int pc_get_max_freq(struct xe_guc_pc *pc) +{ + u32 freq; + + freq = FIELD_GET(SLPC_MAX_UNSLICE_FREQ_MASK, + slpc_shared_data_read(pc, task_state_data.freq)); + + return decode_freq(freq); +} + +static int pc_set_max_freq(struct xe_guc_pc *pc, u32 freq) +{ + /* + * Let's only check for the rpn-rp0 range. If max < min, + * min becomes a fixed request. + * Also, overclocking is not supported. + */ + if (freq < pc->rpn_freq || freq > pc->rp0_freq) + return -EINVAL; + + return pc_action_set_param(pc, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ, + freq); +} + +static u32 mtl_get_rpa_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + if (xe_gt_is_media_type(gt)) + reg = xe_mmio_read32(>->mmio, MTL_MPA_FREQUENCY); + else + reg = xe_mmio_read32(>->mmio, MTL_GT_RPA_FREQUENCY); + + return decode_freq(REG_FIELD_GET(MTL_RPA_MASK, reg)); +} + +static u32 mtl_get_rpe_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + if (xe_gt_is_media_type(gt)) + reg = xe_mmio_read32(>->mmio, MTL_MPE_FREQUENCY); + else + reg = xe_mmio_read32(>->mmio, MTL_GT_RPE_FREQUENCY); + + return decode_freq(REG_FIELD_GET(MTL_RPE_MASK, reg)); +} + +static u32 pvc_get_rpa_freq(struct xe_guc_pc *pc) +{ + /* + * For PVC we still need to use fused RP0 as the approximation for RPa + * For other platforms than PVC we get the resolved RPa directly from + * PCODE at a different register + */ + + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP); + return REG_FIELD_GET(RP0_MASK, reg) * GT_FREQUENCY_MULTIPLIER; +} + +static u32 tgl_get_rpa_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + reg = xe_mmio_read32(>->mmio, FREQ_INFO_REC); + return REG_FIELD_GET(RPA_MASK, reg) * GT_FREQUENCY_MULTIPLIER; +} + +static u32 pvc_get_rpe_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + /* + * For PVC we still need to use fused RP1 as the approximation for RPe + */ + reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP); + return REG_FIELD_GET(RP1_MASK, reg) * GT_FREQUENCY_MULTIPLIER; +} + +static u32 tgl_get_rpe_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + /* + * For other platforms than PVC, we get the resolved RPe directly from + * PCODE at a different register + */ + reg = xe_mmio_read32(>->mmio, FREQ_INFO_REC); + return REG_FIELD_GET(RPE_MASK, reg) * GT_FREQUENCY_MULTIPLIER; +} + +/** + * xe_guc_pc_get_act_freq - Get Actual running frequency + * @pc: The GuC PC + * + * Returns: The Actual running frequency. Which might be 0 if GT is in Render-C sleep state (RC6). + */ +u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + u32 freq; + + /* When in RC6, actual frequency reported will be 0. */ + if (GRAPHICS_VERx100(xe) >= 1270) { + freq = xe_mmio_read32(>->mmio, MTL_MIRROR_TARGET_WP1); + freq = REG_FIELD_GET(MTL_CAGF_MASK, freq); + } else { + freq = xe_mmio_read32(>->mmio, GT_PERF_STATUS); + freq = REG_FIELD_GET(CAGF_MASK, freq); + } + + freq = decode_freq(freq); + + return freq; +} + +static u32 get_cur_freq(struct xe_gt *gt) +{ + u32 freq; + + freq = xe_mmio_read32(>->mmio, RPNSWREQ); + freq = REG_FIELD_GET(REQ_RATIO_MASK, freq); + return decode_freq(freq); +} + +/** + * xe_guc_pc_get_cur_freq_fw - With fw held, get requested frequency + * @pc: The GuC PC + * + * Returns: the requested frequency for that GT instance + */ +u32 xe_guc_pc_get_cur_freq_fw(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + return get_cur_freq(gt); +} + +/** + * xe_guc_pc_get_cur_freq - Get Current requested frequency + * @pc: The GuC PC + * @freq: A pointer to a u32 where the freq value will be returned + * + * Returns: 0 on success, + * -EAGAIN if GuC PC not ready (likely in middle of a reset). + */ +int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq) +{ + struct xe_gt *gt = pc_to_gt(pc); + + /* + * GuC SLPC plays with cur freq request when GuCRC is enabled + * Block RC6 for a more reliable read. + */ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -ETIMEDOUT; + + *freq = get_cur_freq(gt); + + return 0; +} + +/** + * xe_guc_pc_get_rp0_freq - Get the RP0 freq + * @pc: The GuC PC + * + * Returns: RP0 freq. + */ +u32 xe_guc_pc_get_rp0_freq(struct xe_guc_pc *pc) +{ + return pc->rp0_freq; +} + +/** + * xe_guc_pc_get_rpa_freq - Get the RPa freq + * @pc: The GuC PC + * + * Returns: RPa freq. + */ +u32 xe_guc_pc_get_rpa_freq(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + + if (GRAPHICS_VERx100(xe) == 1260) + return pvc_get_rpa_freq(pc); + else if (GRAPHICS_VERx100(xe) >= 1270) + return mtl_get_rpa_freq(pc); + else + return tgl_get_rpa_freq(pc); +} + +/** + * xe_guc_pc_get_rpe_freq - Get the RPe freq + * @pc: The GuC PC + * + * Returns: RPe freq. + */ +u32 xe_guc_pc_get_rpe_freq(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + u32 freq; + + if (GRAPHICS_VERx100(xe) == 1260) + freq = pvc_get_rpe_freq(pc); + else if (GRAPHICS_VERx100(xe) >= 1270) + freq = mtl_get_rpe_freq(pc); + else + freq = tgl_get_rpe_freq(pc); + + return freq; +} + +/** + * xe_guc_pc_get_rpn_freq - Get the RPn freq + * @pc: The GuC PC + * + * Returns: RPn freq. + */ +u32 xe_guc_pc_get_rpn_freq(struct xe_guc_pc *pc) +{ + return pc->rpn_freq; +} + +static int xe_guc_pc_get_min_freq_locked(struct xe_guc_pc *pc, u32 *freq) +{ + int ret; + + lockdep_assert_held(&pc->freq_lock); + + /* Might be in the middle of a gt reset */ + if (!pc->freq_ready) + return -EAGAIN; + + ret = pc_action_query_task_state(pc); + if (ret) + return ret; + + *freq = pc_get_min_freq(pc); + + return 0; +} + +/** + * xe_guc_pc_get_min_freq - Get the min operational frequency + * @pc: The GuC PC + * @freq: A pointer to a u32 where the freq value will be returned + * + * Returns: 0 on success, + * -EAGAIN if GuC PC not ready (likely in middle of a reset). + */ +int xe_guc_pc_get_min_freq(struct xe_guc_pc *pc, u32 *freq) +{ + guard(mutex)(&pc->freq_lock); + + return xe_guc_pc_get_min_freq_locked(pc, freq); +} + +static int xe_guc_pc_set_min_freq_locked(struct xe_guc_pc *pc, u32 freq) +{ + int ret; + + lockdep_assert_held(&pc->freq_lock); + + /* Might be in the middle of a gt reset */ + if (!pc->freq_ready) + return -EAGAIN; + + ret = pc_set_min_freq(pc, freq); + if (ret) + return ret; + + pc->user_requested_min = freq; + + return 0; +} + +/** + * xe_guc_pc_set_min_freq - Set the minimal operational frequency + * @pc: The GuC PC + * @freq: The selected minimal frequency + * + * Returns: 0 on success, + * -EAGAIN if GuC PC not ready (likely in middle of a reset), + * -EINVAL if value out of bounds. + */ +int xe_guc_pc_set_min_freq(struct xe_guc_pc *pc, u32 freq) +{ + guard(mutex)(&pc->freq_lock); + + return xe_guc_pc_set_min_freq_locked(pc, freq); +} + +static int xe_guc_pc_get_max_freq_locked(struct xe_guc_pc *pc, u32 *freq) +{ + int ret; + + lockdep_assert_held(&pc->freq_lock); + + /* Might be in the middle of a gt reset */ + if (!pc->freq_ready) + return -EAGAIN; + + ret = pc_action_query_task_state(pc); + if (ret) + return ret; + + *freq = pc_get_max_freq(pc); + + return 0; +} + +/** + * xe_guc_pc_get_max_freq - Get Maximum operational frequency + * @pc: The GuC PC + * @freq: A pointer to a u32 where the freq value will be returned + * + * Returns: 0 on success, + * -EAGAIN if GuC PC not ready (likely in middle of a reset). + */ +int xe_guc_pc_get_max_freq(struct xe_guc_pc *pc, u32 *freq) +{ + guard(mutex)(&pc->freq_lock); + + return xe_guc_pc_get_max_freq_locked(pc, freq); +} + +static int xe_guc_pc_set_max_freq_locked(struct xe_guc_pc *pc, u32 freq) +{ + int ret; + + lockdep_assert_held(&pc->freq_lock); + + /* Might be in the middle of a gt reset */ + if (!pc->freq_ready) + return -EAGAIN; + + ret = pc_set_max_freq(pc, freq); + if (ret) + return ret; + + pc->user_requested_max = freq; + + return 0; +} + +/** + * xe_guc_pc_set_max_freq - Set the maximum operational frequency + * @pc: The GuC PC + * @freq: The selected maximum frequency value + * + * Returns: 0 on success, + * -EAGAIN if GuC PC not ready (likely in middle of a reset), + * -EINVAL if value out of bounds. + */ +int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq) +{ + if (XE_GT_WA(pc_to_gt(pc), 22019338487)) { + if (wait_for_flush_complete(pc) != 0) + return -EAGAIN; + } + + guard(mutex)(&pc->freq_lock); + + return xe_guc_pc_set_max_freq_locked(pc, freq); +} + +/** + * xe_guc_pc_c_status - get the current GT C state + * @pc: XE_GuC_PC instance + */ +enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg, gt_c_state; + + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { + reg = xe_mmio_read32(>->mmio, MTL_MIRROR_TARGET_WP1); + gt_c_state = REG_FIELD_GET(MTL_CC_MASK, reg); + + /* + * There are higher level sleep states that will cause this + * field to read out as its reset state, and those are only + * possible after the GT is already in C6. + */ + if (gt_c_state == MTL_CRST) + gt_c_state = GT_C6; + } else { + reg = xe_mmio_read32(>->mmio, GT_CORE_STATUS); + gt_c_state = REG_FIELD_GET(RCN_MASK, reg); + } + + switch (gt_c_state) { + case GT_C6: + return GT_IDLE_C6; + case GT_C0: + return GT_IDLE_C0; + default: + return GT_IDLE_UNKNOWN; + } +} + +/** + * xe_guc_pc_rc6_residency - rc6 residency counter + * @pc: Xe_GuC_PC instance + */ +u64 xe_guc_pc_rc6_residency(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + reg = xe_mmio_read32(>->mmio, GT_GFX_RC6); + + return reg; +} + +/** + * xe_guc_pc_mc6_residency - mc6 residency counter + * @pc: Xe_GuC_PC instance + */ +u64 xe_guc_pc_mc6_residency(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u64 reg; + + reg = xe_mmio_read32(>->mmio, MTL_MEDIA_MC6); + + return reg; +} + +static void mtl_init_fused_rp_values(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + xe_device_assert_mem_access(pc_to_xe(pc)); + + if (xe_gt_is_media_type(gt)) + reg = xe_mmio_read32(>->mmio, MTL_MEDIAP_STATE_CAP); + else + reg = xe_mmio_read32(>->mmio, MTL_RP_STATE_CAP); + + pc->rp0_freq = decode_freq(REG_FIELD_GET(MTL_RP0_CAP_MASK, reg)); + + pc->rpn_freq = decode_freq(REG_FIELD_GET(MTL_RPN_CAP_MASK, reg)); +} + +static void tgl_init_fused_rp_values(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + u32 reg; + + xe_device_assert_mem_access(pc_to_xe(pc)); + + if (xe->info.platform == XE_PVC) + reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP); + else + reg = xe_mmio_read32(>->mmio, RP_STATE_CAP); + pc->rp0_freq = REG_FIELD_GET(RP0_MASK, reg) * GT_FREQUENCY_MULTIPLIER; + pc->rpn_freq = REG_FIELD_GET(RPN_MASK, reg) * GT_FREQUENCY_MULTIPLIER; +} + +static void pc_init_fused_rp_values(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + + if (GRAPHICS_VERx100(xe) >= 1270) + mtl_init_fused_rp_values(pc); + else + tgl_init_fused_rp_values(pc); +} + +static u32 pc_max_freq_cap(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + + if (XE_GT_WA(gt, 22019338487)) { + if (xe_gt_is_media_type(gt)) + return min(LNL_MERT_FREQ_CAP, pc->rp0_freq); + else + return min(BMG_MERT_FREQ_CAP, pc->rp0_freq); + } else { + return pc->rp0_freq; + } +} + +/** + * xe_guc_pc_raise_unslice - Initialize RPx values and request a higher GT + * frequency to allow faster GuC load times + * @pc: Xe_GuC_PC instance + */ +void xe_guc_pc_raise_unslice(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + pc_set_cur_freq(pc, pc_max_freq_cap(pc)); +} + +/** + * xe_guc_pc_init_early - Initialize RPx values + * @pc: Xe_GuC_PC instance + */ +void xe_guc_pc_init_early(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + pc_init_fused_rp_values(pc); +} + +static bool pc_needs_min_freq_change(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + + if (XE_DEVICE_WA(xe, 14022085890)) + return true; + + if (xe_gt_is_media_type(gt)) + return false; + + if (xe->info.platform == XE_BATTLEMAGE || + xe->info.platform == XE_CRESCENTISLAND) + return true; + + return false; +} + +static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) +{ + int ret; + u32 min_freq; + + lockdep_assert_held(&pc->freq_lock); + + ret = pc_action_query_task_state(pc); + if (ret) + goto out; + + /* + * GuC defaults to some RPmax that is not actually achievable without + * overclocking. Let's adjust it to the Hardware RP0, which is the + * regular maximum + */ + if (pc_get_max_freq(pc) > pc->rp0_freq) { + ret = pc_set_max_freq(pc, pc->rp0_freq); + if (ret) + goto out; + } + + /* + * Same thing happens for Server platforms where min is listed as + * RPMax + */ + min_freq = pc_get_min_freq(pc); + if (min_freq > pc->rp0_freq) { + ret = pc_set_min_freq(pc, pc->rp0_freq); + if (ret) + goto out; + + min_freq = pc->rp0_freq; + } + + /* + * Setting GT RP min frequency to 1.2GHz by default for + * GT0(Graphics) Tile of BMG and CRI. + * + * While BMG G21 WA will apply min frequency for + * both GT0(Graphics) and GT1(Media) Tile. + * + * This is an active frequency, so if the device is idle + * we aren't expecting high power output across board + * + */ + if (pc_needs_min_freq_change(pc) && min_freq < BMG_MIN_FREQ) + ret = pc_set_min_freq(pc, BMG_MIN_FREQ); + +out: + return ret; +} + +static int pc_adjust_requested_freq(struct xe_guc_pc *pc) +{ + int ret = 0; + + lockdep_assert_held(&pc->freq_lock); + + if (pc->user_requested_min != 0) { + ret = pc_set_min_freq(pc, pc->user_requested_min); + if (ret) + return ret; + } + + if (pc->user_requested_max != 0) { + ret = pc_set_max_freq(pc, pc->user_requested_max); + if (ret) + return ret; + } + + return ret; +} + +static bool needs_flush_freq_limit(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + + return XE_GT_WA(gt, 22019338487) && + pc->rp0_freq > BMG_MERT_FLUSH_FREQ_CAP; +} + +/** + * xe_guc_pc_apply_flush_freq_limit() - Limit max GT freq during L2 flush + * @pc: the xe_guc_pc object + * + * As per the WA, reduce max GT frequency during L2 cache flush + */ +void xe_guc_pc_apply_flush_freq_limit(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + u32 max_freq; + int ret; + + if (!needs_flush_freq_limit(pc)) + return; + + guard(mutex)(&pc->freq_lock); + + ret = xe_guc_pc_get_max_freq_locked(pc, &max_freq); + if (!ret && max_freq > BMG_MERT_FLUSH_FREQ_CAP) { + ret = pc_set_max_freq(pc, BMG_MERT_FLUSH_FREQ_CAP); + if (ret) { + xe_gt_err_once(gt, "Failed to cap max freq on flush to %u, %pe\n", + BMG_MERT_FLUSH_FREQ_CAP, ERR_PTR(ret)); + return; + } + + atomic_set(&pc->flush_freq_limit, 1); + + /* + * If user has previously changed max freq, stash that value to + * restore later, otherwise use the current max. New user + * requests wait on flush. + */ + if (pc->user_requested_max != 0) + pc->stashed_max_freq = pc->user_requested_max; + else + pc->stashed_max_freq = max_freq; + } + + /* + * Wait for actual freq to go below the flush cap: even if the previous + * max was below cap, the current one might still be above it + */ + ret = wait_for_act_freq_max_limit(pc, BMG_MERT_FLUSH_FREQ_CAP); + if (ret) + xe_gt_err_once(gt, "Actual freq did not reduce to %u, %pe\n", + BMG_MERT_FLUSH_FREQ_CAP, ERR_PTR(ret)); +} + +/** + * xe_guc_pc_remove_flush_freq_limit() - Remove max GT freq limit after L2 flush completes. + * @pc: the xe_guc_pc object + * + * Retrieve the previous GT max frequency value. + */ +void xe_guc_pc_remove_flush_freq_limit(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + int ret = 0; + + if (!needs_flush_freq_limit(pc)) + return; + + if (!atomic_read(&pc->flush_freq_limit)) + return; + + mutex_lock(&pc->freq_lock); + + ret = pc_set_max_freq(>->uc.guc.pc, pc->stashed_max_freq); + if (ret) + xe_gt_err_once(gt, "Failed to restore max freq %u:%d", + pc->stashed_max_freq, ret); + + atomic_set(&pc->flush_freq_limit, 0); + mutex_unlock(&pc->freq_lock); + wake_up_var(&pc->flush_freq_limit); +} + +static int pc_set_mert_freq_cap(struct xe_guc_pc *pc) +{ + int ret; + + if (!XE_GT_WA(pc_to_gt(pc), 22019338487)) + return 0; + + guard(mutex)(&pc->freq_lock); + + /* + * Get updated min/max and stash them. + */ + ret = xe_guc_pc_get_min_freq_locked(pc, &pc->stashed_min_freq); + if (!ret) + ret = xe_guc_pc_get_max_freq_locked(pc, &pc->stashed_max_freq); + if (ret) + return ret; + + /* + * Ensure min and max are bound by MERT_FREQ_CAP until driver loads. + */ + ret = pc_set_min_freq(pc, min(xe_guc_pc_get_rpe_freq(pc), pc_max_freq_cap(pc))); + if (!ret) + ret = pc_set_max_freq(pc, min(pc->rp0_freq, pc_max_freq_cap(pc))); + + return ret; +} + +/** + * xe_guc_pc_restore_stashed_freq - Set min/max back to stashed values + * @pc: The GuC PC + * + * Returns: 0 on success, + * error code on failure + */ +int xe_guc_pc_restore_stashed_freq(struct xe_guc_pc *pc) +{ + int ret = 0; + + if (IS_SRIOV_VF(pc_to_xe(pc)) || pc_to_xe(pc)->info.skip_guc_pc) + return 0; + + mutex_lock(&pc->freq_lock); + ret = pc_set_max_freq(pc, pc->stashed_max_freq); + if (!ret) + ret = pc_set_min_freq(pc, pc->stashed_min_freq); + mutex_unlock(&pc->freq_lock); + + return ret; +} + +static void pc_init_pcode_freq(struct xe_guc_pc *pc) +{ + u32 min = DIV_ROUND_CLOSEST(pc->rpn_freq, GT_FREQUENCY_MULTIPLIER); + u32 max = DIV_ROUND_CLOSEST(pc->rp0_freq, GT_FREQUENCY_MULTIPLIER); + + XE_WARN_ON(xe_pcode_init_min_freq_table(gt_to_tile(pc_to_gt(pc)), min, max)); +} + +static int pc_init_freqs(struct xe_guc_pc *pc) +{ + int ret; + + mutex_lock(&pc->freq_lock); + + ret = pc_adjust_freq_bounds(pc); + if (ret) + goto out; + + ret = pc_adjust_requested_freq(pc); + if (ret) + goto out; + + pc_init_pcode_freq(pc); + + /* + * The frequencies are really ready for use only after the user + * requested ones got restored. + */ + pc->freq_ready = true; + +out: + mutex_unlock(&pc->freq_lock); + return ret; +} + +static int pc_action_set_strategy(struct xe_guc_pc *pc, u32 val) +{ + int ret = 0; + + ret = pc_action_set_param(pc, + SLPC_PARAM_STRATEGIES, + val); + + return ret; +} + +static const char *power_profile_to_string(struct xe_guc_pc *pc) +{ + switch (pc->power_profile) { + case SLPC_POWER_PROFILE_BASE: + return "base"; + case SLPC_POWER_PROFILE_POWER_SAVING: + return "power_saving"; + default: + return "invalid"; + } +} + +void xe_guc_pc_get_power_profile(struct xe_guc_pc *pc, char *profile) +{ + switch (pc->power_profile) { + case SLPC_POWER_PROFILE_BASE: + sprintf(profile, "[%s] %s\n", "base", "power_saving"); + break; + case SLPC_POWER_PROFILE_POWER_SAVING: + sprintf(profile, "%s [%s]\n", "base", "power_saving"); + break; + default: + sprintf(profile, "invalid"); + } +} + +int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) +{ + int ret = 0; + u32 val; + + if (strncmp("base", buf, strlen("base")) == 0) + val = SLPC_POWER_PROFILE_BASE; + else if (strncmp("power_saving", buf, strlen("power_saving")) == 0) + val = SLPC_POWER_PROFILE_POWER_SAVING; + else + return -EINVAL; + + guard(mutex)(&pc->freq_lock); + guard(xe_pm_runtime_noresume)(pc_to_xe(pc)); + + ret = pc_action_set_param(pc, + SLPC_PARAM_POWER_PROFILE, + val); + if (ret) + xe_gt_err_once(pc_to_gt(pc), "Failed to set power profile to %d: %pe\n", + val, ERR_PTR(ret)); + else + pc->power_profile = val; + + return ret; +} + +static int pc_action_set_dcc(struct xe_guc_pc *pc, bool enable) +{ + int ret; + + ret = pc_action_set_param(pc, + SLPC_PARAM_TASK_ENABLE_DCC, + enable); + if (!ret) + return pc_action_set_param(pc, + SLPC_PARAM_TASK_DISABLE_DCC, + !enable); + else + return ret; +} + +static int pc_modify_defaults(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + int ret = 0; + + if (xe->info.platform == XE_PANTHERLAKE) { + ret = pc_action_set_dcc(pc, false); + if (unlikely(ret)) + xe_gt_err(gt, "Failed to modify DCC default: %pe\n", ERR_PTR(ret)); + } + + return ret; +} + +/** + * xe_guc_pc_start - Start GuC's Power Conservation component + * @pc: Xe_GuC_PC instance + */ +int xe_guc_pc_start(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); + ktime_t earlier; + int ret; + + xe_gt_assert(gt, xe_device_uc_enabled(xe)); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -ETIMEDOUT; + + if (xe->info.skip_guc_pc) { + /* Request max possible since dynamic freq mgmt is not enabled */ + pc_set_cur_freq(pc, UINT_MAX); + return 0; + } + + xe_map_memset(xe, &pc->bo->vmap, 0, 0, size); + slpc_shared_data_write(pc, header.size, size); + + earlier = ktime_get(); + ret = pc_action_reset(pc); + if (ret) + return ret; + + if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, + SLPC_RESET_TIMEOUT_MS)) { + xe_gt_warn(gt, "GuC PC start taking longer than normal [freq = %dMHz (req = %dMHz), perf_limit_reasons = 0x%08X]\n", + xe_guc_pc_get_act_freq(pc), get_cur_freq(gt), + xe_gt_throttle_get_limit_reasons(gt)); + + if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, + SLPC_RESET_EXTENDED_TIMEOUT_MS)) { + xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n"); + return -EIO; + } + + xe_gt_warn(gt, "GuC PC excessive start time: %lldms", + ktime_ms_delta(ktime_get(), earlier)); + } + + ret = pc_modify_defaults(pc); + if (ret) + return ret; + + ret = pc_init_freqs(pc); + if (ret) + return ret; + + ret = pc_set_mert_freq_cap(pc); + if (ret) + return ret; + + /* Enable SLPC Optimized Strategy for compute */ + ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE); + + /* Set cached value of power_profile */ + ret = xe_guc_pc_set_power_profile(pc, power_profile_to_string(pc)); + if (unlikely(ret)) + xe_gt_err(gt, "Failed to set SLPC power profile: %pe\n", ERR_PTR(ret)); + + return ret; +} + +/** + * xe_guc_pc_stop - Stop GuC's Power Conservation component + * @pc: Xe_GuC_PC instance + */ +void xe_guc_pc_stop(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + + if (xe->info.skip_guc_pc) + return; + + mutex_lock(&pc->freq_lock); + pc->freq_ready = false; + mutex_unlock(&pc->freq_lock); +} + +/** + * xe_guc_pc_fini_hw - Finalize GuC's Power Conservation component + * @arg: opaque pointer that should point to Xe_GuC_PC instance + */ +static void xe_guc_pc_fini_hw(void *arg) +{ + struct xe_guc_pc *pc = arg; + struct xe_device *xe = pc_to_xe(pc); + + if (xe_device_wedged(xe)) + return; + + xe_guc_pc_stop(pc); + + /* Bind requested freq to mert_freq_cap before unload */ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT); + pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc))); +} + +/** + * xe_guc_pc_init - Initialize GuC's Power Conservation component + * @pc: Xe_GuC_PC instance + */ +int xe_guc_pc_init(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *bo; + u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); + int err; + + if (xe->info.skip_guc_pc) + return 0; + + err = drmm_mutex_init(&xe->drm, &pc->freq_lock); + if (err) + return err; + + bo = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + pc->bo = bo; + + pc->power_profile = SLPC_POWER_PROFILE_BASE; + + return devm_add_action_or_reset(xe->drm.dev, xe_guc_pc_fini_hw, pc); +} + +static const char *pc_get_state_string(struct xe_guc_pc *pc) +{ + switch (slpc_shared_data_read(pc, header.global_state)) { + case SLPC_GLOBAL_STATE_NOT_RUNNING: + return "not running"; + case SLPC_GLOBAL_STATE_INITIALIZING: + return "initializing"; + case SLPC_GLOBAL_STATE_RESETTING: + return "resetting"; + case SLPC_GLOBAL_STATE_RUNNING: + return "running"; + case SLPC_GLOBAL_STATE_SHUTTING_DOWN: + return "shutting down"; + case SLPC_GLOBAL_STATE_ERROR: + return "error"; + default: + return "unknown"; + } +} + +/** + * xe_guc_pc_print - Print GuC's Power Conservation information for debug + * @pc: Xe_GuC_PC instance + * @p: drm_printer + */ +void xe_guc_pc_print(struct xe_guc_pc *pc, struct drm_printer *p) +{ + drm_printf(p, "SLPC Shared Data Header:\n"); + drm_printf(p, "\tSize: %x\n", slpc_shared_data_read(pc, header.size)); + drm_printf(p, "\tGlobal State: %s\n", pc_get_state_string(pc)); + + if (pc_action_query_task_state(pc)) + return; + + drm_printf(p, "\nSLPC Tasks Status:\n"); + drm_printf(p, "\tGTPERF enabled: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_GTPERF_TASK_ENABLED)); + drm_printf(p, "\tDCC enabled: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_DCC_TASK_ENABLED)); + drm_printf(p, "\tDCC in use: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_IN_DCC)); + drm_printf(p, "\tBalancer enabled: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_BALANCER_ENABLED)); + drm_printf(p, "\tIBC enabled: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_IBC_TASK_ENABLED)); + drm_printf(p, "\tBalancer IA LMT enabled: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_BALANCER_IA_LMT_ENABLED)); + drm_printf(p, "\tBalancer IA LMT active: %s\n", + str_yes_no(slpc_shared_data_read(pc, task_state_data.status) & + SLPC_BALANCER_IA_LMT_ACTIVE)); +} diff --git a/drivers/gpu/drm/xe/xe_guc_pc.h b/drivers/gpu/drm/xe/xe_guc_pc.h new file mode 100644 index 000000000..1025a2b15 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pc.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_PC_H_ +#define _XE_GUC_PC_H_ + +#include + +struct xe_guc_pc; +struct drm_printer; + +int xe_guc_pc_init(struct xe_guc_pc *pc); +int xe_guc_pc_start(struct xe_guc_pc *pc); +void xe_guc_pc_stop(struct xe_guc_pc *pc); +void xe_guc_pc_print(struct xe_guc_pc *pc, struct drm_printer *p); +int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value); +int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id); + +u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc); +int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq); +u32 xe_guc_pc_get_cur_freq_fw(struct xe_guc_pc *pc); +u32 xe_guc_pc_get_rp0_freq(struct xe_guc_pc *pc); +u32 xe_guc_pc_get_rpa_freq(struct xe_guc_pc *pc); +u32 xe_guc_pc_get_rpe_freq(struct xe_guc_pc *pc); +u32 xe_guc_pc_get_rpn_freq(struct xe_guc_pc *pc); +int xe_guc_pc_get_min_freq(struct xe_guc_pc *pc, u32 *freq); +int xe_guc_pc_set_min_freq(struct xe_guc_pc *pc, u32 freq); +int xe_guc_pc_get_max_freq(struct xe_guc_pc *pc, u32 *freq); +int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq); +int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf); +void xe_guc_pc_get_power_profile(struct xe_guc_pc *pc, char *profile); + +enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc); +u64 xe_guc_pc_rc6_residency(struct xe_guc_pc *pc); +u64 xe_guc_pc_mc6_residency(struct xe_guc_pc *pc); +void xe_guc_pc_init_early(struct xe_guc_pc *pc); +int xe_guc_pc_restore_stashed_freq(struct xe_guc_pc *pc); +void xe_guc_pc_raise_unslice(struct xe_guc_pc *pc); +void xe_guc_pc_apply_flush_freq_limit(struct xe_guc_pc *pc); +void xe_guc_pc_remove_flush_freq_limit(struct xe_guc_pc *pc); + +#endif /* _XE_GUC_PC_H_ */ diff --git a/drivers/gpu/drm/xe/xe_guc_pc_types.h b/drivers/gpu/drm/xe/xe_guc_pc_types.h new file mode 100644 index 000000000..711bbcdcb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pc_types.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_PC_TYPES_H_ +#define _XE_GUC_PC_TYPES_H_ + +#include +#include + +/** + * struct xe_guc_pc - GuC Power Conservation (PC) + */ +struct xe_guc_pc { + /** @bo: GGTT buffer object that is shared with GuC PC */ + struct xe_bo *bo; + /** @flush_freq_limit: 1 when max freq changes are limited by driver */ + atomic_t flush_freq_limit; + /** @rp0_freq: HW RP0 frequency - The Maximum one */ + u32 rp0_freq; + /** @rpn_freq: HW RPN frequency - The Minimum one */ + u32 rpn_freq; + /** @user_requested_min: Stash the minimum requested freq by user */ + u32 user_requested_min; + /** @user_requested_max: Stash the maximum requested freq by user */ + u32 user_requested_max; + /** @stashed_min_freq: Stash the current minimum freq */ + u32 stashed_min_freq; + /** @stashed_max_freq: Stash the current maximum freq */ + u32 stashed_max_freq; + /** @freq_lock: Let's protect the frequencies */ + struct mutex freq_lock; + /** @freq_ready: Only handle freq changes, if they are really ready */ + bool freq_ready; + /** @power_profile: Base or power_saving profile */ + u32 power_profile; +}; + +#endif /* _XE_GUC_PC_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_guc_rc.c b/drivers/gpu/drm/xe/xe_guc_rc.c new file mode 100644 index 000000000..99fa127b2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_rc.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include + +#include "abi/guc_actions_slpc_abi.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_idle.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_pc.h" +#include "xe_guc_rc.h" +#include "xe_pm.h" + +/** + * DOC: GuC RC (Render C-states) + * + * GuC handles the GT transition to deeper C-states in conjunction with Pcode. + * GuC RC can be enabled independently of the frequency component in SLPC, + * which is also controlled by GuC. + * + * This file will contain all H2G related logic for handling Render C-states. + * There are some calls to xe_gt_idle, where we enable host C6 when GuC RC is + * skipped. GuC RC is mostly independent of xe_guc_pc with the exception of + * functions that override the mode for which we have to rely on the SLPC H2G + * calls. + */ + +static int guc_action_setup_gucrc(struct xe_guc *guc, u32 control) +{ + u32 action[] = { + GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC, + control, + }; + int ret; + + ret = xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); + if (ret && !(xe_device_wedged(guc_to_xe(guc)) && ret == -ECANCELED)) + xe_gt_err(guc_to_gt(guc), + "GuC RC setup %s(%u) failed (%pe)\n", + control == GUCRC_HOST_CONTROL ? "HOST_CONTROL" : + control == GUCRC_FIRMWARE_CONTROL ? "FIRMWARE_CONTROL" : + "UNKNOWN", control, ERR_PTR(ret)); + return ret; +} + +/** + * xe_guc_rc_disable() - Disable GuC RC + * @guc: Xe GuC instance + * + * Disables GuC RC by taking control of RC6 back from GuC. + */ +void xe_guc_rc_disable(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + + if (!xe->info.skip_guc_pc && xe->info.platform != XE_PVC) + if (guc_action_setup_gucrc(guc, GUCRC_HOST_CONTROL)) + return; + + xe_gt_WARN_ON(gt, xe_gt_idle_disable_c6(gt)); +} + +static void xe_guc_rc_fini_hw(void *arg) +{ + struct xe_guc *guc = arg; + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + + if (xe_device_wedged(xe)) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + xe_guc_rc_disable(guc); +} + +/** + * xe_guc_rc_init() - Init GuC RC + * @guc: Xe GuC instance + * + * Add callback action for GuC RC + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_rc_init(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + + xe_gt_assert(gt, xe_device_uc_enabled(xe)); + + return devm_add_action_or_reset(xe->drm.dev, xe_guc_rc_fini_hw, guc); +} + +/** + * xe_guc_rc_enable() - Enable GuC RC feature if applicable + * @guc: Xe GuC instance + * + * Enables GuC RC feature. + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_rc_enable(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + + xe_gt_assert(gt, xe_device_uc_enabled(xe)); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -ETIMEDOUT; + + if (xe->info.platform == XE_PVC) { + xe_guc_rc_disable(guc); + return 0; + } + + if (xe->info.skip_guc_pc) { + xe_gt_idle_enable_c6(gt); + return 0; + } + + return guc_action_setup_gucrc(guc, GUCRC_FIRMWARE_CONTROL); +} diff --git a/drivers/gpu/drm/xe/xe_guc_rc.h b/drivers/gpu/drm/xe/xe_guc_rc.h new file mode 100644 index 000000000..b083fc364 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_rc.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_GUC_RC_H_ +#define _XE_GUC_RC_H_ + +struct xe_guc; +enum slpc_gucrc_mode; + +int xe_guc_rc_init(struct xe_guc *guc); +int xe_guc_rc_enable(struct xe_guc *guc); +void xe_guc_rc_disable(struct xe_guc *guc); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_relay.c b/drivers/gpu/drm/xe/xe_guc_relay.c new file mode 100644 index 000000000..eed0a750d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_relay.c @@ -0,0 +1,971 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include + +#include +#include + +#include "abi/guc_actions_sriov_abi.h" +#include "abi/guc_relay_actions_abi.h" +#include "abi/guc_relay_communication_abi.h" + +#include "xe_assert.h" +#include "xe_device_types.h" +#include "xe_gt_sriov_printk.h" +#include "xe_gt_sriov_pf_service.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_hxg_helpers.h" +#include "xe_guc_relay.h" +#include "xe_guc_relay_types.h" +#include "xe_sriov.h" + +/* + * How long should we wait for the response? + * XXX this value is subject for the profiling. + */ +#define RELAY_TIMEOUT_MSEC (2500) + +static void relays_worker_fn(struct work_struct *w); + +static struct xe_guc *relay_to_guc(struct xe_guc_relay *relay) +{ + return container_of(relay, struct xe_guc, relay); +} + +static struct xe_guc_ct *relay_to_ct(struct xe_guc_relay *relay) +{ + return &relay_to_guc(relay)->ct; +} + +static struct xe_gt *relay_to_gt(struct xe_guc_relay *relay) +{ + return guc_to_gt(relay_to_guc(relay)); +} + +static struct xe_device *relay_to_xe(struct xe_guc_relay *relay) +{ + return gt_to_xe(relay_to_gt(relay)); +} + +#define XE_RELAY_DIAG_RATELIMIT_INTERVAL (10 * HZ) +#define XE_RELAY_DIAG_RATELIMIT_BURST 10 + +#define relay_ratelimit_printk(relay, _level, fmt...) ({ \ + typeof(relay) _r = (relay); \ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) || \ + ___ratelimit(&_r->diag_ratelimit, "xe_guc_relay")) \ + xe_gt_sriov_##_level(relay_to_gt(_r), "relay: " fmt); \ +}) + +#define relay_assert(relay, condition) xe_gt_assert(relay_to_gt(relay), condition) +#define relay_notice(relay, msg...) relay_ratelimit_printk((relay), notice, msg) +#define relay_debug(relay, msg...) relay_ratelimit_printk((relay), dbg_verbose, msg) + +static int relay_get_totalvfs(struct xe_guc_relay *relay) +{ + struct xe_device *xe = relay_to_xe(relay); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + KUNIT_STATIC_STUB_REDIRECT(relay_get_totalvfs, relay); + return IS_SRIOV_VF(xe) ? 0 : pci_sriov_get_totalvfs(pdev); +} + +static bool relay_is_ready(struct xe_guc_relay *relay) +{ + return mempool_initialized(&relay->pool); +} + +static u32 relay_get_next_rid(struct xe_guc_relay *relay) +{ + u32 rid; + + spin_lock(&relay->lock); + rid = ++relay->last_rid; + spin_unlock(&relay->lock); + + return rid; +} + +/** + * struct relay_transaction - internal data used to handle transactions + * + * Relation between struct relay_transaction members:: + * + * <-------------------- GUC_CTB_MAX_DWORDS --------------> + * <-------- GUC_RELAY_MSG_MAX_LEN ---> + * <--- offset ---> <--- request_len -------> + * +----------------+-------------------------+----------+--+ + * | | | | | + * +----------------+-------------------------+----------+--+ + * ^ ^ + * / / + * request_buf request + * + * <-------------------- GUC_CTB_MAX_DWORDS --------------> + * <-------- GUC_RELAY_MSG_MAX_LEN ---> + * <--- offset ---> <--- response_len ---> + * +----------------+----------------------+-------------+--+ + * | | | | | + * +----------------+----------------------+-------------+--+ + * ^ ^ + * / / + * response_buf response + */ +struct relay_transaction { + /** + * @incoming: indicates whether this transaction represents an incoming + * request from the remote VF/PF or this transaction + * represents outgoing request to the remote VF/PF. + */ + bool incoming; + + /** + * @remote: PF/VF identifier of the origin (or target) of the relay + * request message. + */ + u32 remote; + + /** @rid: identifier of the VF/PF relay message. */ + u32 rid; + + /** + * @request: points to the inner VF/PF request message, copied to the + * #response_buf starting at #offset. + */ + u32 *request; + + /** @request_len: length of the inner VF/PF request message. */ + u32 request_len; + + /** + * @response: points to the placeholder buffer where inner VF/PF + * response will be located, for outgoing transaction + * this could be caller's buffer (if provided) otherwise + * it points to the #response_buf starting at #offset. + */ + u32 *response; + + /** + * @response_len: length of the inner VF/PF response message (only + * if #status is 0), initially set to the size of the + * placeholder buffer where response message will be + * copied. + */ + u32 response_len; + + /** + * @offset: offset to the start of the inner VF/PF relay message inside + * buffers; this offset is equal the length of the outer GuC + * relay header message. + */ + u32 offset; + + /** + * @request_buf: buffer with VF/PF request message including outer + * transport message. + */ + u32 request_buf[GUC_CTB_MAX_DWORDS]; + + /** + * @response_buf: buffer with VF/PF response message including outer + * transport message. + */ + u32 response_buf[GUC_CTB_MAX_DWORDS]; + + /** + * @reply: status of the reply, 0 means that data pointed by the + * #response is valid. + */ + int reply; + + /** @done: completion of the outgoing transaction. */ + struct completion done; + + /** @link: transaction list link */ + struct list_head link; +}; + +static u32 prepare_pf2guc(u32 *msg, u32 target, u32 rid) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_PF2GUC_RELAY_TO_VF); + msg[1] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID, target); + msg[2] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID, rid); + + return PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN; +} + +static u32 prepare_vf2guc(u32 *msg, u32 rid) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_VF2GUC_RELAY_TO_PF); + msg[1] = FIELD_PREP(VF2GUC_RELAY_TO_PF_REQUEST_MSG_1_RELAY_ID, rid); + + return VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN; +} + +static struct relay_transaction * +__relay_get_transaction(struct xe_guc_relay *relay, bool incoming, u32 remote, u32 rid, + const u32 *action, u32 action_len, u32 *resp, u32 resp_size) +{ + struct relay_transaction *txn; + + relay_assert(relay, action_len >= GUC_RELAY_MSG_MIN_LEN); + relay_assert(relay, action_len <= GUC_RELAY_MSG_MAX_LEN); + relay_assert(relay, !(!!resp ^ !!resp_size)); + relay_assert(relay, resp_size <= GUC_RELAY_MSG_MAX_LEN); + relay_assert(relay, resp_size == 0 || resp_size >= GUC_RELAY_MSG_MIN_LEN); + + if (unlikely(!relay_is_ready(relay))) + return ERR_PTR(-ENODEV); + + /* + * For incoming requests we can't use GFP_KERNEL as those are delivered + * with CTB lock held which is marked as used in the reclaim path. + * Btw, that's one of the reason why we use mempool here! + */ + txn = mempool_alloc(&relay->pool, incoming ? GFP_ATOMIC : GFP_NOWAIT); + if (!txn) + return ERR_PTR(-ENOMEM); + + txn->incoming = incoming; + txn->remote = remote; + txn->rid = rid; + txn->offset = remote ? + prepare_pf2guc(incoming ? txn->response_buf : txn->request_buf, remote, rid) : + prepare_vf2guc(incoming ? txn->response_buf : txn->request_buf, rid); + + relay_assert(relay, txn->offset); + relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->request_buf)); + relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->response_buf)); + + txn->request = txn->request_buf + txn->offset; + memcpy(&txn->request_buf[txn->offset], action, sizeof(u32) * action_len); + txn->request_len = action_len; + + txn->response = resp ?: txn->response_buf + txn->offset; + txn->response_len = resp_size ?: GUC_RELAY_MSG_MAX_LEN; + txn->reply = -ENOMSG; + INIT_LIST_HEAD(&txn->link); + init_completion(&txn->done); + + return txn; +} + +static struct relay_transaction * +relay_new_transaction(struct xe_guc_relay *relay, u32 target, const u32 *action, u32 len, + u32 *resp, u32 resp_size) +{ + u32 rid = relay_get_next_rid(relay); + + return __relay_get_transaction(relay, false, target, rid, action, len, resp, resp_size); +} + +static struct relay_transaction * +relay_new_incoming_transaction(struct xe_guc_relay *relay, u32 origin, u32 rid, + const u32 *action, u32 len) +{ + return __relay_get_transaction(relay, true, origin, rid, action, len, NULL, 0); +} + +static void relay_release_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn) +{ + relay_assert(relay, list_empty(&txn->link)); + + txn->offset = 0; + txn->response = NULL; + txn->reply = -ESTALE; + mempool_free(txn, &relay->pool); +} + +static int relay_send_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn) +{ + u32 len = txn->incoming ? txn->response_len : txn->request_len; + u32 *buf = txn->incoming ? txn->response_buf : txn->request_buf; + u32 *msg = buf + txn->offset; + int ret; + + relay_assert(relay, txn->offset); + relay_assert(relay, txn->offset + len <= GUC_CTB_MAX_DWORDS); + relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN); + relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN); + + relay_debug(relay, "sending %s.%u to %u = %*ph\n", + guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])), + txn->rid, txn->remote, (int)sizeof(u32) * len, msg); + + ret = xe_guc_ct_send_block(relay_to_ct(relay), buf, len + txn->offset); + + if (unlikely(ret > 0)) { + relay_notice(relay, "Unexpected data=%d from GuC, wrong ABI?\n", ret); + ret = -EPROTO; + } + if (unlikely(ret < 0)) { + relay_notice(relay, "Failed to send %s.%x to GuC (%pe) %*ph ...\n", + guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, buf[0])), + FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, buf[0]), + ERR_PTR(ret), (int)sizeof(u32) * txn->offset, buf); + relay_notice(relay, "Failed to send %s.%u to %u (%pe) %*ph\n", + guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])), + txn->rid, txn->remote, ERR_PTR(ret), (int)sizeof(u32) * len, msg); + } + + return ret; +} + +static void __fini_relay(struct drm_device *drm, void *arg) +{ + struct xe_guc_relay *relay = arg; + + mempool_exit(&relay->pool); +} + +/** + * xe_guc_relay_init - Initialize a &xe_guc_relay + * @relay: the &xe_guc_relay to initialize + * + * Initialize remaining members of &xe_guc_relay that may depend + * on the SR-IOV mode. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_relay_init(struct xe_guc_relay *relay) +{ + const int XE_RELAY_MEMPOOL_MIN_NUM = 1; + struct xe_device *xe = relay_to_xe(relay); + int err; + + relay_assert(relay, !relay_is_ready(relay)); + + if (!IS_SRIOV(xe)) + return 0; + + spin_lock_init(&relay->lock); + INIT_WORK(&relay->worker, relays_worker_fn); + INIT_LIST_HEAD(&relay->pending_relays); + INIT_LIST_HEAD(&relay->incoming_actions); + ratelimit_state_init(&relay->diag_ratelimit, + XE_RELAY_DIAG_RATELIMIT_INTERVAL, + XE_RELAY_DIAG_RATELIMIT_BURST); + + err = mempool_init_kmalloc_pool(&relay->pool, XE_RELAY_MEMPOOL_MIN_NUM + + relay_get_totalvfs(relay), + sizeof(struct relay_transaction)); + if (err) + return err; + + relay_debug(relay, "using mempool with %d elements\n", relay->pool.min_nr); + + return drmm_add_action_or_reset(&xe->drm, __fini_relay, relay); +} +ALLOW_ERROR_INJECTION(xe_guc_relay_init, ERRNO); /* See xe_pci_probe() */ + +static u32 to_relay_error(int err) +{ + /* XXX: assume that relay errors match errno codes */ + return err < 0 ? -err : GUC_RELAY_ERROR_UNDISCLOSED; +} + +static int from_relay_error(u32 error) +{ + /* XXX: assume that relay errors match errno codes */ + return error ? -error : -ENODATA; +} + +static u32 sanitize_relay_error(u32 error) +{ + /* XXX TBD if generic error codes will be allowed */ + if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG)) + error = GUC_RELAY_ERROR_UNDISCLOSED; + return error; +} + +static u32 sanitize_relay_error_hint(u32 hint) +{ + /* XXX TBD if generic error codes will be allowed */ + if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG)) + hint = 0; + return hint; +} + +static u32 prepare_error_reply(u32 *msg, u32 error, u32 hint) +{ + msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_FAILURE) | + FIELD_PREP(GUC_HXG_FAILURE_MSG_0_HINT, hint) | + FIELD_PREP(GUC_HXG_FAILURE_MSG_0_ERROR, error); + + XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_ERROR, error)); + XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_HINT, hint)); + + return GUC_HXG_FAILURE_MSG_LEN; +} + +static void relay_testonly_nop(struct xe_guc_relay *relay) +{ + KUNIT_STATIC_STUB_REDIRECT(relay_testonly_nop, relay); +} + +static int relay_send_message_and_wait(struct xe_guc_relay *relay, + struct relay_transaction *txn, + u32 *buf, u32 buf_size) +{ + unsigned long timeout = msecs_to_jiffies(RELAY_TIMEOUT_MSEC); + u32 *msg = &txn->request_buf[txn->offset]; + u32 len = txn->request_len; + u32 type, action, data0; + int ret; + long n; + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); + action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); + data0 = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]); + + relay_debug(relay, "%s.%u to %u action %#x:%u\n", + guc_hxg_type_to_string(type), + txn->rid, txn->remote, action, data0); + + /* list ordering does not need to match RID ordering */ + spin_lock(&relay->lock); + list_add_tail(&txn->link, &relay->pending_relays); + spin_unlock(&relay->lock); + +resend: + ret = relay_send_transaction(relay, txn); + if (unlikely(ret < 0)) + goto unlink; + +wait: + n = wait_for_completion_timeout(&txn->done, timeout); + if (unlikely(n == 0 && txn->reply)) { + ret = -ETIME; + goto unlink; + } + + relay_debug(relay, "%u.%u reply %d after %u msec\n", + txn->remote, txn->rid, txn->reply, jiffies_to_msecs(timeout - n)); + if (unlikely(txn->reply)) { + reinit_completion(&txn->done); + if (txn->reply == -EAGAIN) + goto resend; + if (txn->reply == -EBUSY) { + relay_testonly_nop(relay); + goto wait; + } + if (txn->reply > 0) + ret = from_relay_error(txn->reply); + else + ret = txn->reply; + goto unlink; + } + + relay_debug(relay, "%u.%u response %*ph\n", txn->remote, txn->rid, + (int)sizeof(u32) * txn->response_len, txn->response); + relay_assert(relay, txn->response_len >= GUC_RELAY_MSG_MIN_LEN); + ret = txn->response_len; + +unlink: + spin_lock(&relay->lock); + list_del_init(&txn->link); + spin_unlock(&relay->lock); + + if (unlikely(ret < 0)) { + relay_notice(relay, "Unsuccessful %s.%u %#x:%u to %u (%pe) %*ph\n", + guc_hxg_type_to_string(type), txn->rid, + action, data0, txn->remote, ERR_PTR(ret), + (int)sizeof(u32) * len, msg); + } + + return ret; +} + +static int relay_send_to(struct xe_guc_relay *relay, u32 target, + const u32 *msg, u32 len, u32 *buf, u32 buf_size) +{ + struct relay_transaction *txn; + int ret; + + relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN); + relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN); + relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_HOST); + relay_assert(relay, guc_hxg_type_is_action(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]))); + + if (unlikely(!relay_is_ready(relay))) + return -ENODEV; + + txn = relay_new_transaction(relay, target, msg, len, buf, buf_size); + if (IS_ERR(txn)) + return PTR_ERR(txn); + + switch (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])) { + case GUC_HXG_TYPE_REQUEST: + ret = relay_send_message_and_wait(relay, txn, buf, buf_size); + break; + case GUC_HXG_TYPE_FAST_REQUEST: + relay_assert(relay, !GUC_HXG_TYPE_FAST_REQUEST); + fallthrough; + case GUC_HXG_TYPE_EVENT: + ret = relay_send_transaction(relay, txn); + break; + default: + ret = -EINVAL; + break; + } + + relay_release_transaction(relay, txn); + return ret; +} + +#ifdef CONFIG_PCI_IOV +/** + * xe_guc_relay_send_to_vf - Send a message to the VF. + * @relay: the &xe_guc_relay which will send the message + * @target: target VF number + * @msg: request message to be sent + * @len: length of the request message (in dwords, can't be 0) + * @buf: placeholder for the response message + * @buf_size: size of the response message placeholder (in dwords) + * + * This function can only be used by the driver running in the SR-IOV PF mode. + * + * Return: Non-negative response length (in dwords) or + * a negative error code on failure. + */ +int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target, + const u32 *msg, u32 len, u32 *buf, u32 buf_size) +{ + relay_assert(relay, IS_SRIOV_PF(relay_to_xe(relay))); + + return relay_send_to(relay, target, msg, len, buf, buf_size); +} +#endif + +/** + * xe_guc_relay_send_to_pf - Send a message to the PF. + * @relay: the &xe_guc_relay which will send the message + * @msg: request message to be sent + * @len: length of the message (in dwords, can't be 0) + * @buf: placeholder for the response message + * @buf_size: size of the response message placeholder (in dwords) + * + * This function can only be used by driver running in SR-IOV VF mode. + * + * Return: Non-negative response length (in dwords) or + * a negative error code on failure. + */ +int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay, + const u32 *msg, u32 len, u32 *buf, u32 buf_size) +{ + relay_assert(relay, IS_SRIOV_VF(relay_to_xe(relay))); + + return relay_send_to(relay, PFID, msg, len, buf, buf_size); +} + +static int relay_handle_reply(struct xe_guc_relay *relay, u32 origin, + u32 rid, int reply, const u32 *msg, u32 len) +{ + struct relay_transaction *pending; + int err = -ESRCH; + + spin_lock(&relay->lock); + list_for_each_entry(pending, &relay->pending_relays, link) { + if (pending->remote != origin || pending->rid != rid) { + relay_debug(relay, "%u.%u still awaits response\n", + pending->remote, pending->rid); + continue; + } + err = 0; /* found! */ + if (reply == 0) { + if (len > pending->response_len) { + reply = -ENOBUFS; + err = -ENOBUFS; + } else { + memcpy(pending->response, msg, 4 * len); + pending->response_len = len; + } + } + pending->reply = reply; + complete_all(&pending->done); + break; + } + spin_unlock(&relay->lock); + + return err; +} + +static int relay_handle_failure(struct xe_guc_relay *relay, u32 origin, + u32 rid, const u32 *msg, u32 len) +{ + int error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[0]); + u32 hint __maybe_unused = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[0]); + + relay_assert(relay, len); + relay_debug(relay, "%u.%u error %#x (%pe) hint %u debug %*ph\n", + origin, rid, error, ERR_PTR(-error), hint, 4 * (len - 1), msg + 1); + + return relay_handle_reply(relay, origin, rid, error ?: -EREMOTEIO, NULL, 0); +} + +static int relay_testloop_action_handler(struct xe_guc_relay *relay, u32 origin, + const u32 *msg, u32 len, u32 *response, u32 size) +{ + static ktime_t last_reply = 0; + u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); + u32 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); + u32 opcode = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]); + ktime_t now = ktime_get(); + bool busy; + int ret; + + relay_assert(relay, guc_hxg_type_is_action(type)); + relay_assert(relay, action == GUC_RELAY_ACTION_VFXPF_TESTLOOP); + + if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) + return -ECONNREFUSED; + + if (!last_reply) + last_reply = now; + busy = ktime_before(now, ktime_add_ms(last_reply, 2 * RELAY_TIMEOUT_MSEC)); + if (!busy) + last_reply = now; + + switch (opcode) { + case VFXPF_TESTLOOP_OPCODE_NOP: + if (type == GUC_HXG_TYPE_EVENT) + return 0; + return guc_hxg_msg_encode_success(response, 0); + case VFXPF_TESTLOOP_OPCODE_BUSY: + if (type == GUC_HXG_TYPE_EVENT) + return -EPROTO; + msleep(RELAY_TIMEOUT_MSEC / 8); + if (busy) + return -EINPROGRESS; + return guc_hxg_msg_encode_success(response, 0); + case VFXPF_TESTLOOP_OPCODE_RETRY: + if (type == GUC_HXG_TYPE_EVENT) + return -EPROTO; + msleep(RELAY_TIMEOUT_MSEC / 8); + if (busy) + return guc_hxg_msg_encode_retry(response, 0); + return guc_hxg_msg_encode_success(response, 0); + case VFXPF_TESTLOOP_OPCODE_ECHO: + if (type == GUC_HXG_TYPE_EVENT) + return -EPROTO; + if (size < len) + return -ENOBUFS; + ret = guc_hxg_msg_encode_success(response, len); + memcpy(response + ret, msg + ret, (len - ret) * sizeof(u32)); + return len; + case VFXPF_TESTLOOP_OPCODE_FAIL: + return -EHWPOISON; + default: + break; + } + + relay_notice(relay, "Unexpected action %#x opcode %#x\n", action, opcode); + return -EBADRQC; +} + +static int relay_action_handler(struct xe_guc_relay *relay, u32 origin, + const u32 *msg, u32 len, u32 *response, u32 size) +{ + struct xe_gt *gt = relay_to_gt(relay); + u32 type; + int ret; + + relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN); + + if (FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]) == GUC_RELAY_ACTION_VFXPF_TESTLOOP) + return relay_testloop_action_handler(relay, origin, msg, len, response, size); + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); + relay_assert(relay, guc_hxg_type_is_action(type)); + + if (IS_SRIOV_PF(relay_to_xe(relay))) { + if (type == GUC_HXG_TYPE_REQUEST) + ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, + response, size); + else + ret = -EOPNOTSUPP; + } else { + ret = -EOPNOTSUPP; + } + if (type == GUC_HXG_TYPE_EVENT) + relay_assert(relay, ret <= 0); + + return ret; +} + +static struct relay_transaction *relay_dequeue_transaction(struct xe_guc_relay *relay) +{ + struct relay_transaction *txn; + + spin_lock(&relay->lock); + txn = list_first_entry_or_null(&relay->incoming_actions, struct relay_transaction, link); + if (txn) + list_del_init(&txn->link); + spin_unlock(&relay->lock); + + return txn; +} + +static void relay_process_incoming_action(struct xe_guc_relay *relay) +{ + struct relay_transaction *txn; + bool again = false; + u32 type; + int ret; + + txn = relay_dequeue_transaction(relay); + if (!txn) + return; + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, txn->request_buf[txn->offset]); + + ret = relay_action_handler(relay, txn->remote, + txn->request_buf + txn->offset, txn->request_len, + txn->response_buf + txn->offset, + ARRAY_SIZE(txn->response_buf) - txn->offset); + + if (ret == -EINPROGRESS) { + again = true; + ret = guc_hxg_msg_encode_busy(txn->response_buf + txn->offset, 0); + } + + if (ret > 0) { + txn->response_len = ret; + ret = relay_send_transaction(relay, txn); + } + + if (ret < 0) { + u32 error = to_relay_error(ret); + + relay_notice(relay, "Failed to handle %s.%u from %u (%pe) %*ph\n", + guc_hxg_type_to_string(type), txn->rid, txn->remote, + ERR_PTR(ret), 4 * txn->request_len, txn->request_buf + txn->offset); + + txn->response_len = prepare_error_reply(txn->response_buf + txn->offset, + txn->remote ? + sanitize_relay_error(error) : error, + txn->remote ? + sanitize_relay_error_hint(-ret) : -ret); + ret = relay_send_transaction(relay, txn); + again = false; + } + + if (again) { + spin_lock(&relay->lock); + list_add(&txn->link, &relay->incoming_actions); + spin_unlock(&relay->lock); + return; + } + + if (unlikely(ret < 0)) + relay_notice(relay, "Failed to process action.%u (%pe) %*ph\n", + txn->rid, ERR_PTR(ret), 4 * txn->request_len, + txn->request_buf + txn->offset); + + relay_release_transaction(relay, txn); +} + +static bool relay_needs_worker(struct xe_guc_relay *relay) +{ + bool is_empty; + + spin_lock(&relay->lock); + is_empty = list_empty(&relay->incoming_actions); + spin_unlock(&relay->lock); + + return !is_empty; + +} + +static void relay_kick_worker(struct xe_guc_relay *relay) +{ + KUNIT_STATIC_STUB_REDIRECT(relay_kick_worker, relay); + queue_work(relay_to_xe(relay)->sriov.wq, &relay->worker); +} + +static void relays_worker_fn(struct work_struct *w) +{ + struct xe_guc_relay *relay = container_of(w, struct xe_guc_relay, worker); + + relay_process_incoming_action(relay); + + if (relay_needs_worker(relay)) + relay_kick_worker(relay); +} + +static int relay_queue_action_msg(struct xe_guc_relay *relay, u32 origin, u32 rid, + const u32 *msg, u32 len) +{ + struct relay_transaction *txn; + + txn = relay_new_incoming_transaction(relay, origin, rid, msg, len); + if (IS_ERR(txn)) + return PTR_ERR(txn); + + spin_lock(&relay->lock); + list_add_tail(&txn->link, &relay->incoming_actions); + spin_unlock(&relay->lock); + + relay_kick_worker(relay); + return 0; +} + +static int relay_process_msg(struct xe_guc_relay *relay, u32 origin, u32 rid, + const u32 *msg, u32 len) +{ + u32 type; + int err; + + if (unlikely(len < GUC_HXG_MSG_MIN_LEN)) + return -EPROTO; + + if (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) != GUC_HXG_ORIGIN_HOST) + return -EPROTO; + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); + relay_debug(relay, "received %s.%u from %u = %*ph\n", + guc_hxg_type_to_string(type), rid, origin, 4 * len, msg); + + switch (type) { + case GUC_HXG_TYPE_REQUEST: + case GUC_HXG_TYPE_FAST_REQUEST: + case GUC_HXG_TYPE_EVENT: + err = relay_queue_action_msg(relay, origin, rid, msg, len); + break; + case GUC_HXG_TYPE_RESPONSE_SUCCESS: + err = relay_handle_reply(relay, origin, rid, 0, msg, len); + break; + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: + err = relay_handle_reply(relay, origin, rid, -EBUSY, NULL, 0); + break; + case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + err = relay_handle_reply(relay, origin, rid, -EAGAIN, NULL, 0); + break; + case GUC_HXG_TYPE_RESPONSE_FAILURE: + err = relay_handle_failure(relay, origin, rid, msg, len); + break; + default: + err = -EBADRQC; + } + + if (unlikely(err)) + relay_notice(relay, "Failed to process %s.%u from %u (%pe) %*ph\n", + guc_hxg_type_to_string(type), rid, origin, + ERR_PTR(err), 4 * len, msg); + + return err; +} + +/** + * xe_guc_relay_process_guc2vf - Handle relay notification message from the GuC. + * @relay: the &xe_guc_relay which will handle the message + * @msg: message to be handled + * @len: length of the message (in dwords) + * + * This function will handle relay messages received from the GuC. + * + * This function is can only be used if driver is running in SR-IOV mode. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len) +{ + u32 rid; + + relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN); + relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC); + relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT); + relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) == + XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF); + + if (unlikely(!IS_SRIOV_VF(relay_to_xe(relay)) && !kunit_get_current_test())) + return -EPERM; + + if (unlikely(!relay_is_ready(relay))) + return -ENODEV; + + if (unlikely(len < GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN)) + return -EPROTO; + + if (unlikely(len > GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN)) + return -EMSGSIZE; + + if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0]))) + return -EPFNOSUPPORT; + + rid = FIELD_GET(GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID, msg[1]); + + return relay_process_msg(relay, PFID, rid, + msg + GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN, + len - GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN); +} + +#ifdef CONFIG_PCI_IOV +/** + * xe_guc_relay_process_guc2pf - Handle relay notification message from the GuC. + * @relay: the &xe_guc_relay which will handle the message + * @msg: message to be handled + * @len: length of the message (in dwords) + * + * This function will handle relay messages received from the GuC. + * + * This function can only be used if driver is running in SR-IOV PF mode. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len) +{ + u32 origin, rid; + int err; + + relay_assert(relay, len >= GUC_HXG_EVENT_MSG_MIN_LEN); + relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC); + relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT); + relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) == + XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF); + + if (unlikely(!IS_SRIOV_PF(relay_to_xe(relay)) && !kunit_get_current_test())) + return -EPERM; + + if (unlikely(!relay_is_ready(relay))) + return -ENODEV; + + if (unlikely(len < GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN)) + return -EPROTO; + + if (unlikely(len > GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN)) + return -EMSGSIZE; + + if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0]))) + return -EPFNOSUPPORT; + + origin = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID, msg[1]); + rid = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID, msg[2]); + + if (unlikely(origin > relay_get_totalvfs(relay))) + return -ENOENT; + + err = relay_process_msg(relay, origin, rid, + msg + GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN, + len - GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN); + + return err; +} +#endif + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_guc_relay_test.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_relay.h b/drivers/gpu/drm/xe/xe_guc_relay.h new file mode 100644 index 000000000..385429aa1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_relay.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GUC_RELAY_H_ +#define _XE_GUC_RELAY_H_ + +#include +#include + +struct xe_guc_relay; + +int xe_guc_relay_init(struct xe_guc_relay *relay); + +int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay, + const u32 *msg, u32 len, u32 *buf, u32 buf_size); + +int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len); + +#ifdef CONFIG_PCI_IOV +int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target, + const u32 *msg, u32 len, u32 *buf, u32 buf_size); +int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len); +#else +static inline int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target, + const u32 *msg, u32 len, u32 *buf, u32 buf_size) +{ + return -ENODEV; +} +static inline int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len) +{ + return -ENODEV; +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_relay_types.h b/drivers/gpu/drm/xe/xe_guc_relay_types.h new file mode 100644 index 000000000..b99a66864 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_relay_types.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GUC_RELAY_TYPES_H_ +#define _XE_GUC_RELAY_TYPES_H_ + +#include +#include +#include +#include + +/** + * struct xe_guc_relay - Data used by the VF-PF Relay Communication over GuC. + */ +struct xe_guc_relay { + /** @lock: protects all internal data. */ + spinlock_t lock; + + /** @worker: dispatches incoming action messages. */ + struct work_struct worker; + + /** @pending_relays: list of sent requests that await a response. */ + struct list_head pending_relays; + + /** @incoming_actions: list of incoming relay action messages to process. */ + struct list_head incoming_actions; + + /** @pool: pool of the relay message buffers. */ + mempool_t pool; + + /** @last_rid: last Relay-ID used while sending a message. */ + u32 last_rid; + + /** @diag_ratelimit: ratelimit state used to throttle diagnostics messages. */ + struct ratelimit_state diag_ratelimit; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c new file mode 100644 index 000000000..864859bd0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -0,0 +1,3736 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_guc_submit.h" + +#include +#include +#include +#include + +#include +#include + +#include "abi/guc_actions_abi.h" +#include "abi/guc_actions_slpc_abi.h" +#include "abi/guc_klvs_abi.h" +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_devcoredump.h" +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_gpu_scheduler.h" +#include "xe_gt.h" +#include "xe_gt_clock.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_guc_capture.h" +#include "xe_guc_ct.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_guc_id_mgr.h" +#include "xe_guc_klv_helpers.h" +#include "xe_guc_submit_types.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_mocs.h" +#include "xe_module.h" +#include "xe_pm.h" +#include "xe_ring_ops_types.h" +#include "xe_sched_job.h" +#include "xe_sleep.h" +#include "xe_trace.h" +#include "xe_uc_fw.h" +#include "xe_vm.h" + +#define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6 + +static int guc_submit_reset_prepare(struct xe_guc *guc); + +static struct xe_guc * +exec_queue_to_guc(struct xe_exec_queue *q) +{ + return &q->gt->uc.guc; +} + +/* + * Helpers for engine state, using an atomic as some of the bits can transition + * as the same time (e.g. a suspend can be happning at the same time as schedule + * engine done being processed). + */ +#define EXEC_QUEUE_STATE_REGISTERED (1 << 0) +#define EXEC_QUEUE_STATE_ENABLED (1 << 1) +#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) +#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) +#define EXEC_QUEUE_STATE_DESTROYED (1 << 4) +#define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) +#define EXEC_QUEUE_STATE_RESET (1 << 6) +#define EXEC_QUEUE_STATE_KILLED (1 << 7) +#define EXEC_QUEUE_STATE_WEDGED (1 << 8) +#define EXEC_QUEUE_STATE_BANNED (1 << 9) +#define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 10) + +static bool exec_queue_registered(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; +} + +static void set_exec_queue_registered(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); +} + +static void clear_exec_queue_registered(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); +} + +static bool exec_queue_enabled(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; +} + +static void set_exec_queue_enabled(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); +} + +static void clear_exec_queue_enabled(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); +} + +static bool exec_queue_pending_enable(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; +} + +static void set_exec_queue_pending_enable(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); +} + +static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); +} + +static bool exec_queue_pending_disable(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; +} + +static void set_exec_queue_pending_disable(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); +} + +static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); +} + +static bool exec_queue_destroyed(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; +} + +static void set_exec_queue_destroyed(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); +} + +static void clear_exec_queue_destroyed(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); +} + +static bool exec_queue_banned(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; +} + +static void set_exec_queue_banned(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); +} + +static void clear_exec_queue_banned(struct xe_exec_queue *q) +{ + atomic_andnot(EXEC_QUEUE_STATE_BANNED, &q->guc->state); +} + +static bool exec_queue_suspended(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; +} + +static void set_exec_queue_suspended(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); +} + +static void clear_exec_queue_suspended(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); +} + +static bool exec_queue_reset(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; +} + +static void set_exec_queue_reset(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); +} + +static bool exec_queue_killed(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; +} + +static void set_exec_queue_killed(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); +} + +static bool exec_queue_wedged(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; +} + +static void set_exec_queue_wedged(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); +} + +static bool exec_queue_pending_resume(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_RESUME; +} + +static void set_exec_queue_pending_resume(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); +} + +static void clear_exec_queue_pending_resume(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); +} + +static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) +{ + return (atomic_read(&q->guc->state) & + (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | + EXEC_QUEUE_STATE_BANNED)); +} + +static void guc_submit_sw_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc *guc = arg; + struct xe_gt *gt = guc_to_gt(guc); + + xe_gt_assert(gt, xa_empty(&guc->submission_state.exec_queue_lookup)); + + xa_destroy(&guc->submission_state.exec_queue_lookup); +} + +static void guc_submit_fini(void *arg) +{ + struct xe_guc *guc = arg; + struct xe_exec_queue *q; + unsigned long index; + + /* Drop any wedged queue refs */ + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + if (exec_queue_wedged(q)) { + mutex_unlock(&guc->submission_state.lock); + xe_exec_queue_put(q); + mutex_lock(&guc->submission_state.lock); + } + } + mutex_unlock(&guc->submission_state.lock); + + /* Forcefully kill any remaining exec queues */ + xe_guc_ct_stop(&guc->ct); + guc_submit_reset_prepare(guc); + xe_guc_softreset(guc); + xe_guc_submit_stop(guc); + xe_uc_fw_sanitize(&guc->fw); + xe_guc_submit_pause_abort(guc); +} + +static const struct xe_exec_queue_ops guc_exec_queue_ops; + +static void primelockdep(struct xe_guc *guc) +{ + if (!IS_ENABLED(CONFIG_LOCKDEP)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + + mutex_lock(&guc->submission_state.lock); + mutex_unlock(&guc->submission_state.lock); + + fs_reclaim_release(GFP_KERNEL); +} + +/** + * xe_guc_submit_init() - Initialize GuC submission. + * @guc: the &xe_guc to initialize + * @num_ids: number of GuC context IDs to use + * + * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all + * GuC context IDs supported by the GuC firmware should be used for submission. + * + * Only VF drivers will have to provide explicit number of GuC context IDs + * that they can use for submission. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + int err; + + err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); + if (err) + return err; + + err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); + if (err) + return err; + + gt->exec_queue_ops = &guc_exec_queue_ops; + + xa_init(&guc->submission_state.exec_queue_lookup); + + primelockdep(guc); + + guc->submission_state.initialized = true; + + err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc); + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc); +} + +/* + * Given that we want to guarantee enough RCS throughput to avoid missing + * frames, we set the yield policy to 20% of each 80ms interval. + */ +#define RC_YIELD_DURATION 80 /* in ms */ +#define RC_YIELD_RATIO 20 /* in percent */ +static u32 *emit_render_compute_yield_klv(u32 *emit) +{ + *emit++ = PREP_GUC_KLV_TAG(SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD); + *emit++ = RC_YIELD_DURATION; + *emit++ = RC_YIELD_RATIO; + + return emit; +} + +#define SCHEDULING_POLICY_MAX_DWORDS 16 +static int guc_init_global_schedule_policy(struct xe_guc *guc) +{ + u32 data[SCHEDULING_POLICY_MAX_DWORDS]; + u32 *emit = data; + u32 count = 0; + int ret; + + if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0)) + return 0; + + *emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV; + + if (CCS_INSTANCES(guc_to_gt(guc))) + emit = emit_render_compute_yield_klv(emit); + + count = emit - data; + if (count > 1) { + xe_assert(guc_to_xe(guc), count <= SCHEDULING_POLICY_MAX_DWORDS); + + ret = xe_guc_ct_send_block(&guc->ct, data, count); + if (ret < 0) { + xe_gt_err(guc_to_gt(guc), + "failed to enable GuC scheduling policies: %pe\n", + ERR_PTR(ret)); + return ret; + } + } + + return 0; +} + +int xe_guc_submit_enable(struct xe_guc *guc) +{ + int ret; + + ret = guc_init_global_schedule_policy(guc); + if (ret) + return ret; + + guc->submission_state.enabled = true; + + return 0; +} + +void xe_guc_submit_disable(struct xe_guc *guc) +{ + guc->submission_state.enabled = false; +} + +static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, + int count) +{ + int i; + + mutex_lock(&guc->submission_state.lock); + + for (i = 0; i < count; ++i) + xa_erase(&guc->submission_state.exec_queue_lookup, + q->guc->id + i); + + xe_guc_id_mgr_release_locked(&guc->submission_state.idm, + q->guc->id, q->width); + + mutex_unlock(&guc->submission_state.lock); +} + +static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) +{ + int ret, i; + + mutex_lock(&guc->submission_state.lock); + ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, + q->width); + mutex_unlock(&guc->submission_state.lock); + if (ret < 0) + return ret; + + q->guc->id = ret; + + /* Reserve empty slots. */ + for (i = 0; i < q->width; ++i) { + ret = xa_insert(&guc->submission_state.exec_queue_lookup, + q->guc->id + i, NULL, GFP_KERNEL); + if (ret) + goto err_release; + } + + return 0; + +err_release: + __release_guc_id(guc, q, i); + + return ret; +} + +static void publish_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) +{ + int i; + + lockdep_assert_held(&guc->submission_state.lock); + + for (i = 0; i < q->width; ++i) { + void *old; + + old = xa_store(&guc->submission_state.exec_queue_lookup, + q->guc->id + i, q, GFP_NOWAIT); + XE_WARN_ON(old || xa_is_err(old)); + } +} + +static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) +{ + __release_guc_id(guc, q, q->width); +} + +struct exec_queue_policy { + u32 count; + struct guc_update_exec_queue_policy h2g; +}; + +static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) +{ + size_t bytes = sizeof(policy->h2g.header) + + (sizeof(policy->h2g.klv[0]) * policy->count); + + return bytes / sizeof(u32); +} + +static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, + u16 guc_id) +{ + policy->h2g.header.action = + XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; + policy->h2g.header.guc_id = guc_id; + policy->count = 0; +} + +#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ +static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ + u32 data) \ +{ \ + XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ +\ + policy->h2g.klv[policy->count].kl = \ + FIELD_PREP(GUC_KLV_0_KEY, \ + GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ + FIELD_PREP(GUC_KLV_0_LEN, 1); \ + policy->h2g.klv[policy->count].value = data; \ + policy->count++; \ +} + +MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) +MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) +MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) +MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY) +#undef MAKE_EXEC_QUEUE_POLICY_ADD + +static const int xe_exec_queue_prio_to_guc[] = { + [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, + [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, + [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, + [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, +}; + +static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) +{ + struct exec_queue_policy policy; + enum xe_exec_queue_priority prio = q->sched_props.priority; + u32 timeslice_us = q->sched_props.timeslice_us; + u32 slpc_exec_queue_freq_req = 0; + u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; + + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) && + !xe_exec_queue_is_multi_queue_secondary(q)); + + if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) + slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; + + __guc_exec_queue_policy_start_klv(&policy, q->guc->id); + __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); + __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); + __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); + __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy, + slpc_exec_queue_freq_req); + + xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, + __guc_exec_queue_policy_action_size(&policy), 0, 0); +} + +static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) +{ + struct exec_queue_policy policy; + + xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q)); + + __guc_exec_queue_policy_start_klv(&policy, q->guc->id); + __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); + + xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, + __guc_exec_queue_policy_action_size(&policy), 0, 0); +} + +static bool vf_recovery(struct xe_guc *guc) +{ + return xe_gt_recovery_pending(guc_to_gt(guc)); +} + +static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + + /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ + wake_up_all(&xe->ufence_wq); + + xe_sched_tdr_queue_imm(&q->guc->sched); +} + +static void xe_guc_exec_queue_group_stop(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq, *next; + LIST_HEAD(tmp); + + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), + xe_exec_queue_is_multi_queue(q)); + + mutex_lock(&group->list_lock); + + /* + * Stop all future queues being from executing while group is stopped. + */ + group->stopped = true; + + list_for_each_entry_safe(eq, next, &group->list, multi_queue.link) + /* + * Refcount prevents an attempted removal from &group->list, + * temporary list allows safe iteration after dropping + * &group->list_lock. + */ + if (xe_exec_queue_get_unless_zero(eq)) + list_move_tail(&eq->multi_queue.link, &tmp); + + mutex_unlock(&group->list_lock); + + /* We cannot stop under list lock without getting inversions */ + xe_sched_submission_stop(&primary->guc->sched); + list_for_each_entry(eq, &tmp, multi_queue.link) + xe_sched_submission_stop(&eq->guc->sched); + + mutex_lock(&group->list_lock); + list_for_each_entry_safe(eq, next, &tmp, multi_queue.link) { + /* + * Corner where we got banned while stopping and not on + * &group->list + */ + if (READ_ONCE(group->banned)) + xe_guc_exec_queue_trigger_cleanup(eq); + + list_move_tail(&eq->multi_queue.link, &group->list); + xe_exec_queue_put(eq); + } + mutex_unlock(&group->list_lock); +} + +static void xe_guc_exec_queue_group_start(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), + xe_exec_queue_is_multi_queue(q)); + + xe_sched_submission_start(&primary->guc->sched); + + mutex_lock(&group->list_lock); + group->stopped = false; + list_for_each_entry(eq, &group->list, multi_queue.link) + xe_sched_submission_start(&eq->guc->sched); + mutex_unlock(&group->list_lock); +} + +static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), + xe_exec_queue_is_multi_queue(q)); + + /* Group banned, skip timeout check in TDR */ + WRITE_ONCE(group->banned, true); + xe_guc_exec_queue_trigger_cleanup(primary); + + mutex_lock(&group->list_lock); + list_for_each_entry(eq, &group->list, multi_queue.link) + xe_guc_exec_queue_trigger_cleanup(eq); + mutex_unlock(&group->list_lock); +} + +static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) +{ + if (xe_exec_queue_is_multi_queue(q)) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + /* Group banned, skip timeout check in TDR */ + WRITE_ONCE(group->banned, true); + + set_exec_queue_reset(primary); + if (!exec_queue_banned(primary)) + xe_guc_exec_queue_trigger_cleanup(primary); + + mutex_lock(&group->list_lock); + list_for_each_entry(eq, &group->list, multi_queue.link) { + set_exec_queue_reset(eq); + if (!exec_queue_banned(eq)) + xe_guc_exec_queue_trigger_cleanup(eq); + } + mutex_unlock(&group->list_lock); + } else { + set_exec_queue_reset(q); + if (!exec_queue_banned(q)) + xe_guc_exec_queue_trigger_cleanup(q); + } +} + +static void set_exec_queue_group_banned(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + /* Ban all queues of the multi-queue group */ + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), + xe_exec_queue_is_multi_queue(q)); + set_exec_queue_banned(primary); + + mutex_lock(&group->list_lock); + list_for_each_entry(eq, &group->list, multi_queue.link) + set_exec_queue_banned(eq); + mutex_unlock(&group->list_lock); +} + +/* Helper for context registration H2G */ +struct guc_ctxt_registration_info { + u32 flags; + u32 context_idx; + u32 engine_class; + u32 engine_submit_mask; + u32 wq_desc_lo; + u32 wq_desc_hi; + u32 wq_base_lo; + u32 wq_base_hi; + u32 wq_size; + u32 cgp_lo; + u32 cgp_hi; + u32 hwlrca_lo; + u32 hwlrca_hi; +}; + +#define parallel_read(xe_, map_, field_) \ + xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ + field_) +#define parallel_write(xe_, map_, field_, val_) \ + xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ + field_, val_) + +/** + * DOC: Multi Queue Group GuC interface + * + * The multi queue group coordination between KMD and GuC is through a software + * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page + * allocated in the global GTT. + * + * CGP format: + * + * +-----------+---------------------------+---------------------------------------------+ + * | DWORD | Name | Description | + * +-----------+---------------------------+---------------------------------------------+ + * | 0 | Version | Bits [15:8]=Major ver, [7:0]=Minor ver | + * +-----------+---------------------------+---------------------------------------------+ + * | 1..15 | RESERVED | MBZ | + * +-----------+---------------------------+---------------------------------------------+ + * | 16 | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0 | + * +-----------+---------------------------+---------------------------------------------+ + * | 17 | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32 | + * +-----------+---------------------------+---------------------------------------------+ + * | 18..31 | RESERVED | MBZ | + * +-----------+---------------------------+---------------------------------------------+ + * | 32 | Q0CD_DW0 | Queue 0 context LRC descriptor lower DWORD | + * +-----------+---------------------------+---------------------------------------------+ + * | 33 | Q0ContextIndex | Context ID for Queue 0 | + * +-----------+---------------------------+---------------------------------------------+ + * | 34 | Q1CD_DW0 | Queue 1 context LRC descriptor lower DWORD | + * +-----------+---------------------------+---------------------------------------------+ + * | 35 | Q1ContextIndex | Context ID for Queue 1 | + * +-----------+---------------------------+---------------------------------------------+ + * | ... |... | ... | + * +-----------+---------------------------+---------------------------------------------+ + * | 158 | Q63CD_DW0 | Queue 63 context LRC descriptor lower DWORD | + * +-----------+---------------------------+---------------------------------------------+ + * | 159 | Q63ContextIndex | Context ID for Queue 63 | + * +-----------+---------------------------+---------------------------------------------+ + * | 160..1024 | RESERVED | MBZ | + * +-----------+---------------------------+---------------------------------------------+ + * + * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified + * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies + * the CGP address. When the secondary queues are added to the group, the CGP is + * updated with entry for that queue and GuC is notified through the H2G interface + * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages + * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also + * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any + * error in the CGP. Only one of these CGP update messages can be outstanding + * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW* + * fields indicate which queue entry is being updated in the CGP. + * + * The primary queue (Q0) represents the multi queue group context in GuC and + * submission on any queue of the group must be through Q0 GuC interface only. + * + * As it is not required to register secondary queues with GuC, the secondary queue + * context ids in the CGP are populated with Q0 context id. + */ + +#define CGP_VERSION_MAJOR_SHIFT 8 + +static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe, + struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 guc_id = group->primary->guc->id; + + /* Currently implementing CGP version 1.0 */ + xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32, + 1 << CGP_VERSION_MAJOR_SHIFT); + + xe_map_wr(xe, &group->cgp_bo->vmap, + (32 + q->multi_queue.pos * 2) * sizeof(u32), + u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0]))); + + xe_map_wr(xe, &group->cgp_bo->vmap, + (33 + q->multi_queue.pos * 2) * sizeof(u32), + u32, guc_id); + + if (q->multi_queue.pos / 32) { + xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), + u32, BIT(q->multi_queue.pos % 32)); + xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0); + } else { + xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), + u32, BIT(q->multi_queue.pos)); + xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0); + } +} + +static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, + struct xe_exec_queue *q, + const u32 *action, u32 len) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_device *xe = guc_to_xe(guc); + enum xe_multi_queue_priority priority; + long ret; + + /* + * As all queues of a multi queue group use single drm scheduler + * submit workqueue, CGP synchronization with GuC are serialized. + * Hence, no locking is required here. + * Wait for any pending CGP_SYNC_DONE response before updating the + * CGP page and sending CGP_SYNC message. + * + * FIXME: Support VF migration + */ + ret = wait_event_timeout(guc->ct.wq, + !READ_ONCE(group->sync_pending) || + xe_guc_read_stopped(guc), HZ); + if (!ret || xe_guc_read_stopped(guc)) { + /* CGP_SYNC failed. Reset gt, cleanup the group */ + xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n"); + set_exec_queue_group_banned(q); + xe_gt_reset_async(q->gt); + xe_guc_exec_queue_group_trigger_cleanup(q); + return; + } + + scoped_guard(spinlock, &q->multi_queue.lock) + priority = q->multi_queue.priority; + + xe_lrc_set_multi_queue_priority(q->lrc[0], priority); + xe_guc_exec_queue_group_cgp_update(xe, q); + + WRITE_ONCE(group->sync_pending, true); + xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); +} + +static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q) +{ +#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; + action[len++] = group->primary->guc->id; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); +#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE + + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + +static void __register_exec_queue_group(struct xe_exec_queue *q, + struct guc_ctxt_registration_info *info) +{ + struct xe_guc *guc = exec_queue_to_guc(q); +#define MAX_MULTI_QUEUE_REG_SIZE (8) + u32 action[MAX_MULTI_QUEUE_REG_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE; + action[len++] = info->flags; + action[len++] = info->context_idx; + action[len++] = info->engine_class; + action[len++] = info->engine_submit_mask; + action[len++] = 0; /* Reserved */ + action[len++] = info->cgp_lo; + action[len++] = info->cgp_hi; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE); +#undef MAX_MULTI_QUEUE_REG_SIZE + + /* + * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a + * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response + * from guc. + */ + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + +static void __register_mlrc_exec_queue(struct xe_guc *guc, + struct xe_exec_queue *q, + struct guc_ctxt_registration_info *info) +{ +#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) + u32 action[MAX_MLRC_REG_SIZE]; + int len = 0; + int i; + + xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q)); + + action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; + action[len++] = info->flags; + action[len++] = info->context_idx; + action[len++] = info->engine_class; + action[len++] = info->engine_submit_mask; + action[len++] = info->wq_desc_lo; + action[len++] = info->wq_desc_hi; + action[len++] = info->wq_base_lo; + action[len++] = info->wq_base_hi; + action[len++] = info->wq_size; + action[len++] = q->width; + action[len++] = info->hwlrca_lo; + action[len++] = info->hwlrca_hi; + + for (i = 1; i < q->width; ++i) { + struct xe_lrc *lrc = q->lrc[i]; + + action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); + action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); + } + + /* explicitly checks some fields that we might fixup later */ + xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == + action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER]); + xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == + action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER]); + xe_gt_assert(guc_to_gt(guc), q->width == + action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS]); + xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == + action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR]); + xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE); +#undef MAX_MLRC_REG_SIZE + + xe_guc_ct_send(&guc->ct, action, len, 0, 0); +} + +static void __register_exec_queue(struct xe_guc *guc, + struct guc_ctxt_registration_info *info) +{ + u32 action[] = { + XE_GUC_ACTION_REGISTER_CONTEXT, + info->flags, + info->context_idx, + info->engine_class, + info->engine_submit_mask, + info->wq_desc_lo, + info->wq_desc_hi, + info->wq_base_lo, + info->wq_base_hi, + info->wq_size, + info->hwlrca_lo, + info->hwlrca_hi, + }; + + /* explicitly checks some fields that we might fixup later */ + xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == + action[XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER]); + xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == + action[XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER]); + xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == + action[XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR]); + + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); +} + +static u32 xe_hwe_guc_logical_to_submit_mask(struct xe_hw_engine *hwe, u32 logical_mask) +{ + struct xe_gt *gt = hwe->gt; + + if (xe_gt_is_usm_hwe(gt, hwe)) { + int shift = gt->usm.paging_hwe0->logical_instance; + u32 paging_logical_mask = gt->usm.paging_logical_mask; + + xe_gt_assert(gt, (logical_mask & paging_logical_mask) == logical_mask); + + /* + * Remap to GUC_PAGING_CLASS logical instance mask, if + * applicable. + */ + if (xe_guc_has_paging_engine(&hwe->gt->uc.guc)) + return logical_mask >> shift; + } + + return logical_mask; +} + +static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct xe_lrc *lrc = q->lrc[0]; + struct guc_ctxt_registration_info info; + + xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT); + + memset(&info, 0, sizeof(info)); + info.context_idx = q->guc->id; + info.engine_class = xe_hwe_to_guc_class(q->hwe); + info.engine_submit_mask = + xe_hwe_guc_logical_to_submit_mask(q->hwe, q->logical_mask); + info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); + info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); + info.flags = CONTEXT_REGISTRATION_FLAG_KMD | + FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type); + + if (xe_exec_queue_is_multi_queue(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo); + info.cgp_hi = 0; + } + + if (xe_exec_queue_is_parallel(q)) { + u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); + struct iosys_map map = xe_lrc_parallel_map(lrc); + + info.wq_desc_lo = lower_32_bits(ggtt_addr + + offsetof(struct guc_submit_parallel_scratch, wq_desc)); + info.wq_desc_hi = upper_32_bits(ggtt_addr + + offsetof(struct guc_submit_parallel_scratch, wq_desc)); + info.wq_base_lo = lower_32_bits(ggtt_addr + + offsetof(struct guc_submit_parallel_scratch, wq[0])); + info.wq_base_hi = upper_32_bits(ggtt_addr + + offsetof(struct guc_submit_parallel_scratch, wq[0])); + info.wq_size = WQ_SIZE; + + q->guc->wqi_head = 0; + q->guc->wqi_tail = 0; + xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); + parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); + } + + set_exec_queue_registered(q); + trace_xe_exec_queue_register(q); + if (xe_exec_queue_is_multi_queue_primary(q)) + __register_exec_queue_group(q, &info); + else if (xe_exec_queue_is_parallel(q)) + __register_mlrc_exec_queue(guc, q, &info); + else if (!xe_exec_queue_is_multi_queue_secondary(q)) + __register_exec_queue(guc, &info); + + if (!xe_exec_queue_is_multi_queue_secondary(q)) + init_policies(guc, q); + + if (xe_exec_queue_is_multi_queue_secondary(q)) + guc_exec_queue_send_cgp_sync(q); +} + +static u32 wq_space_until_wrap(struct xe_exec_queue *q) +{ + return (WQ_SIZE - q->guc->wqi_tail); +} + +static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); + unsigned int sleep_period_ms = 1, sleep_total_ms = 0; + +#define AVAILABLE_SPACE \ + CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) + if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { +try_again: + q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); + if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { + if (sleep_total_ms > 2000) { + xe_gt_reset_async(q->gt); + return -ENODEV; + } + + sleep_total_ms += xe_sleep_exponential_ms(&sleep_period_ms, 64); + goto try_again; + } + } +#undef AVAILABLE_SPACE + + return 0; +} + +static int wq_noop_append(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); + u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; + + if (wq_wait_for_space(q, wq_space_until_wrap(q))) + return -ENODEV; + + xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw)); + + parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], + FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | + FIELD_PREP(WQ_LEN_MASK, len_dw)); + q->guc->wqi_tail = 0; + + return 0; +} + +static void wq_item_append(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); +#define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ + u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; + u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); + u32 len_dw = (wqi_size / sizeof(u32)) - 1; + int i = 0, j; + + if (wqi_size > wq_space_until_wrap(q)) { + if (wq_noop_append(q)) + return; + } + if (wq_wait_for_space(q, wqi_size)) + return; + + wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | + FIELD_PREP(WQ_LEN_MASK, len_dw); + wqi[i++] = xe_lrc_descriptor(q->lrc[0]); + wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | + FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); + wqi[i++] = 0; + for (j = 1; j < q->width; ++j) { + struct xe_lrc *lrc = q->lrc[j]; + + wqi[i++] = lrc->ring.tail / sizeof(u64); + } + + xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32)); + + iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, + wq[q->guc->wqi_tail / sizeof(u32)])); + xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); + q->guc->wqi_tail += wqi_size; + xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE); + + xe_device_wmb(xe); + + map = xe_lrc_parallel_map(q->lrc[0]); + parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); +} + +#define RESUME_PENDING ~0x0ull +static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_lrc *lrc = q->lrc[0]; + u32 action[3]; + u32 g2h_len = 0; + u32 num_g2h = 0; + int len = 0; + bool extra_submit = false; + + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + + if (!job->restore_replay || job->last_replay) { + if (xe_exec_queue_is_parallel(q)) + wq_item_append(q); + else + xe_lrc_set_ring_tail(lrc, lrc->ring.tail); + job->last_replay = false; + } + + if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) + return; + + /* + * All queues in a multi-queue group will use the primary queue + * of the group to interface with GuC. If primay is suspended, + * just return. Jobs will get scheduled once primary is resumed. + */ + q = xe_exec_queue_multi_queue_primary(q); + if (exec_queue_suspended(q)) + return; + + if (!exec_queue_enabled(q)) { + action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; + action[len++] = q->guc->id; + action[len++] = GUC_CONTEXT_ENABLE; + g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; + num_g2h = 1; + if (xe_exec_queue_is_parallel(q)) + extra_submit = true; + + q->guc->resume_time = RESUME_PENDING; + set_exec_queue_pending_enable(q); + set_exec_queue_enabled(q); + trace_xe_exec_queue_scheduling_enable(q); + } else { + action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; + action[len++] = q->guc->id; + trace_xe_exec_queue_submit(q); + } + + xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); + + if (extra_submit) { + len = 0; + action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; + action[len++] = q->guc->id; + trace_xe_exec_queue_submit(q); + + xe_guc_ct_send(&guc->ct, action, len, 0, 0); + } +} + +static struct dma_fence * +guc_exec_queue_run_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + struct xe_exec_queue *q = job->q; + struct xe_guc *guc = exec_queue_to_guc(q); + bool killed_or_banned_or_wedged = + exec_queue_killed_or_banned_or_wedged(q); + + xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || + exec_queue_banned(q) || exec_queue_suspended(q)); + + trace_xe_sched_job_run(job); + + if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + + if (exec_queue_killed_or_banned_or_wedged(primary)) + goto run_job_out; + + if (!exec_queue_registered(primary)) + register_exec_queue(primary, GUC_CONTEXT_NORMAL); + } + + if (!exec_queue_registered(q)) + register_exec_queue(q, GUC_CONTEXT_NORMAL); + if (!job->restore_replay) + q->ring_ops->emit_job(job); + submit_exec_queue(q, job); + job->restore_replay = false; + } + +run_job_out: + + return job->fence; +} + +static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + + trace_xe_sched_job_free(job); + xe_sched_job_put(job); +} + +int xe_guc_read_stopped(struct xe_guc *guc) +{ + return atomic_read(&guc->submission_state.stopped); +} + +static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, + struct xe_exec_queue *q, + u32 runnable_state); +static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q); + +#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ + u32 action[] = { \ + XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ + q->guc->id, \ + GUC_CONTEXT_##enable_disable, \ + } + +static void disable_scheduling_deregister(struct xe_guc *guc, + struct xe_exec_queue *q) +{ + MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); + int ret; + + if (!xe_exec_queue_is_multi_queue_secondary(q)) + set_min_preemption_timeout(guc, q); + + smp_rmb(); + ret = wait_event_timeout(guc->ct.wq, + (!exec_queue_pending_enable(q) && + !exec_queue_pending_disable(q)) || + xe_guc_read_stopped(guc) || + vf_recovery(guc), + HZ * 5); + if (!ret && !vf_recovery(guc)) { + struct xe_gpu_scheduler *sched = &q->guc->sched; + + xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n"); + xe_sched_submission_start(sched); + xe_gt_reset_async(q->gt); + xe_sched_tdr_queue_imm(sched); + return; + } + + clear_exec_queue_enabled(q); + set_exec_queue_pending_disable(q); + set_exec_queue_destroyed(q); + trace_xe_exec_queue_scheduling_disable(q); + + /* + * Reserve space for both G2H here as the 2nd G2H is sent from a G2H + * handler and we are not allowed to reserved G2H space in handlers. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_multi_queue_secondary_sched_done(guc, q, 0); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + + G2H_LEN_DW_DEREGISTER_CONTEXT, 2); +} + +/** + * xe_guc_submit_wedge() - Wedge GuC submission + * @guc: the GuC object + * + * Save exec queue's registered with GuC state by taking a ref to each queue. + * Register a DRMM handler to drop refs upon driver unload. + */ +void xe_guc_submit_wedge(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_exec_queue *q; + unsigned long index; + + xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); + + /* + * If device is being wedged even before submission_state is + * initialized, there's nothing to do here. + */ + if (!guc->submission_state.initialized) + return; + + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + if (xe_exec_queue_get_unless_zero(q)) + set_exec_queue_wedged(q); + mutex_unlock(&guc->submission_state.lock); + } else { + /* Forcefully kill any remaining exec queues, signal fences */ + guc_submit_reset_prepare(guc); + xe_guc_submit_stop(guc); + xe_guc_softreset(guc); + xe_uc_fw_sanitize(&guc->fw); + xe_guc_submit_pause_abort(guc); + } +} + +static bool guc_submit_hint_wedged(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + + if (xe->wedged.mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) + return false; + + if (xe_device_wedged(xe)) + return true; + + xe_device_declare_wedged(xe); + + return true; +} + +#define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) + +static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) +{ + struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); + u32 ctx_timestamp, ctx_job_timestamp; + u32 timeout_ms = q->sched_props.job_timeout_ms; + u32 diff; + u64 running_time_ms; + + if (!xe_sched_job_started(job)) { + xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started", + xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), + q->guc->id); + + /* GuC never scheduled this job - let the caller trigger a GT reset. */ + return true; + } + + ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0])); + if (ctx_timestamp == job->sample_timestamp) { + if (IS_SRIOV_VF(gt_to_xe(gt))) + xe_gt_notice(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck", + xe_sched_job_seqno(job), + xe_sched_job_lrc_seqno(job), q->guc->id); + else + xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck", + xe_sched_job_seqno(job), + xe_sched_job_lrc_seqno(job), q->guc->id); + + return xe_sched_invalidate_job(job, 0); + } + + job->sample_timestamp = ctx_timestamp; + ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); + + /* + * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch + * possible overflows with a high timeout. + */ + xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); + + diff = ctx_timestamp - ctx_job_timestamp; + + /* + * Ensure timeout is within 5% to account for an GuC scheduling latency + */ + running_time_ms = + ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); + + xe_gt_dbg(gt, + "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", + xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), + q->guc->id, running_time_ms, timeout_ms, diff); + + return running_time_ms >= timeout_ms; +} + +static void enable_scheduling(struct xe_exec_queue *q) +{ + MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); + struct xe_guc *guc = exec_queue_to_guc(q); + int ret; + + xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); + + set_exec_queue_pending_enable(q); + set_exec_queue_enabled(q); + trace_xe_exec_queue_scheduling_enable(q); + + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_multi_queue_secondary_sched_done(guc, q, 1); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); + + ret = wait_event_timeout(guc->ct.wq, + !exec_queue_pending_enable(q) || + xe_guc_read_stopped(guc) || + vf_recovery(guc), HZ * 5); + if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) { + xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); + set_exec_queue_banned(q); + xe_gt_reset_async(q->gt); + xe_sched_tdr_queue_imm(&q->guc->sched); + } +} + +static void disable_scheduling(struct xe_exec_queue *q, bool immediate) +{ + MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); + struct xe_guc *guc = exec_queue_to_guc(q); + + xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); + + if (immediate && !xe_exec_queue_is_multi_queue_secondary(q)) + set_min_preemption_timeout(guc, q); + clear_exec_queue_enabled(q); + set_exec_queue_pending_disable(q); + trace_xe_exec_queue_scheduling_disable(q); + + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_multi_queue_secondary_sched_done(guc, q, 0); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); +} + +/* + * Recover via GT reset for a kernel queue, or for a GuC scheduling failure (job + * never started) on a queue that was not already killed or banned. An already + * banned queue must stay banned, so its unstarted jobs do not clear the ban or + * trigger a reset. + */ +static bool timeout_needs_gt_reset(struct xe_exec_queue *q, struct xe_sched_job *job, + bool skip_timeout_check) +{ + if (q->flags & EXEC_QUEUE_FLAG_KERNEL) + return true; + + return !skip_timeout_check && !xe_sched_job_started(job); +} + +static enum drm_gpu_sched_stat +guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + struct drm_sched_job *tmp_job; + struct xe_exec_queue *q = job->q, *primary; + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_guc *guc = exec_queue_to_guc(q); + const char *process_name = "no process"; + struct xe_device *xe = guc_to_xe(guc); + int err = -ETIME; + pid_t pid = -1; + bool wedged = false, wedge_device = false, skip_timeout_check; + + xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); + + primary = xe_exec_queue_multi_queue_primary(q); + + /* + * TDR has fired before free job worker. Common if exec queue + * immediately closed after last fence signaled. Add back to pending + * list so job can be freed and kick scheduler ensuring free job is not + * lost. + */ + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags) || + vf_recovery(guc)) + return DRM_GPU_SCHED_STAT_NO_HANG; + + /* Kill the run_job entry point */ + if (xe_exec_queue_is_multi_queue(q)) + xe_guc_exec_queue_group_stop(q); + else + xe_sched_submission_stop(sched); + + /* Must check all state after stopping scheduler */ + skip_timeout_check = exec_queue_reset(q) || + exec_queue_killed_or_banned_or_wedged(q); + + /* Skip timeout check if multi-queue group is banned */ + if (xe_exec_queue_is_multi_queue(q) && + READ_ONCE(q->multi_queue.group->banned)) + skip_timeout_check = true; + + /* LR jobs can only get here if queue has been killed or hit an error */ + if (xe_exec_queue_is_lr(q)) + xe_gt_assert(guc_to_gt(guc), skip_timeout_check); + + /* + * If devcoredump not captured and GuC capture for the job is not ready + * do manual capture first and decide later if we need to use it + */ + if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q) && !xe->devcoredump.captured && + !xe_guc_capture_get_matching_and_lock(q)) { + /* take force wake before engine register manual capture */ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n"); + + xe_engine_snapshot_capture_for_queue(q); + } + + /* + * Check if job is actually timed out, if so restart job execution and TDR + */ + if (!skip_timeout_check && !check_timeout(q, job)) + goto rearm; + + /* + * Killed queues must not newly wedge the device, but preserve an + * already-wedged state to avoid warning on teardown timeouts. + */ + if (!exec_queue_killed(q)) + wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); + else + wedged = xe_device_wedged(xe); + + set_exec_queue_banned(q); + + /* Kick job / queue off hardware */ + if (!xe_device_is_in_reset(xe) && !wedged && + (exec_queue_enabled(primary) || exec_queue_pending_disable(primary))) { + int ret; + + if (exec_queue_reset(primary)) + err = -EIO; + + if (xe_uc_fw_is_running(&guc->fw)) { + /* + * Wait for any pending G2H to flush out before + * modifying state + */ + ret = wait_event_timeout(guc->ct.wq, + (!exec_queue_pending_enable(primary) && + !exec_queue_pending_disable(primary)) || + xe_guc_read_stopped(guc) || + vf_recovery(guc), HZ * 5); + if (vf_recovery(guc)) + goto handle_vf_resume; + if (!ret || xe_guc_read_stopped(guc)) + goto trigger_reset; + + disable_scheduling(primary, skip_timeout_check); + } + + /* + * Must wait for scheduling to be disabled before signalling + * any fences, if GT broken the GT reset code should signal us. + * + * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault + * error) messages which can cause the schedule disable to get + * lost. If this occurs, trigger a GT reset to recover. + */ + smp_rmb(); + ret = wait_event_timeout(guc->ct.wq, + !xe_uc_fw_is_running(&guc->fw) || + !exec_queue_pending_disable(primary) || + xe_guc_read_stopped(guc) || + vf_recovery(guc), HZ * 5); + if (vf_recovery(guc)) + goto handle_vf_resume; + if (!ret || xe_guc_read_stopped(guc)) { +trigger_reset: + if (!ret) + xe_gt_warn(guc_to_gt(guc), + "Schedule disable failed to respond, guc_id=%d", + primary->guc->id); + xe_devcoredump(primary, job, + "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d", + primary->guc->id, ret, xe_guc_read_stopped(guc)); + xe_gt_reset_async(primary->gt); + xe_sched_tdr_queue_imm(sched); + goto rearm; + } + } + + if (q->vm && q->vm->xef) { + process_name = q->vm->xef->process_name; + pid = q->vm->xef->pid; + } + + if (!exec_queue_killed(q)) + xe_gt_notice(guc_to_gt(guc), + "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", + xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), + q->guc->id, q->flags, process_name, pid); + + trace_xe_sched_job_timedout(job); + + /* Do not access device if in reset */ + if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q)) + xe_devcoredump(q, job, + "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", + xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), + q->guc->id, q->flags); + + if (!wedged) { + if (timeout_needs_gt_reset(q, job, skip_timeout_check)) { + if (!xe_sched_invalidate_job(job, 2)) { + clear_exec_queue_banned(q); + xe_gt_reset_async(q->gt); + goto rearm; + } + if (q->flags & EXEC_QUEUE_FLAG_KERNEL) { + xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n"); + wedge_device = true; + } + } else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) { + xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n"); + } + } + + /* Mark all outstanding jobs as bad, thus completing them */ + xe_sched_job_set_error(job, err); + drm_sched_for_each_pending_job(tmp_job, &sched->base, NULL) + xe_sched_job_set_error(to_xe_sched_job(tmp_job), -ECANCELED); + + if (xe_exec_queue_is_multi_queue(q)) { + xe_guc_exec_queue_group_start(q); + xe_guc_exec_queue_group_trigger_cleanup(q); + } else { + xe_sched_submission_start(sched); + xe_guc_exec_queue_trigger_cleanup(q); + } + + if (wedge_device) + xe_device_declare_wedged(gt_to_xe(q->gt)); + + /* + * We want the job added back to the pending list so it gets freed; this + * is what DRM_GPU_SCHED_STAT_NO_HANG does. + */ + return DRM_GPU_SCHED_STAT_NO_HANG; + +rearm: + /* + * XXX: Ideally want to adjust timeout based on current execution time + * but there is not currently an easy way to do in DRM scheduler. With + * some thought, do this in a follow up. + */ + if (xe_exec_queue_is_multi_queue(q)) + xe_guc_exec_queue_group_start(q); + else + xe_sched_submission_start(sched); +handle_vf_resume: + return DRM_GPU_SCHED_STAT_NO_HANG; +} + +static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q); +static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q); + +static void guc_exec_queue_fini(struct xe_exec_queue *q) +{ + struct xe_guc_exec_queue *ge = q->guc; + struct xe_guc *guc = exec_queue_to_guc(q); + struct drm_device *drm = &guc_to_xe(guc)->drm; + + /* + * A secondary can leave the group while still preempt suspended (e.g. + * xe_vm_remove_compute_exec_queue() forces its preempt fence to signal, + * which suspends it). It holds one forwarded suspend reference on the + * primary, so drop it and resume the primary if it was the last member + * that had it suspended. Primaries forward to nobody, so they don't need + * this. + * + * First make sure the primary's forwarded suspend has completed. If the + * secondary was killed/reset before its preempt fence worker ran, that + * worker skips suspend_wait() (see preempt_fence_work_func()), leaving + * the primary's suspend possibly in flight. drop_suspend() runs under a + * spinlock and cannot wait, so drain it here with the uninterruptible + * blocking wait; otherwise resuming the primary in drop_suspend() could + * trip the !suspend_pending assert. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + if (READ_ONCE(q->guc->suspend_count)) + guc_exec_queue_suspend_wait_blocking(q); + guc_exec_queue_multi_queue_drop_suspend(q); + } + + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + mutex_lock(&group->list_lock); + list_del(&q->multi_queue.link); + mutex_unlock(&group->list_lock); + } + + release_guc_id(guc, q); + xe_sched_entity_fini(&ge->entity); + xe_sched_fini(&ge->sched); + + /* + * RCU free due sched being exported via DRM scheduler fences + * (timeline name). + */ + kfree_rcu(ge, rcu); + + drm_dev_put(drm); +} + +static void guc_exec_queue_do_destroy(struct xe_exec_queue *q) +{ + struct xe_guc_exec_queue *ge = q->guc; + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct drm_device *drm = &xe->drm; + + /* + * guc_exec_queue_fini() drops the queue's drm_device ref. + * Keep the device alive until the PM-runtime guard unwinds. + */ + drm_dev_get(drm); + + scoped_guard(xe_pm_runtime, xe) { + trace_xe_exec_queue_destroy(q); + + /* Confirm no work left behind accessing device structures */ + cancel_delayed_work_sync(&ge->sched.base.work_tdr); + + xe_exec_queue_fini(q); + } + + drm_dev_put(drm); +} + +static void __guc_exec_queue_destroy_async(struct work_struct *w) +{ + struct xe_guc_exec_queue *ge = + container_of(w, struct xe_guc_exec_queue, destroy_async); + + guc_exec_queue_do_destroy(ge->q); +} + +static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) +{ + INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async); + + /* We must block on kernel engines so slabs are empty on driver unload */ + if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) + guc_exec_queue_do_destroy(q); + else + xe_destroy_wq_queue(&q->guc->destroy_async); +} + +static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q) +{ + /* + * Might be done from within the GPU scheduler, need to do async as we + * fini the scheduler when the engine is fini'd, the scheduler can't + * complete fini within itself (circular dependency). Async resolves + * this we and don't really care when everything is fini'd, just that it + * is. + */ + guc_exec_queue_destroy_async(q); +} + +static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) +{ + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); + + xe_gt_assert(guc_to_gt(guc), !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); + trace_xe_exec_queue_cleanup_entity(q); + + /* + * Expected state transitions for cleanup: + * - If the exec queue is registered and GuC firmware is running, we must first + * disable scheduling and deregister the queue to ensure proper teardown and + * resource release in the GuC, then destroy the exec queue on driver side. + * - If the GuC is already stopped (e.g., during driver unload or GPU reset), + * we cannot expect a response for the deregister request. In this case, + * it is safe to directly destroy the exec queue on driver side, as the GuC + * will not process further requests and all resources must be cleaned up locally. + */ + if (exec_queue_registered(q) && xe_uc_fw_is_running(&guc->fw)) + disable_scheduling_deregister(guc, q); + else + __guc_exec_queue_destroy(guc, q); +} + +static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) +{ + return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); +} + +static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) +{ + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); + + if (guc_exec_queue_allowed_to_change_state(q)) + init_policies(guc, q); + kfree(msg); +} + +static void __suspend_fence_signal(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + + if (!q->guc->suspend_pending) + return; + + WRITE_ONCE(q->guc->suspend_pending, false); + + /* + * We use a GuC shared wait queue for VFs because the VF resfix start + * interrupt must be able to wake all instances of suspend_wait. This + * prevents the VF migration worker from being starved during + * scheduling. + */ + if (IS_SRIOV_VF(xe)) + wake_up_all(&guc->ct.wq); + else + wake_up(&q->guc->suspend_wait); +} + +static void suspend_fence_signal(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + + xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) || + xe_guc_read_stopped(guc)); + xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending); + + __suspend_fence_signal(q); +} + +static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) +{ + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); + + if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && + exec_queue_enabled(q)) { + wait_event(guc->ct.wq, vf_recovery(guc) || + ((q->guc->resume_time != RESUME_PENDING || + xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q))); + + if (!xe_guc_read_stopped(guc)) { + s64 since_resume_ms = + ktime_ms_delta(ktime_get(), + q->guc->resume_time); + s64 wait_ms = q->vm->preempt.min_run_period_ms - + since_resume_ms; + + if (wait_ms > 0 && q->guc->resume_time) + xe_sleep_relaxed_ms(wait_ms); + + set_exec_queue_suspended(q); + disable_scheduling(q, false); + } + } else if (q->guc->suspend_pending) { + set_exec_queue_suspended(q); + suspend_fence_signal(q); + } +} + +static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) +{ + struct xe_exec_queue *q = msg->private_data; + + if (guc_exec_queue_allowed_to_change_state(q)) { + clear_exec_queue_suspended(q); + if (!exec_queue_enabled(q)) { + q->guc->resume_time = RESUME_PENDING; + set_exec_queue_pending_resume(q); + enable_scheduling(q); + } + } else { + clear_exec_queue_suspended(q); + } +} + +static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg) +{ + struct xe_exec_queue *q = msg->private_data; + + if (guc_exec_queue_allowed_to_change_state(q)) + guc_exec_queue_send_cgp_sync(q); + + kfree(msg); +} + +#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ +#define SET_SCHED_PROPS 2 +#define SUSPEND 3 +#define RESUME 4 +#define SET_MULTI_QUEUE_PRIORITY 5 +#define OPCODE_MASK 0xf +#define MSG_LOCKED BIT(8) +#define MSG_HEAD BIT(9) + +static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) +{ + struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data)); + + trace_xe_sched_msg_recv(msg); + + switch (msg->opcode) { + case CLEANUP: + __guc_exec_queue_process_msg_cleanup(msg); + break; + case SET_SCHED_PROPS: + __guc_exec_queue_process_msg_set_sched_props(msg); + break; + case SUSPEND: + __guc_exec_queue_process_msg_suspend(msg); + break; + case RESUME: + __guc_exec_queue_process_msg_resume(msg); + break; + case SET_MULTI_QUEUE_PRIORITY: + __guc_exec_queue_process_msg_set_multi_queue_priority(msg); + break; + default: + XE_WARN_ON("Unknown message type"); + } + + xe_pm_runtime_put(xe); +} + +static const struct drm_sched_backend_ops drm_sched_ops = { + .run_job = guc_exec_queue_run_job, + .free_job = guc_exec_queue_free_job, + .timedout_job = guc_exec_queue_timedout_job, +}; + +static const struct xe_sched_backend_ops xe_sched_ops = { + .process_msg = guc_exec_queue_process_msg, +}; + +static int guc_exec_queue_init(struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched; + struct xe_guc *guc = exec_queue_to_guc(q); + struct drm_device *drm = &guc_to_xe(guc)->drm; + struct workqueue_struct *submit_wq = NULL; + struct xe_guc_exec_queue *ge; + long timeout; + int err, i; + + xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc))); + + ge = kzalloc_obj(*ge); + if (!ge) + return -ENOMEM; + + drm_dev_get(drm); + + q->guc = ge; + ge->q = q; + init_rcu_head(&ge->rcu); + init_waitqueue_head(&ge->suspend_wait); + + for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i) + INIT_LIST_HEAD(&ge->static_msgs[i].link); + + timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : + msecs_to_jiffies(q->sched_props.job_timeout_ms); + + err = alloc_guc_id(guc, q); + if (err) + goto err_free; + + xe_exec_queue_assign_name(q, q->guc->id); + + strscpy(ge->name, q->name, sizeof(ge->name)); + + /* + * Use primary queue's submit_wq for all secondary queues of a + * multi queue group. This serialization avoids any locking around + * CGP synchronization with GuC. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + + submit_wq = primary->guc->sched.base.submit_wq; + } + + err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, + submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, + timeout, guc_to_gt(guc)->ordered_wq, NULL, + ge->name, gt_to_xe(q->gt)->drm.dev); + if (err) + goto err_release_id; + + sched = &ge->sched; + err = xe_sched_entity_init(&ge->entity, sched); + if (err) + goto err_sched; + + q->entity = &ge->entity; + + mutex_lock(&guc->submission_state.lock); + if (xe_guc_read_stopped(guc) || vf_recovery(guc)) + xe_sched_stop(sched); + publish_guc_id(guc, q); + mutex_unlock(&guc->submission_state.lock); + + /* + * Maintain secondary queues of the multi queue group in a list + * for handling dependencies across the queues in the group. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + INIT_LIST_HEAD(&q->multi_queue.link); + mutex_lock(&group->list_lock); + if (group->stopped) + WRITE_ONCE(q->guc->sched.base.pause_submit, true); + list_add_tail(&q->multi_queue.link, &group->list); + mutex_unlock(&group->list_lock); + } + + if (xe_exec_queue_is_multi_queue(q)) + trace_xe_exec_queue_create_multi_queue(q); + else + trace_xe_exec_queue_create(q); + + return 0; + +err_sched: + xe_sched_fini(&ge->sched); +err_release_id: + release_guc_id(guc, q); +err_free: + kfree(ge); + drm_dev_put(drm); + + return err; +} + +static void guc_exec_queue_kill(struct xe_exec_queue *q) +{ + trace_xe_exec_queue_kill(q); + set_exec_queue_killed(q); + __suspend_fence_signal(q); + xe_guc_exec_queue_trigger_cleanup(q); +} + +static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, + u32 opcode) +{ + xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); + + INIT_LIST_HEAD(&msg->link); + msg->opcode = opcode & OPCODE_MASK; + msg->private_data = q; + + trace_xe_sched_msg_add(msg); + if (opcode & MSG_HEAD) + xe_sched_add_msg_head(&q->guc->sched, msg); + else if (opcode & MSG_LOCKED) + xe_sched_add_msg_locked(&q->guc->sched, msg); + else + xe_sched_add_msg(&q->guc->sched, msg); +} + +static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q, + struct xe_sched_msg *msg, + u32 opcode) +{ + if (!list_empty(&msg->link)) + return; + + guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED | MSG_HEAD); +} + +static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q, + struct xe_sched_msg *msg, + u32 opcode) +{ + if (!list_empty(&msg->link)) + return false; + + guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED); + + return true; +} + +#define STATIC_MSG_CLEANUP 0 +#define STATIC_MSG_SUSPEND 1 +#define STATIC_MSG_RESUME 2 +static void guc_exec_queue_destroy(struct xe_exec_queue *q) +{ + struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; + + if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) + guc_exec_queue_add_msg(q, msg, CLEANUP); + else + __guc_exec_queue_destroy(exec_queue_to_guc(q), q); +} + +static int guc_exec_queue_set_priority(struct xe_exec_queue *q, + enum xe_exec_queue_priority priority) +{ + struct xe_sched_msg *msg; + + if (q->sched_props.priority == priority || + exec_queue_killed_or_banned_or_wedged(q)) + return 0; + + msg = kmalloc_obj(*msg); + if (!msg) + return -ENOMEM; + + q->sched_props.priority = priority; + guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); + + return 0; +} + +static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) +{ + struct xe_sched_msg *msg; + + if (q->sched_props.timeslice_us == timeslice_us || + exec_queue_killed_or_banned_or_wedged(q)) + return 0; + + msg = kmalloc_obj(*msg); + if (!msg) + return -ENOMEM; + + q->sched_props.timeslice_us = timeslice_us; + guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); + + return 0; +} + +static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, + u32 preempt_timeout_us) +{ + struct xe_sched_msg *msg; + + if (q->sched_props.preempt_timeout_us == preempt_timeout_us || + exec_queue_killed_or_banned_or_wedged(q)) + return 0; + + msg = kmalloc_obj(*msg); + if (!msg) + return -ENOMEM; + + q->sched_props.preempt_timeout_us = preempt_timeout_us; + guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); + + return 0; +} + +static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q, + enum xe_multi_queue_priority priority) +{ + struct xe_sched_msg *msg; + + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); + + if (exec_queue_killed_or_banned_or_wedged(q)) + return 0; + + msg = kmalloc_obj(*msg); + if (!msg) + return -ENOMEM; + + scoped_guard(spinlock, &q->multi_queue.lock) { + if (q->multi_queue.priority == priority) { + kfree(msg); + return 0; + } + + q->multi_queue.priority = priority; + } + + guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY); + + return 0; +} + +/* + * Core suspend: take a suspend reference on @q and, on the first reference, + * disable its GuC context so the GPU is actually preempted. Caller must have + * ensured @q is not killed/banned/wedged. Returns true if this was the first + * suspend reference (the 0->1 transition). + */ +static bool __guc_exec_queue_suspend(struct xe_exec_queue *q) +{ + struct xe_guc_exec_queue *ge = q->guc; + struct xe_gpu_scheduler *sched = &ge->sched; + struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_SUSPEND; + bool first; + + xe_sched_msg_lock(sched); + first = (++ge->suspend_count == 1); + if (first) { + bool added = guc_exec_queue_try_add_msg(q, msg, SUSPEND); + + /* slot must be free at 0->1 */ + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), added); + ge->suspend_pending = true; + } + xe_sched_msg_unlock(sched); + + return first; +} + +/* + * Core resume: drop a suspend reference on @q and, on the last reference, + * re-enable its GuC context. Returns true if this dropped the last suspend + * reference (the 1->0 transition). + */ +static bool __guc_exec_queue_resume(struct xe_exec_queue *q) +{ + struct xe_guc_exec_queue *ge = q->guc; + struct xe_gpu_scheduler *sched = &ge->sched; + struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_RESUME; + struct xe_guc *guc = exec_queue_to_guc(q); + bool last; + + xe_sched_msg_lock(sched); + xe_gt_assert(guc_to_gt(guc), !ge->suspend_pending); + xe_gt_assert(guc_to_gt(guc), ge->suspend_count > 0); + last = (--ge->suspend_count == 0); + if (last) { + bool added = guc_exec_queue_try_add_msg(q, msg, RESUME); + + /* slot must be free at 1->0 */ + xe_gt_assert(guc_to_gt(guc), added); + } + xe_sched_msg_unlock(sched); + + return last; +} + +static int guc_exec_queue_suspend(struct xe_exec_queue *q) +{ + if (exec_queue_killed_or_banned_or_wedged(q)) + return -EINVAL; + + /* + * Non-multi-queue queues and multi-queue primaries suspend themselves + * directly: their own msg_lock makes the suspend_count 0->1 transition + * and the suspend_pending update atomic, so no group level serialization + * is needed. + */ + if (!xe_exec_queue_is_multi_queue_secondary(q)) { + __guc_exec_queue_suspend(q); + return 0; + } + + /* + * A secondary's suspend is meaningless once the primary - which owns the + * group's GuC context - is gone, so fail it too. This keeps the + * secondary's effective state consistent with guc_exec_queue_reset_status(), + * which already reports the primary's killed/banned/wedged state for + * secondaries. A primary killed *after* this check is still handled at + * message-processing time, where the SUSPEND is a no-op for a killed + * context; this only covers an already-dead primary. + */ + if (exec_queue_killed_or_banned_or_wedged(xe_exec_queue_multi_queue_primary(q))) + return -EINVAL; + + /* + * A secondary doesn't interface with GuC: suspend it like any other + * queue (its own suspend_count drives its internally handled scheduler + * state) and, only on its own 0->1 transition, forward the suspend to the + * primary so the GPU is actually preempted. Hold @suspend_lock so that + * observing the secondary's transition and forwarding it to the primary + * happen atomically; this keeps the primary's refcount paired with member + * transitions even if the same secondary is suspended and resumed + * concurrently across rebind cycles. + */ + scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) { + if (__guc_exec_queue_suspend(q)) + __guc_exec_queue_suspend(xe_exec_queue_multi_queue_primary(q)); + } + + return 0; +} + +static void guc_exec_queue_suspend_timeout_ban(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + + xe_gt_warn(guc_to_gt(guc), + "Suspend fence, guc_id=%d, failed to respond, banning queue", + q->guc->id); + /* + * The GuC failed to respond to the suspend within the timeout. This is + * not recoverable for this context, so ban it and tear it down via + * cleanup rather than leave it suspended forever. __suspend_fence_signal + * clears suspend_pending and wakes any waiter. + * + * @q is the primary here; it owns the group's GuC context, so a failure + * to suspend it wedges the whole group. Ban and tear down the entire + * group in the multi-queue case. + */ + if (xe_exec_queue_is_multi_queue(q)) { + set_exec_queue_group_banned(q); + __suspend_fence_signal(q); + xe_guc_exec_queue_group_trigger_cleanup(q); + } else { + set_exec_queue_banned(q); + __suspend_fence_signal(q); + xe_guc_exec_queue_trigger_cleanup(q); + } +} + +/* + * Wait for @q's own suspend to complete: suspend_pending cleared, or the queue + * killed / GuC stopped. With @blocking, wait uninterruptibly and do not handle + * VF recovery (for callers that must complete on behalf of a possibly + * cross-process queue); otherwise wait interruptibly. + * + * Returns 0 on completion or -ETIME on timeout. Interruptible waits may also + * return -EAGAIN (VF recovery in progress, retry) or -ERESTARTSYS (aborted by a + * signal; suspend_pending may still be set, so callers must not resume() + * without re-confirming the suspend). + */ +static int guc_exec_queue_wait_suspend_done(struct xe_exec_queue *q, bool blocking) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + int ret; + + /* + * Likely don't need to check exec_queue_killed() as we clear + * suspend_pending upon kill but to be paranoid but races in which + * suspend_pending is set after kill also check kill here. + */ +#define WAIT_COND \ + (!READ_ONCE(q->guc->suspend_pending) || exec_queue_killed(q) || \ + xe_guc_read_stopped(guc)) + +retry: + if (blocking) { + if (IS_SRIOV_VF(xe)) + ret = wait_event_timeout(guc->ct.wq, WAIT_COND, HZ * 5); + else + ret = wait_event_timeout(q->guc->suspend_wait, WAIT_COND, + HZ * 5); + } else if (IS_SRIOV_VF(xe)) { + ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND || + vf_recovery(guc), HZ * 5); + } else { + ret = wait_event_interruptible_timeout(q->guc->suspend_wait, + WAIT_COND, HZ * 5); + } + + if (!blocking && vf_recovery(guc) && !xe_device_wedged(xe)) + return -EAGAIN; + + if (!ret) + return -ETIME; + else if (!blocking && IS_SRIOV_VF(xe) && !WAIT_COND) + /* Corner case on RESFIX DONE where vf_recovery() changes */ + goto retry; + +#undef WAIT_COND + + return ret < 0 ? ret : 0; +} + +static int guc_exec_queue_suspend_wait_common(struct xe_exec_queue *q, bool blocking) +{ + int ret; + + /* + * A secondary's suspend rides the sched-message worker (short-circuited, + * no GuC round-trip) and so is not synchronous with + * guc_exec_queue_suspend(): its own suspend_pending may still be set + * here. Waiting on the primary alone is not sufficient - if the primary + * was already suspended, the forward is a refcount-only transition that + * queues no new primary SUSPEND and leaves the primary's suspend_pending + * clear, so the primary wait would return immediately while the + * secondary's suspend is still in flight, and a later resume() would trip + * the secondary's !suspend_pending assert. So first wait for the + * secondary's own suspend to complete, then wait on the primary. + * + * A timeout on either bans the queue (being multi-queue, that tears down + * the whole group). A secondary suspend has no real GuC round-trip, so + * its timeout is a software scheduler stall rather than a GuC fault, but + * banning is still the safe recovery: otherwise the queue is left with + * suspend_pending set and a subsequent resume() trips the !suspend_pending + * assert. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + ret = guc_exec_queue_wait_suspend_done(q, blocking); + if (ret == -ETIME) + guc_exec_queue_suspend_timeout_ban(q); + if (ret) + return ret; + } + + q = xe_exec_queue_multi_queue_primary(q); + ret = guc_exec_queue_wait_suspend_done(q, blocking); + if (ret == -ETIME) + guc_exec_queue_suspend_timeout_ban(q); + + return ret; +} + +static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q) +{ + return guc_exec_queue_suspend_wait_common(q, false); +} + +/* + * Uninterruptible variant of guc_exec_queue_suspend_wait() for callers that + * must complete the wait on behalf of a queue possibly owned by a different + * process (e.g. cleanup/undo paths). An interruptible wait could return + * -ERESTARTSYS if the calling task is signalled, leaving that queue suspended + * forever (cross-process DoS). VF recovery is deliberately not handled (no + * -EAGAIN) since a blocking caller cannot retry. + */ +static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q) +{ + return guc_exec_queue_suspend_wait_common(q, true); +} + +static void guc_exec_queue_resume(struct xe_exec_queue *q) +{ + /* + * Non-multi-queue queues and multi-queue primaries resume themselves + * directly; their own msg_lock is sufficient. + */ + if (!xe_exec_queue_is_multi_queue_secondary(q)) { + __guc_exec_queue_resume(q); + return; + } + + /* + * Mirror of guc_exec_queue_suspend(): resume the secondary like any + * other queue and, only on its own 1->0 transition, forward the resume + * to the primary so the primary's GuC context is re-enabled once the + * last member that suspended it resumes. @suspend_lock keeps the + * secondary transition and the primary forward atomic. + */ + scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) { + if (__guc_exec_queue_resume(q)) + __guc_exec_queue_resume(xe_exec_queue_multi_queue_primary(q)); + } +} + +/* + * Drop a leaving secondary's forwarded suspend reference on the primary and + * resume the primary if this was the last member that had it suspended. + * See guc_exec_queue_fini(). + */ +static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q) +{ + scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + + /* + * A suspended secondary holds exactly one suspend reference on the + * primary (forwarded on its 0->1 transition). If it leaves while + * still suspended, release that reference so the primary is not + * kept disabled forever. + */ + if (!READ_ONCE(q->guc->suspend_count)) + break; + + if (exec_queue_killed_or_banned_or_wedged(primary)) + break; + + /* + * No suspend_wait() here (and we can't - suspend_lock is a + * spinlock). guc_exec_queue_fini() has already drained the + * primary's forwarded suspend with the blocking wait, so its + * suspend has completed (suspend_pending cleared) by the time we + * resume it here. __guc_exec_queue_resume() asserts this. + */ + __guc_exec_queue_resume(primary); + } +} + +static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) +{ + if (xe_exec_queue_is_multi_queue_secondary(q) && + guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q))) + return true; + + return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); +} + +/* + * All of these functions are an abstraction layer which other parts of Xe can + * use to trap into the GuC backend. All of these functions, aside from init, + * really shouldn't do much other than trap into the DRM scheduler which + * synchronizes these operations. + */ +static const struct xe_exec_queue_ops guc_exec_queue_ops = { + .init = guc_exec_queue_init, + .kill = guc_exec_queue_kill, + .fini = guc_exec_queue_fini, + .destroy = guc_exec_queue_destroy, + .set_priority = guc_exec_queue_set_priority, + .set_timeslice = guc_exec_queue_set_timeslice, + .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, + .set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority, + .suspend = guc_exec_queue_suspend, + .suspend_wait = guc_exec_queue_suspend_wait, + .suspend_wait_blocking = guc_exec_queue_suspend_wait_blocking, + .resume = guc_exec_queue_resume, + .reset_status = guc_exec_queue_reset_status, +}; + +static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + bool do_destroy = false; + + /* Stop scheduling + flush any DRM scheduler operations */ + xe_sched_submission_stop(sched); + + /* Clean up lost G2H + reset engine state */ + if (exec_queue_registered(q)) { + if (exec_queue_destroyed(q)) + do_destroy = true; + } + if (q->guc->suspend_pending) { + set_exec_queue_suspended(q); + suspend_fence_signal(q); + } + atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | + EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | + EXEC_QUEUE_STATE_SUSPENDED, + &q->guc->state); + q->guc->resume_time = 0; + trace_xe_exec_queue_stop(q); + + /* + * Ban any engine (aside from kernel and engines used for VM ops) with a + * started but not complete job or if a job has gone through a GT reset + * more than twice. + */ + if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { + struct xe_sched_job *job = xe_sched_first_pending_job(sched); + bool ban = false; + + if (job) { + if ((xe_sched_job_started(job) && + !xe_sched_job_completed(job)) || + xe_sched_invalidate_job(job, 2)) { + trace_xe_sched_job_ban(job); + ban = true; + } + } + + if (ban) { + set_exec_queue_banned(q); + xe_guc_exec_queue_trigger_cleanup(q); + } + } + + if (do_destroy) + __guc_exec_queue_destroy(guc, q); +} + +static int guc_submit_reset_prepare(struct xe_guc *guc) +{ + int ret; + + /* + * Using an atomic here rather than submission_state.lock as this + * function can be called while holding the CT lock (engine reset + * failure). submission_state.lock needs the CT lock to resubmit jobs. + * Atomic is not ideal, but it works to prevent against concurrent reset + * and releasing any TDRs waiting on guc->submission_state.stopped. + */ + ret = atomic_fetch_or(1, &guc->submission_state.stopped); + smp_wmb(); + wake_up_all(&guc->ct.wq); + + return ret; +} + +int xe_guc_submit_reset_prepare(struct xe_guc *guc) +{ + if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc))) + return 0; + + if (!guc->submission_state.initialized) + return 0; + + return guc_submit_reset_prepare(guc); +} + +void xe_guc_submit_reset_wait(struct xe_guc *guc) +{ + wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || + !xe_guc_read_stopped(guc)); +} + +void xe_guc_submit_stop(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); + + mutex_lock(&guc->submission_state.lock); + + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + /* Prevent redundant attempts to stop parallel queues */ + if (q->guc->id != index) + continue; + + guc_exec_queue_stop(guc, q); + } + + mutex_unlock(&guc->submission_state.lock); + + /* + * No one can enter the backend at this point, aside from new engine + * creation which is protected by guc->submission_state.lock. + */ + +} + +static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc, + struct xe_exec_queue *q) +{ + bool pending_enable, pending_disable, pending_resume; + + pending_enable = exec_queue_pending_enable(q); + pending_resume = exec_queue_pending_resume(q); + + if (pending_enable && pending_resume) { + q->guc->needs_resume = true; + xe_gt_dbg(guc_to_gt(guc), "Replay RESUME - guc_id=%d", + q->guc->id); + } + + if (pending_enable && !pending_resume) { + clear_exec_queue_registered(q); + xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER - guc_id=%d", + q->guc->id); + } + + if (pending_enable) { + clear_exec_queue_enabled(q); + clear_exec_queue_pending_resume(q); + clear_exec_queue_pending_enable(q); + xe_gt_dbg(guc_to_gt(guc), "Replay ENABLE - guc_id=%d", + q->guc->id); + } + + if (exec_queue_destroyed(q) && exec_queue_registered(q)) { + clear_exec_queue_destroyed(q); + q->guc->needs_cleanup = true; + xe_gt_dbg(guc_to_gt(guc), "Replay CLEANUP - guc_id=%d", + q->guc->id); + } + + pending_disable = exec_queue_pending_disable(q); + + if (pending_disable && exec_queue_suspended(q)) { + clear_exec_queue_suspended(q); + q->guc->needs_suspend = true; + xe_gt_dbg(guc_to_gt(guc), "Replay SUSPEND - guc_id=%d", + q->guc->id); + } + + if (pending_disable) { + if (!pending_enable) + set_exec_queue_enabled(q); + clear_exec_queue_pending_disable(q); + xe_gt_dbg(guc_to_gt(guc), "Replay DISABLE - guc_id=%d", + q->guc->id); + } + + q->guc->resume_time = 0; +} + +static void lrc_parallel_clear(struct xe_lrc *lrc) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + struct iosys_map map = xe_lrc_parallel_map(lrc); + int i; + + for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) + parallel_write(xe, map, wq[i], + FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | + FIELD_PREP(WQ_LEN_MASK, 0)); +} + +/* + * This function is quite complex but only real way to ensure no state is lost + * during VF resume flows. The function scans the queue state, make adjustments + * as needed, and queues jobs / messages which replayed upon unpause. + */ +static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_sched_job *job; + int i; + + lockdep_assert_held(&guc->submission_state.lock); + + /* Stop scheduling + flush any DRM scheduler operations */ + xe_sched_submission_stop(sched); + cancel_delayed_work_sync(&sched->base.work_tdr); + + guc_exec_queue_revert_pending_state_change(guc, q); + + if (xe_exec_queue_is_parallel(q)) { + /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ + struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); + + /* + * NOP existing WQ commands that may contain stale GGTT + * addresses. These will be replayed upon unpause. The hardware + * seems to get confused if the WQ head/tail pointers are + * adjusted. + */ + if (lrc) + lrc_parallel_clear(lrc); + } + + job = xe_sched_first_pending_job(sched); + if (job) { + job->restore_replay = true; + + /* + * Adjust software tail so jobs submitted overwrite previous + * position in ring buffer with new GGTT addresses. + */ + for (i = 0; i < q->width; ++i) + q->lrc[i]->ring.tail = job->ptrs[i].head; + } +} + +/** + * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be disabled + */ +void xe_guc_submit_pause(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + xe_sched_submission_stop(&q->guc->sched); + mutex_unlock(&guc->submission_state.lock); +} + +/** + * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF. + * @guc: the &xe_guc struct instance whose scheduler is to be disabled + */ +void xe_guc_submit_pause_vf(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); + xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + /* Prevent redundant attempts to stop parallel queues */ + if (q->guc->id != index) + continue; + + guc_exec_queue_pause(guc, q); + } + mutex_unlock(&guc->submission_state.lock); +} + +static void guc_exec_queue_start(struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + + if (!exec_queue_killed_or_banned_or_wedged(q)) { + struct xe_sched_job *job = xe_sched_first_pending_job(sched); + int i; + + trace_xe_exec_queue_resubmit(q); + if (job) { + for (i = 0; i < q->width; ++i) { + /* + * The GuC context is unregistered at this point + * time, adjusting software ring tail ensures + * jobs are rewritten in original placement, + * adjusting LRC tail ensures the newly loaded + * GuC / contexts only view the LRC tail + * increasing as jobs are written out. + */ + q->lrc[i]->ring.tail = job->ptrs[i].head; + xe_lrc_set_ring_tail(q->lrc[i], + xe_lrc_ring_head(q->lrc[i])); + } + } + xe_sched_resubmit_jobs(sched); + } + + xe_sched_submission_start(sched); + xe_sched_submission_resume_tdr(sched); +} + +int xe_guc_submit_start(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); + + mutex_lock(&guc->submission_state.lock); + atomic_dec(&guc->submission_state.stopped); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + /* Prevent redundant attempts to start parallel queues */ + if (q->guc->id != index) + continue; + + guc_exec_queue_start(q); + } + mutex_unlock(&guc->submission_state.lock); + + wake_up_all(&guc->ct.wq); + + return 0; +} + +static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, + struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_sched_job *job = NULL; + struct drm_sched_job *s_job; + bool restore_replay = false; + + drm_sched_for_each_pending_job(s_job, &sched->base, NULL) { + job = to_xe_sched_job(s_job); + restore_replay |= job->restore_replay; + if (restore_replay) { + xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", + q->guc->id, xe_sched_job_seqno(job)); + + q->ring_ops->emit_job(job); + job->restore_replay = true; + } + } + + if (job) + job->last_replay = true; +} + +/** + * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF. + * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause + */ +void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); + xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + /* Prevent redundant attempts to stop parallel queues */ + if (q->guc->id != index) + continue; + + guc_exec_queue_unpause_prepare(guc, q); + } + mutex_unlock(&guc->submission_state.lock); +} + +static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_sched_msg *msg; + + if (q->guc->needs_cleanup) { + msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; + + guc_exec_queue_add_msg(q, msg, CLEANUP); + q->guc->needs_cleanup = false; + } + + if (q->guc->needs_suspend) { + msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; + + xe_sched_msg_lock(sched); + guc_exec_queue_try_add_msg_head(q, msg, SUSPEND); + xe_sched_msg_unlock(sched); + + q->guc->needs_suspend = false; + } + + /* + * The resume must be in the message queue before the suspend as it is + * not possible for a resume to be issued if a suspend pending is, but + * the inverse is possible. + */ + if (q->guc->needs_resume) { + msg = q->guc->static_msgs + STATIC_MSG_RESUME; + + xe_sched_msg_lock(sched); + guc_exec_queue_try_add_msg_head(q, msg, RESUME); + xe_sched_msg_unlock(sched); + + q->guc->needs_resume = false; + } +} + +static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + bool needs_tdr = exec_queue_killed_or_banned_or_wedged(q); + + lockdep_assert_held(&guc->submission_state.lock); + + xe_sched_resubmit_jobs(sched); + guc_exec_queue_replay_pending_state_change(q); + xe_sched_submission_start(sched); + if (needs_tdr) + xe_guc_exec_queue_trigger_cleanup(q); + xe_sched_submission_resume_tdr(sched); +} + +/** + * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be enabled + */ +void xe_guc_submit_unpause(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + xe_sched_submission_start(&q->guc->sched); + mutex_unlock(&guc->submission_state.lock); +} + +/** + * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF. + * @guc: the &xe_guc struct instance whose scheduler is to be enabled + */ +void xe_guc_submit_unpause_vf(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + /* + * Prevent redundant attempts to stop parallel queues, or queues + * created after resfix done. + */ + if (q->guc->id != index || + !drm_sched_is_stopped(&q->guc->sched.base)) + continue; + + guc_exec_queue_unpause(guc, q); + } + mutex_unlock(&guc->submission_state.lock); +} + +/** + * xe_guc_submit_pause_abort - Abort all paused submission task on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be aborted + */ +void xe_guc_submit_pause_abort(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + struct xe_gpu_scheduler *sched = &q->guc->sched; + + /* Prevent redundant attempts to stop parallel queues */ + if (q->guc->id != index) + continue; + + xe_sched_submission_start(sched); + guc_exec_queue_kill(q); + } + mutex_unlock(&guc->submission_state.lock); +} + +static struct xe_exec_queue * +g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_exec_queue *q; + + if (unlikely(guc_id >= GUC_ID_MAX)) { + xe_gt_err(gt, "Invalid guc_id %u\n", guc_id); + return NULL; + } + + q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); + if (unlikely(!q)) { + xe_gt_err(gt, "No exec queue found for guc_id %u\n", guc_id); + return NULL; + } + + xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id); + xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width)); + + return q; +} + +static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) +{ + u32 action[] = { + XE_GUC_ACTION_DEREGISTER_CONTEXT, + q->guc->id, + }; + + xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); + + trace_xe_exec_queue_deregister(q); + + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_deregister_done(guc, q); + else + xe_guc_ct_send_g2h_handler(&guc->ct, action, + ARRAY_SIZE(action)); +} + +static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, + u32 runnable_state) +{ + trace_xe_exec_queue_scheduling_done(q); + + if (runnable_state == 1) { + xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); + + q->guc->resume_time = ktime_get(); + clear_exec_queue_pending_resume(q); + clear_exec_queue_pending_enable(q); + smp_wmb(); + wake_up_all(&guc->ct.wq); + } else { + xe_gt_assert(guc_to_gt(guc), runnable_state == 0); + xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); + + if (q->guc->suspend_pending) { + clear_exec_queue_pending_disable(q); + suspend_fence_signal(q); + } else { + if (exec_queue_banned(q)) { + smp_wmb(); + wake_up_all(&guc->ct.wq); + } + if (exec_queue_destroyed(q)) { + /* + * Make sure to clear the pending_disable only + * after sampling the destroyed state. We want + * to ensure we don't trigger the unregister too + * early with something intending to only + * disable scheduling. The caller doing the + * destroy must wait for an ongoing + * pending_disable before marking as destroyed. + */ + clear_exec_queue_pending_disable(q); + deregister_exec_queue(guc, q); + } else { + clear_exec_queue_pending_disable(q); + } + } + } +} + +static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, + struct xe_exec_queue *q, + u32 runnable_state) +{ + /* Take CT lock here as handle_sched_done() do send a h2g message */ + mutex_lock(&guc->ct.lock); + handle_sched_done(guc, q, runnable_state); + mutex_unlock(&guc->ct.lock); +} + +int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_exec_queue *q; + u32 guc_id, runnable_state; + + if (unlikely(len < 2)) + return -EPROTO; + + guc_id = msg[0]; + runnable_state = msg[1]; + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + if (unlikely(!exec_queue_pending_enable(q) && + !exec_queue_pending_disable(q))) { + xe_gt_err(guc_to_gt(guc), + "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", + atomic_read(&q->guc->state), q->guc->id, + runnable_state); + return -EPROTO; + } + + handle_sched_done(guc, q, runnable_state); + + return 0; +} + +static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) +{ + trace_xe_exec_queue_deregister_done(q); + + clear_exec_queue_registered(q); + __guc_exec_queue_destroy(guc, q); +} + +int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_exec_queue *q; + u32 guc_id; + + if (unlikely(len < 1)) + return -EPROTO; + + guc_id = msg[0]; + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || + exec_queue_pending_enable(q) || exec_queue_enabled(q)) { + xe_gt_err(guc_to_gt(guc), + "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", + atomic_read(&q->guc->state), q->guc->id); + return -EPROTO; + } + + handle_deregister_done(guc, q); + + return 0; +} + +int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_exec_queue *q; + u32 guc_id; + + if (unlikely(len < 1)) + return -EPROTO; + + guc_id = msg[0]; + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + if (!exec_queue_killed(q)) + xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", + xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, + atomic_read(&q->guc->state)); + + trace_xe_exec_queue_reset(q); + + /* + * A banned engine is a NOP at this point (came from + * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel + * jobs by setting timeout of the job to the minimum value kicking + * guc_exec_queue_timedout_job. + */ + xe_guc_exec_queue_reset_trigger_cleanup(q); + + return 0; +} + +/* + * xe_guc_error_capture_handler - Handler of GuC captured message + * @guc: The GuC object + * @msg: Point to the message + * @len: The message length + * + * When GuC captured data is ready, GuC will send message + * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be + * called 1st to check status before process the data comes with the message. + * + * Returns: error code. 0 if success + */ +int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + u32 status; + + if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN)) + return -EPROTO; + + status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK; + if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE) + xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space"); + + xe_guc_capture_process(guc); + + return 0; +} + +int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, + u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_exec_queue *q; + u32 guc_id; + u32 type = XE_GUC_CAT_ERR_TYPE_INVALID; + + if (unlikely(!len || len > 2)) + return -EPROTO; + + guc_id = msg[0]; + + if (len == 2) + type = msg[1]; + + if (guc_id == GUC_ID_UNKNOWN) { + /* + * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF + * context. In such case only PF will be notified about that fault. + */ + xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n"); + return 0; + } + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + /* + * The type is HW-defined and changes based on platform, so we don't + * decode it in the kernel and only check if it is valid. + * See bspec 54047 and 72187 for details. + */ + if (type != XE_GUC_CAT_ERR_TYPE_INVALID) + xe_gt_info(gt, + "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d", + type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); + else + xe_gt_info(gt, + "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d", + xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); + + trace_xe_exec_queue_memory_cat_error(q); + + /* Treat the same as engine reset */ + xe_guc_exec_queue_reset_trigger_cleanup(q); + + return 0; +} + +int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_exec_queue *q; + u32 guc_id; + + if (unlikely(!len || len > 1)) + return -EPROTO; + + guc_id = msg[0]; + + if (guc_id == GUC_ID_UNKNOWN) { + xe_gt_err(gt, "GuC: Uncorrectable local error with unknown GuC id\n"); + return 0; + } + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + xe_gt_err(gt, + "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x", + guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask); + + trace_xe_guc_uncorrectable_error(q); + + /* Treat the same as engine reset */ + xe_guc_exec_queue_reset_trigger_cleanup(q); + + return 0; +} + +int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + u8 guc_class, instance; + u32 reason; + + if (unlikely(len != 3)) + return -EPROTO; + + guc_class = msg[0]; + instance = msg[1]; + reason = msg[2]; + + /* Unexpected failure of a hardware feature, log an actual error */ + xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X", + guc_class, instance, reason); + + xe_gt_reset_async(gt); + + return 0; +} + +int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg, + u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = guc_to_xe(guc); + struct xe_exec_queue *q; + u32 guc_id = msg[2]; + + if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + xe_gt_dbg(gt, + "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x", + msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]); + + trace_xe_exec_queue_cgp_context_error(q); + + /* Treat the same as engine reset */ + xe_guc_exec_queue_reset_trigger_cleanup(q); + + return 0; +} + +/** + * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler + * @guc: guc + * @msg: message indicating CGP sync done + * @len: length of message + * + * Set multi queue group's sync_pending flag to false and wakeup anyone waiting + * for CGP synchronization to complete. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_exec_queue *q; + u32 guc_id = msg[0]; + + if (unlikely(len < 1)) { + drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len); + return -EPROTO; + } + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + if (!xe_exec_queue_is_multi_queue_primary(q)) { + drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response"); + return -EPROTO; + } + + /* Wakeup the serialized cgp update wait */ + WRITE_ONCE(q->multi_queue.group->sync_pending, false); + xe_guc_ct_wake_waiters(&guc->ct); + + return 0; +} + +static void +guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, + struct xe_guc_submit_exec_queue_snapshot *snapshot) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); + int i; + + snapshot->guc.wqi_head = q->guc->wqi_head; + snapshot->guc.wqi_tail = q->guc->wqi_tail; + snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); + snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); + snapshot->parallel.wq_desc.status = parallel_read(xe, map, + wq_desc.wq_status); + + if (snapshot->parallel.wq_desc.head != + snapshot->parallel.wq_desc.tail) { + for (i = snapshot->parallel.wq_desc.head; + i != snapshot->parallel.wq_desc.tail; + i = (i + sizeof(u32)) % WQ_SIZE) + snapshot->parallel.wq[i / sizeof(u32)] = + parallel_read(xe, map, wq[i / sizeof(u32)]); + } +} + +static void +guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, + struct drm_printer *p) +{ + int i; + + drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", + snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); + drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", + snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); + drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); + + if (snapshot->parallel.wq_desc.head != + snapshot->parallel.wq_desc.tail) { + for (i = snapshot->parallel.wq_desc.head; + i != snapshot->parallel.wq_desc.tail; + i = (i + sizeof(u32)) % WQ_SIZE) + drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), + snapshot->parallel.wq[i / sizeof(u32)]); + } +} + +/** + * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. + * @q: faulty exec queue + * + * This can be printed out in a later stage like during dev_coredump + * analysis. + * + * Returns: a GuC Submit Engine snapshot object that must be freed by the + * caller, using `xe_guc_exec_queue_snapshot_free`. + */ +struct xe_guc_submit_exec_queue_snapshot * +xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_guc_submit_exec_queue_snapshot *snapshot; + int i; + + snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC); + + if (!snapshot) + return NULL; + + snapshot->guc.id = q->guc->id; + memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); + snapshot->class = q->class; + snapshot->logical_mask = q->logical_mask; + snapshot->width = q->width; + snapshot->refcount = kref_read(&q->refcount); + snapshot->sched_timeout = sched->base.timeout; + snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; + snapshot->sched_props.preempt_timeout_us = + q->sched_props.preempt_timeout_us; + + snapshot->lrc = kmalloc_objs(struct xe_lrc_snapshot *, q->width, + GFP_ATOMIC); + + if (snapshot->lrc) { + for (i = 0; i < q->width; ++i) { + struct xe_lrc *lrc = q->lrc[i]; + + snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); + } + } + + snapshot->schedule_state = atomic_read(&q->guc->state); + snapshot->exec_queue_flags = q->flags; + + snapshot->parallel_execution = xe_exec_queue_is_parallel(q); + if (snapshot->parallel_execution) + guc_exec_queue_wq_snapshot_capture(q, snapshot); + + if (xe_exec_queue_is_multi_queue(q)) { + snapshot->multi_queue.valid = true; + snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id; + snapshot->multi_queue.pos = q->multi_queue.pos; + } + + return snapshot; +} + +/** + * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. + * @snapshot: Previously captured snapshot of job. + * + * This captures some data that requires taking some locks, so it cannot be done in signaling path. + */ +void +xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) +{ + int i; + + if (!snapshot || !snapshot->lrc) + return; + + for (i = 0; i < snapshot->width; ++i) + xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); +} + +/** + * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. + * @snapshot: GuC Submit Engine snapshot object. + * @p: drm_printer where it will be printed out. + * + * This function prints out a given GuC Submit Engine snapshot object. + */ +void +xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, + struct drm_printer *p) +{ + int i; + + if (!snapshot) + return; + + drm_printf(p, "GuC ID: %d\n", snapshot->guc.id); + drm_printf(p, "\tName: %s\n", snapshot->name); + drm_printf(p, "\tClass: %d\n", snapshot->class); + drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); + drm_printf(p, "\tWidth: %d\n", snapshot->width); + drm_printf(p, "\tRef: %d\n", snapshot->refcount); + drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); + drm_printf(p, "\tTimeslice: %u (us)\n", + snapshot->sched_props.timeslice_us); + drm_printf(p, "\tPreempt timeout: %u (us)\n", + snapshot->sched_props.preempt_timeout_us); + + for (i = 0; snapshot->lrc && i < snapshot->width; ++i) + xe_lrc_snapshot_print(snapshot->lrc[i], p); + + drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); + drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); + + if (snapshot->parallel_execution) + guc_exec_queue_wq_snapshot_print(snapshot, p); + + if (snapshot->multi_queue.valid) { + drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary); + drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos); + } +} + +/** + * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given + * snapshot. + * @snapshot: GuC Submit Engine snapshot object. + * + * This function free all the memory that needed to be allocated at capture + * time. + */ +void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) +{ + int i; + + if (!snapshot) + return; + + if (snapshot->lrc) { + for (i = 0; i < snapshot->width; i++) + xe_lrc_snapshot_free(snapshot->lrc[i]); + kfree(snapshot->lrc); + } + kfree(snapshot); +} + +static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) +{ + struct xe_guc_submit_exec_queue_snapshot *snapshot; + + snapshot = xe_guc_exec_queue_snapshot_capture(q); + xe_guc_exec_queue_snapshot_print(snapshot, p); + xe_guc_exec_queue_snapshot_free(snapshot); +} + +/** + * xe_guc_register_vf_exec_queue - Register exec queue for a given context type. + * @q: Execution queue + * @ctx_type: Type of the context + * + * This function registers the execution queue with the guc. Special context + * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE + * are only applicable for IGPU and in the VF. + * Submits the execution queue to GUC after registering it. + * + * Returns - None. + */ +void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + + xe_gt_assert(gt, IS_SRIOV_VF(xe)); + xe_gt_assert(gt, !IS_DGFX(xe)); + xe_gt_assert(gt, ctx_type == GUC_CONTEXT_COMPRESSION_SAVE || + ctx_type == GUC_CONTEXT_COMPRESSION_RESTORE); + xe_gt_assert(gt, GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 23, 0)); + + register_exec_queue(q, ctx_type); + enable_scheduling(q); +} + +/** + * xe_guc_submit_print - GuC Submit Print. + * @guc: GuC. + * @p: drm_printer where it will be printed out. + * + * This function capture and prints snapshots of **all** GuC Engines. + */ +void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) +{ + struct xe_exec_queue *q; + unsigned long index; + + if (!xe_device_uc_enabled(guc_to_xe(guc))) + return; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + guc_exec_queue_print(q, p); + mutex_unlock(&guc->submission_state.lock); +} + +/** + * xe_guc_has_registered_mlrc_queues - check whether there are any MLRC queues + * registered with the GuC + * @guc: GuC. + * + * Return: true if any MLRC queue is registered with the GuC, false otherwise. + */ +bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + guard(mutex)(&guc->submission_state.lock); + + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + if (q->width > 1) + return true; + + return false; +} + +/** + * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all + * exec queues registered to given GuC. + * @guc: the &xe_guc struct instance + * @scratch: scratch buffer to be used as temporary storage + * + * Returns: zero on success, negative error code on failure. + */ +int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) +{ + struct xe_exec_queue *q; + unsigned long index; + int err = 0; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + /* Prevent redundant attempts to stop parallel queues */ + if (q->guc->id != index) + continue; + + err = xe_exec_queue_contexts_hwsp_rebase(q, scratch); + if (err) + break; + } + mutex_unlock(&guc->submission_state.lock); + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h new file mode 100644 index 000000000..ccade320d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_SUBMIT_H_ +#define _XE_GUC_SUBMIT_H_ + +#include + +struct drm_printer; +struct xe_exec_queue; +struct xe_guc; + +int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids); +int xe_guc_submit_enable(struct xe_guc *guc); +void xe_guc_submit_disable(struct xe_guc *guc); + +int xe_guc_submit_reset_prepare(struct xe_guc *guc); +void xe_guc_submit_reset_wait(struct xe_guc *guc); +void xe_guc_submit_stop(struct xe_guc *guc); +int xe_guc_submit_start(struct xe_guc *guc); +void xe_guc_submit_pause(struct xe_guc *guc); +void xe_guc_submit_pause_abort(struct xe_guc *guc); +void xe_guc_submit_pause_vf(struct xe_guc *guc); +void xe_guc_submit_unpause(struct xe_guc *guc); +void xe_guc_submit_unpause_vf(struct xe_guc *guc); +void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc); +void xe_guc_submit_wedge(struct xe_guc *guc); + +int xe_guc_read_stopped(struct xe_guc *guc); +int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, + u32 len); +int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg, + u32 len); + +struct xe_guc_submit_exec_queue_snapshot * +xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q); +void +xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot); +void +xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, + struct drm_printer *p); +void +xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot); +void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); +void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type); + +bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc); + +int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h new file mode 100644 index 000000000..7824f61b1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GUC_SUBMIT_TYPES_H_ +#define _XE_GUC_SUBMIT_TYPES_H_ + +#include "xe_hw_engine_types.h" + +/* Work item for submitting workloads into work queue of GuC. */ +#define WQ_STATUS_ACTIVE 1 +#define WQ_STATUS_SUSPENDED 2 +#define WQ_STATUS_CMD_ERROR 3 +#define WQ_STATUS_ENGINE_ID_NOT_USED 4 +#define WQ_STATUS_SUSPENDED_FROM_RESET 5 +#define WQ_TYPE_NOOP 0x4 +#define WQ_TYPE_MULTI_LRC 0x5 +#define WQ_TYPE_MASK GENMASK(7, 0) +#define WQ_LEN_MASK GENMASK(26, 16) + +#define WQ_GUC_ID_MASK GENMASK(15, 0) +#define WQ_RING_TAIL_MASK GENMASK(28, 18) + +#define PARALLEL_SCRATCH_SIZE 2048 +#define WQ_SIZE (PARALLEL_SCRATCH_SIZE / 2) +#define WQ_OFFSET (PARALLEL_SCRATCH_SIZE - WQ_SIZE) +#define CACHELINE_BYTES 64 + +struct guc_sched_wq_desc { + u32 head; + u32 tail; + u32 error_offset; + u32 wq_status; + u32 reserved[28]; +} __packed; + +struct sync_semaphore { + u32 semaphore; + u8 unused[CACHELINE_BYTES - sizeof(u32)]; +}; + +/** + * struct guc_submit_parallel_scratch - A scratch shared mapped buffer. + */ +struct guc_submit_parallel_scratch { + /** @wq_desc: Guc scheduler workqueue descriptor */ + struct guc_sched_wq_desc wq_desc; + + /** @go: Go Semaphore */ + struct sync_semaphore go; + /** @join: Joined semaphore for the relevant hw engine instances */ + struct sync_semaphore join[XE_HW_ENGINE_MAX_INSTANCE]; + + /** @unused: Unused/Reserved memory space */ + u8 unused[WQ_OFFSET - sizeof(struct guc_sched_wq_desc) - + sizeof(struct sync_semaphore) * + (XE_HW_ENGINE_MAX_INSTANCE + 1)]; + + /** @wq: Workqueue info */ + u32 wq[WQ_SIZE / sizeof(u32)]; +}; + +/** + * struct xe_guc_submit_exec_queue_snapshot - Snapshot for devcoredump + */ +struct xe_guc_submit_exec_queue_snapshot { + /** @name: name of this exec queue */ + char name[MAX_FENCE_NAME_LEN]; + /** @class: class of this exec queue */ + enum xe_engine_class class; + /** + * @logical_mask: logical mask of where job submitted to exec queue can run + */ + u32 logical_mask; + /** @width: width (number BB submitted per exec) of this exec queue */ + u16 width; + /** @refcount: ref count of this exec queue */ + u32 refcount; + /** + * @sched_timeout: the time after which a job is removed from the + * scheduler. + */ + long sched_timeout; + + /** @sched_props: scheduling properties */ + struct { + /** @sched_props.timeslice_us: timeslice period in micro-seconds */ + u32 timeslice_us; + /** @sched_props.preempt_timeout_us: preemption timeout in micro-seconds */ + u32 preempt_timeout_us; + } sched_props; + + /** @lrc: LRC Snapshot */ + struct xe_lrc_snapshot **lrc; + + /** @schedule_state: Schedule State at the moment of Crash */ + u32 schedule_state; + /** @exec_queue_flags: Flags of the faulty exec_queue */ + unsigned long exec_queue_flags; + + /** @guc: GuC Engine Snapshot */ + struct { + /** @guc.wqi_head: work queue item head */ + u32 wqi_head; + /** @guc.wqi_tail: work queue item tail */ + u32 wqi_tail; + /** @guc.id: GuC id for this exec_queue */ + u16 id; + } guc; + + /** + * @parallel_execution: Indication if the failure was during parallel + * execution + */ + bool parallel_execution; + /** @parallel: snapshot of the useful parallel scratch */ + struct { + /** @parallel.wq_desc: Workqueue description */ + struct { + /** @parallel.wq_desc.head: Workqueue Head */ + u32 head; + /** @parallel.wq_desc.tail: Workqueue Tail */ + u32 tail; + /** @parallel.wq_desc.status: Workqueue Status */ + u32 status; + } wq_desc; + /** @wq: Workqueue Items */ + u32 wq[WQ_SIZE / sizeof(u32)]; + } parallel; + + /** @multi_queue: snapshot of the multi queue information */ + struct { + /** + * @multi_queue.primary: GuC id of the primary exec queue + * of the multi queue group. + */ + u32 primary; + /** @multi_queue.pos: Position of the exec queue within the multi queue group */ + u8 pos; + /** @multi_queue.valid: The exec queue is part of a multi queue group */ + bool valid; + } multi_queue; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c new file mode 100644 index 000000000..046d06551 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "abi/guc_actions_abi.h" + +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_exec_queue_types.h" +#include "xe_gt_stats.h" +#include "xe_gt_types.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_guc_tlb_inval.h" +#include "xe_force_wake.h" +#include "xe_mmio.h" +#include "xe_sa.h" +#include "xe_tlb_inval.h" +#include "xe_vm.h" + +#include "regs/xe_guc_regs.h" + +/* + * XXX: The seqno algorithm relies on TLB invalidation being processed in order + * which they currently are by the GuC, if that changes the algorithm will need + * to be updated. + */ + +static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len) +{ + struct xe_gt *gt = guc_to_gt(guc); + + xe_gt_assert(gt, action[1]); /* Seqno */ + + xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); + return xe_guc_ct_send(&guc->ct, action, len, + G2H_LEN_DW_TLB_INVALIDATE, 1); +} + +#define MAKE_INVAL_OP_FLUSH(type, flush_cache) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ + XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \ + (flush_cache ? \ + XE_GUC_TLB_INVAL_FLUSH_CACHE : 0)) + +#define MAKE_INVAL_OP(type) MAKE_INVAL_OP_FLUSH(type, true) + +static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno) +{ + struct xe_guc *guc = tlb_inval->private; + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION_ALL, + seqno, + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL), + }; + + return send_tlb_inval(guc, action, ARRAY_SIZE(action)); +} + +static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) +{ + struct xe_guc *guc = tlb_inval->private; + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = guc_to_xe(guc); + + /* + * Returning -ECANCELED in this function is squashed at the caller and + * signals waiters. + */ + + if (xe_guc_ct_enabled(&guc->ct) && guc->submission_state.enabled) { + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION, + seqno, + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), + }; + + return send_tlb_inval(guc, action, ARRAY_SIZE(action)); + } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { + struct xe_mmio *mmio = >->mmio; + + if (IS_SRIOV_VF(xe)) + return -ECANCELED; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1, + PVC_GUC_TLB_INV_DESC1_INVALIDATE); + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0, + PVC_GUC_TLB_INV_DESC0_VALID); + } else { + xe_mmio_write32(mmio, GUC_TLB_INV_CR, + GUC_TLB_INV_CR_INVALIDATE); + } + } + + return -ECANCELED; +} + +static int send_page_reclaim(struct xe_guc *guc, u32 seqno, + u64 gpu_addr) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 action[] = { + XE_GUC_ACTION_PAGE_RECLAMATION, + seqno, + lower_32_bits(gpu_addr), + upper_32_bits(gpu_addr), + }; + + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_ISSUED_COUNT, 1); + + return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_PAGE_RECLAMATION, 1); +} + +static u64 normalize_invalidation_range(struct xe_gt *gt, u64 *start, u64 *end) +{ + u64 orig_start = *start; + u64 length = *end - *start; + u64 align; + + if (length < SZ_4K) + length = SZ_4K; + + align = roundup_pow_of_two(length); + *start = ALIGN_DOWN(*start, align); + *end = ALIGN(*end, align); + length = align; + while (*start + length < *end) { + length <<= 1; + *start = ALIGN_DOWN(orig_start, length); + } + + if (length >= SZ_2M) { + length = max_t(u64, SZ_16M, length); + *start = ALIGN_DOWN(orig_start, length); + } + + xe_gt_assert(gt, length >= SZ_4K); + xe_gt_assert(gt, is_power_of_2(length)); + xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, + ilog2(SZ_2M) + 1))); + xe_gt_assert(gt, IS_ALIGNED(*start, length)); + + return length; +} + +/* + * Ensure that roundup_pow_of_two(length) doesn't overflow. + * Note that roundup_pow_of_two() operates on unsigned long, + * not on u64. + */ +#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) + +static int send_tlb_inval_ppgtt(struct xe_guc *guc, u32 seqno, u64 start, + u64 end, u32 id, u32 type, + struct drm_suballoc *prl_sa) +{ +#define MAX_TLB_INVALIDATION_LEN 7 + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = guc_to_xe(guc); + u32 action[MAX_TLB_INVALIDATION_LEN]; + u64 length = end - start; + int len = 0, err; + + xe_gt_assert(gt, (type == XE_GUC_TLB_INVAL_PAGE_SELECTIVE && + !xe->info.has_ctx_tlb_inval) || + (type == XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX && + xe->info.has_ctx_tlb_inval)); + + action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; + action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID; + if (!gt_to_xe(gt)->info.has_range_tlb_inval || + length > MAX_RANGE_TLB_INVALIDATION_LENGTH) { + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); + } else { + u64 normalize_len = normalize_invalidation_range(gt, &start, + &end); + bool need_flush = !prl_sa && + seqno != TLB_INVALIDATION_SEQNO_INVALID; + + /* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */ + action[len++] = MAKE_INVAL_OP_FLUSH(type, need_flush); + action[len++] = id; + action[len++] = lower_32_bits(start); + action[len++] = upper_32_bits(start); + action[len++] = ilog2(normalize_len) - ilog2(SZ_4K); + } + + xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); +#undef MAX_TLB_INVALIDATION_LEN + + err = send_tlb_inval(guc, action, len); + if (!err && prl_sa) { + xe_gt_assert(gt, seqno != TLB_INVALIDATION_SEQNO_INVALID); + err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa)); + } + return err; +} + +static int send_tlb_inval_asid_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, + u64 start, u64 end, u32 asid, + struct drm_suballoc *prl_sa) +{ + struct xe_guc *guc = tlb_inval->private; + + lockdep_assert_held(&tlb_inval->seqno_lock); + + return send_tlb_inval_ppgtt(guc, seqno, start, end, asid, + XE_GUC_TLB_INVAL_PAGE_SELECTIVE, prl_sa); +} + +static int send_tlb_inval_ctx_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, + u64 start, u64 end, u32 asid, + struct drm_suballoc *prl_sa) +{ + struct xe_guc *guc = tlb_inval->private; + struct xe_device *xe = guc_to_xe(guc); + struct xe_exec_queue *q, *next, *last_q = NULL; + struct xe_vm *vm; + LIST_HEAD(tlb_inval_list); + int err = 0, id = guc_to_gt(guc)->info.id; + + lockdep_assert_held(&tlb_inval->seqno_lock); + + vm = xe_device_asid_to_vm(xe, asid); + if (IS_ERR(vm)) + return PTR_ERR(vm); + + down_read(&vm->exec_queues.lock); + + /* + * XXX: Randomly picking a threshold for now. This will need to be + * tuned based on expected UMD queue counts and performance profiling. + */ +#define EXEC_QUEUE_COUNT_FULL_THRESHOLD 8 + if (vm->exec_queues.count[id] >= EXEC_QUEUE_COUNT_FULL_THRESHOLD) { + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION, + seqno, + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL), + }; + + err = send_tlb_inval(guc, action, ARRAY_SIZE(action)); + goto err_unlock; + } +#undef EXEC_QUEUE_COUNT_FULL_THRESHOLD + + /* + * Move exec queues to a temporary list to issue invalidations. A + * reference must be taken to prevent concurrent deregistrations. + * + * List modification is safe because we hold 'vm->exec_queues.lock' for + * reading, which prevents external modifications. Using a per-GT list + * is also safe since 'tlb_inval->seqno_lock' ensures no other GT users + * can enter this code path. + */ + list_for_each_entry_safe(q, next, &vm->exec_queues.list[id], + vm_exec_queue_link) { + if (xe_exec_queue_get_unless_zero(q)) { + last_q = q; + list_move_tail(&q->vm_exec_queue_link, &tlb_inval_list); + } + } + + if (!last_q) { + /* + * We can't break fence ordering for TLB invalidation jobs, if + * TLB invalidations are inflight issue a dummy invalidation to + * maintain ordering. Nor can we move safely the seqno_recv when + * returning -ECANCELED if TLB invalidations are in flight. Use + * GGTT invalidation as dummy invalidation given ASID + * invalidations are unsupported here. + */ + if (xe_tlb_inval_idle(tlb_inval)) + err = -ECANCELED; + else + err = send_tlb_inval_ggtt(tlb_inval, seqno); + goto err_unlock; + } + + list_for_each_entry_safe(q, next, &tlb_inval_list, vm_exec_queue_link) { + struct drm_suballoc *__prl_sa = NULL; + int __seqno = TLB_INVALIDATION_SEQNO_INVALID; + u32 type = XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX; + + xe_assert(xe, q->vm == vm); + + if (err) + goto unref; + + if (last_q == q) { + __prl_sa = prl_sa; + __seqno = seqno; + } + + err = send_tlb_inval_ppgtt(guc, __seqno, start, end, + q->guc->id, type, __prl_sa); + +unref: + /* + * Must always return exec queue to original list / drop + * reference + */ + list_move_tail(&q->vm_exec_queue_link, + &vm->exec_queues.list[id]); + xe_exec_queue_put(q); + } + +err_unlock: + up_read(&vm->exec_queues.lock); + xe_vm_put(vm); + + return err; +} + +static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval) +{ + struct xe_guc *guc = tlb_inval->private; + + return xe_guc_ct_initialized(&guc->ct); +} + +static void tlb_inval_flush(struct xe_tlb_inval *tlb_inval) +{ + struct xe_guc *guc = tlb_inval->private; + + LNL_FLUSH_WORK(&guc->ct.g2h_worker); +} + +static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval) +{ + struct xe_guc *guc = tlb_inval->private; + + /* this reflects what HW/GuC needs to process TLB inv request */ + const long hw_tlb_timeout = HZ / 4; + + /* this estimates actual delay caused by the CTB transport */ + long delay = xe_guc_ct_queue_proc_time_jiffies(&guc->ct); + + return hw_tlb_timeout + 2 * delay; +} + +static const struct xe_tlb_inval_ops guc_tlb_inval_asid_ops = { + .all = send_tlb_inval_all, + .ggtt = send_tlb_inval_ggtt, + .ppgtt = send_tlb_inval_asid_ppgtt, + .initialized = tlb_inval_initialized, + .flush = tlb_inval_flush, + .timeout_delay = tlb_inval_timeout_delay, +}; + +static const struct xe_tlb_inval_ops guc_tlb_inval_ctx_ops = { + .ggtt = send_tlb_inval_ggtt, + .all = send_tlb_inval_all, + .ppgtt = send_tlb_inval_ctx_ppgtt, + .initialized = tlb_inval_initialized, + .flush = tlb_inval_flush, + .timeout_delay = tlb_inval_timeout_delay, +}; + +/** + * xe_guc_tlb_inval_init_early() - Init GuC TLB invalidation early + * @guc: GuC object + * @tlb_inval: TLB invalidation client + * + * Initialize GuC TLB invalidation by setting back pointer in TLB invalidation + * client to the GuC and setting GuC backend ops. + */ +void xe_guc_tlb_inval_init_early(struct xe_guc *guc, + struct xe_tlb_inval *tlb_inval) +{ + struct xe_device *xe = guc_to_xe(guc); + + tlb_inval->private = guc; + + if (xe->info.has_ctx_tlb_inval) + tlb_inval->ops = &guc_tlb_inval_ctx_ops; + else + tlb_inval->ops = &guc_tlb_inval_asid_ops; +} + +/** + * xe_guc_tlb_inval_done_handler() - TLB invalidation done handler + * @guc: guc + * @msg: message indicating TLB invalidation done + * @len: length of message + * + * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any + * invalidation fences for seqno. Algorithm for this depends on seqno being + * received in-order and asserts this assumption. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + + if (unlikely(len != 1)) + return -EPROTO; + + xe_tlb_inval_done_handler(>->tlb_inval, msg[0]); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.h b/drivers/gpu/drm/xe/xe_guc_tlb_inval.h new file mode 100644 index 000000000..07d668b02 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GUC_TLB_INVAL_H_ +#define _XE_GUC_TLB_INVAL_H_ + +#include + +struct xe_guc; +struct xe_tlb_inval; + +void xe_guc_tlb_inval_init_early(struct xe_guc *guc, + struct xe_tlb_inval *tlb_inval); + +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h new file mode 100644 index 000000000..31a2acb63 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_types.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_TYPES_H_ +#define _XE_GUC_TYPES_H_ + +#include +#include + +#include "regs/xe_reg_defs.h" +#include "xe_guc_ads_types.h" +#include "xe_guc_buf_types.h" +#include "xe_guc_ct_types.h" +#include "xe_guc_engine_activity_types.h" +#include "xe_guc_fwif.h" +#include "xe_guc_log_types.h" +#include "xe_guc_pc_types.h" +#include "xe_guc_relay_types.h" +#include "xe_uc_fw_types.h" + +/** + * struct xe_guc_db_mgr - GuC Doorbells Manager. + * + * Note: GuC Doorbells Manager is relying on &xe_guc::submission_state.lock + * to protect its members. + */ +struct xe_guc_db_mgr { + /** @count: number of doorbells to manage */ + unsigned int count; + /** @bitmap: bitmap to track allocated doorbells */ + unsigned long *bitmap; +}; + +/** + * struct xe_guc_id_mgr - GuC context ID Manager. + * + * Note: GuC context ID Manager is relying on &xe_guc::submission_state.lock + * to protect its members. + */ +struct xe_guc_id_mgr { + /** @bitmap: bitmap to track allocated IDs */ + unsigned long *bitmap; + /** @total: total number of IDs being managed */ + unsigned int total; + /** @used: number of IDs currently in use */ + unsigned int used; +}; + +/** + * struct xe_guc - Graphic micro controller + */ +struct xe_guc { + /** @fw: Generic uC firmware management */ + struct xe_uc_fw fw; + /** @log: GuC log */ + struct xe_guc_log log; + /** @ads: GuC ads */ + struct xe_guc_ads ads; + /** @ct: GuC ct */ + struct xe_guc_ct ct; + /** @buf: GuC Buffer Cache manager */ + struct xe_guc_buf_cache buf; + /** @capture: the error-state-capture module's data and objects */ + struct xe_guc_state_capture *capture; + /** @pc: GuC Power Conservation */ + struct xe_guc_pc pc; + /** @dbm: GuC Doorbell Manager */ + struct xe_guc_db_mgr dbm; + + /** @g2g: GuC to GuC communication state */ + struct { + /** @g2g.bo: Storage for GuC to GuC communication channels */ + struct xe_bo *bo; + /** @g2g.owned: Is the BO owned by this GT or just mapped in */ + bool owned; + } g2g; + + /** @submission_state: GuC submission state */ + struct { + /** @submission_state.idm: GuC context ID Manager */ + struct xe_guc_id_mgr idm; + /** @submission_state.exec_queue_lookup: Lookup an xe_engine from guc_id */ + struct xarray exec_queue_lookup; + /** @submission_state.stopped: submissions are stopped */ + atomic_t stopped; + /** + * @submission_state.reset_blocked: reset attempts are blocked; + * blocking reset in order to delay it may be required if running + * an operation which is sensitive to resets. + */ + atomic_t reset_blocked; + /** @submission_state.lock: protects submission state */ + struct mutex lock; + /** @submission_state.enabled: submission is enabled */ + bool enabled; + /** + * @submission_state.initialized: mark when submission state is + * even initialized - before that not even the lock is valid + */ + bool initialized; + } submission_state; + + /** @hwconfig: Hardware config state */ + struct { + /** @hwconfig.bo: buffer object of the hardware config */ + struct xe_bo *bo; + /** @hwconfig.size: size of the hardware config */ + u32 size; + } hwconfig; + + /** @relay: GuC Relay Communication used in SR-IOV */ + struct xe_guc_relay relay; + + /** @engine_activity: Device specific engine activity */ + struct xe_guc_engine_activity engine_activity; + + /** + * @notify_reg: Register which is written to notify GuC of H2G messages + */ + struct xe_reg notify_reg; + /** @params: Control params for fw initialization */ + u32 params[GUC_CTL_MAX_DWORDS]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_heci_gsc.c b/drivers/gpu/drm/xe/xe_heci_gsc.c new file mode 100644 index 000000000..5af8903e1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_heci_gsc.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2023, Intel Corporation. All rights reserved. + */ + +#include +#include +#include +#include + +#include + +#include "xe_device_types.h" +#include "xe_heci_gsc.h" +#include "regs/xe_gsc_regs.h" +#include "xe_platform_types.h" +#include "xe_survivability_mode.h" + +#define GSC_BAR_LENGTH 0x00000FFC + +static void heci_gsc_irq_mask(struct irq_data *d) +{ + /* generic irq handling */ +} + +static void heci_gsc_irq_unmask(struct irq_data *d) +{ + /* generic irq handling */ +} + +static const struct irq_chip heci_gsc_irq_chip = { + .name = "gsc_irq_chip", + .irq_mask = heci_gsc_irq_mask, + .irq_unmask = heci_gsc_irq_unmask, +}; + +static int heci_gsc_irq_init(int irq) +{ + irq_set_chip_and_handler_name(irq, &heci_gsc_irq_chip, + handle_simple_irq, "heci_gsc_irq_handler"); + + return irq_set_chip_data(irq, NULL); +} + +/** + * struct heci_gsc_def - graphics security controller heci interface definitions + * + * @name: name of the heci device + * @bar: address of the mmio bar + * @bar_size: size of the mmio bar + * @use_polling: indication of using polling mode for the device + * @slow_firmware: indication of whether the device is slow (needs longer timeouts) + */ +struct heci_gsc_def { + const char *name; + unsigned long bar; + size_t bar_size; + bool use_polling; + bool slow_firmware; +}; + +/* gsc resources and definitions */ +static const struct heci_gsc_def heci_gsc_def_dg1 = { + .name = "mei-gscfi", + .bar = DG1_GSC_HECI2_BASE, + .bar_size = GSC_BAR_LENGTH, +}; + +static const struct heci_gsc_def heci_gsc_def_dg2 = { + .name = "mei-gscfi", + .bar = DG2_GSC_HECI2_BASE, + .bar_size = GSC_BAR_LENGTH, +}; + +static const struct heci_gsc_def heci_gsc_def_pvc = { + .name = "mei-gscfi", + .bar = PVC_GSC_HECI2_BASE, + .bar_size = GSC_BAR_LENGTH, + .slow_firmware = true, +}; + +static void heci_gsc_release_dev(struct device *dev) +{ + struct auxiliary_device *aux_dev = to_auxiliary_dev(dev); + struct mei_aux_device *adev = auxiliary_dev_to_mei_aux_dev(aux_dev); + + kfree(adev); +} + +static void xe_heci_gsc_fini(void *arg) +{ + struct xe_heci_gsc *heci_gsc = arg; + + if (heci_gsc->adev) { + struct auxiliary_device *aux_dev = &heci_gsc->adev->aux_dev; + + auxiliary_device_delete(aux_dev); + auxiliary_device_uninit(aux_dev); + heci_gsc->adev = NULL; + } + + if (heci_gsc->irq >= 0) + irq_free_desc(heci_gsc->irq); + + heci_gsc->irq = -1; +} + +static int heci_gsc_irq_setup(struct xe_device *xe) +{ + struct xe_heci_gsc *heci_gsc = &xe->heci_gsc; + int ret; + + heci_gsc->irq = irq_alloc_desc(0); + if (heci_gsc->irq < 0) { + drm_err(&xe->drm, "gsc irq error %d\n", heci_gsc->irq); + return heci_gsc->irq; + } + + ret = heci_gsc_irq_init(heci_gsc->irq); + if (ret < 0) + drm_err(&xe->drm, "gsc irq init failed %d\n", ret); + + return ret; +} + +static int heci_gsc_add_device(struct xe_device *xe, const struct heci_gsc_def *def) +{ + struct xe_heci_gsc *heci_gsc = &xe->heci_gsc; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct auxiliary_device *aux_dev; + struct mei_aux_device *adev; + int ret; + + adev = kzalloc_obj(*adev); + if (!adev) + return -ENOMEM; + adev->irq = heci_gsc->irq; + adev->bar.parent = &pdev->resource[0]; + adev->bar.start = def->bar + pdev->resource[0].start; + adev->bar.end = adev->bar.start + def->bar_size - 1; + adev->bar.flags = IORESOURCE_MEM; + adev->bar.desc = IORES_DESC_NONE; + adev->slow_firmware = def->slow_firmware; + + aux_dev = &adev->aux_dev; + aux_dev->name = def->name; + aux_dev->id = (pci_domain_nr(pdev->bus) << 16) | + PCI_DEVID(pdev->bus->number, pdev->devfn); + aux_dev->dev.parent = &pdev->dev; + aux_dev->dev.release = heci_gsc_release_dev; + + ret = auxiliary_device_init(aux_dev); + if (ret < 0) { + drm_err(&xe->drm, "gsc aux init failed %d\n", ret); + kfree(adev); + return ret; + } + + heci_gsc->adev = adev; /* needed by the notifier */ + ret = auxiliary_device_add(aux_dev); + if (ret < 0) { + drm_err(&xe->drm, "gsc aux add failed %d\n", ret); + heci_gsc->adev = NULL; + + /* adev will be freed with the put_device() and .release sequence */ + auxiliary_device_uninit(aux_dev); + } + return ret; +} + +int xe_heci_gsc_init(struct xe_device *xe) +{ + struct xe_heci_gsc *heci_gsc = &xe->heci_gsc; + const struct heci_gsc_def *def = NULL; + int ret; + + if (!xe->info.has_heci_gscfi && !xe->info.has_heci_cscfi) + return 0; + + heci_gsc->irq = -1; + + if (xe->info.platform == XE_BATTLEMAGE) { + def = &heci_gsc_def_dg2; + } else if (xe->info.platform == XE_PVC) { + def = &heci_gsc_def_pvc; + } else if (xe->info.platform == XE_DG2) { + def = &heci_gsc_def_dg2; + } else if (xe->info.platform == XE_DG1) { + def = &heci_gsc_def_dg1; + } + + if (!def || !def->name) { + drm_warn(&xe->drm, "HECI is not implemented!\n"); + return 0; + } + + ret = devm_add_action_or_reset(xe->drm.dev, xe_heci_gsc_fini, heci_gsc); + if (ret) + return ret; + + if (!def->use_polling && !xe_survivability_mode_is_boot_enabled(xe)) { + ret = heci_gsc_irq_setup(xe); + if (ret) + return ret; + } + + return heci_gsc_add_device(xe, def); +} + +void xe_heci_gsc_irq_handler(struct xe_device *xe, u32 iir) +{ + int ret; + + if ((iir & GSC_IRQ_INTF(1)) == 0) + return; + + if (!xe->info.has_heci_gscfi) { + drm_warn_once(&xe->drm, "GSC irq: not supported"); + return; + } + + if (xe->heci_gsc.irq < 0) + return; + + ret = generic_handle_irq_safe(xe->heci_gsc.irq); + if (ret) + drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret); +} + +void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir) +{ + int ret; + + if ((iir & CSC_IRQ_INTF(1)) == 0) + return; + + if (!xe->info.has_heci_cscfi) { + drm_warn_once(&xe->drm, "CSC irq: not supported"); + return; + } + + if (xe->heci_gsc.irq < 0) + return; + + ret = generic_handle_irq_safe(xe->heci_gsc.irq); + if (ret) + drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret); +} diff --git a/drivers/gpu/drm/xe/xe_heci_gsc.h b/drivers/gpu/drm/xe/xe_heci_gsc.h new file mode 100644 index 000000000..a76f4122b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_heci_gsc.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2023, Intel Corporation. All rights reserved. + */ +#ifndef _XE_HECI_GSC_H_ +#define _XE_HECI_GSC_H_ + +#include + +struct xe_device; +struct mei_aux_device; + +/* + * GSC HECI1 bit corresponds to bit15 and HECI2 to bit14. + * The reason for this is to allow growth for more interfaces in the future. + */ +#define GSC_IRQ_INTF(_x) BIT(15 - (_x)) + +/* + * CSC HECI1 bit corresponds to bit9 and HECI2 to bit10. + */ +#define CSC_IRQ_INTF(_x) BIT(9 + (_x)) + +/** + * struct xe_heci_gsc - graphics security controller for xe, HECI interface + * + * @adev : pointer to mei auxiliary device structure + * @irq : irq number + * + */ +struct xe_heci_gsc { + struct mei_aux_device *adev; + int irq; +}; + +int xe_heci_gsc_init(struct xe_device *xe); +void xe_heci_gsc_irq_handler(struct xe_device *xe, u32 iir); +void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir); + +#endif /* _XE_HECI_GSC_H_ */ diff --git a/drivers/gpu/drm/xe/xe_huc.c b/drivers/gpu/drm/xe/xe_huc.c new file mode 100644 index 000000000..57afe2144 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_huc.h" + +#include + +#include + +#include "abi/gsc_pxp_commands_abi.h" +#include "regs/xe_gsc_regs.h" +#include "regs/xe_guc_regs.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gsc_submit.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_guc.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_uc_fw.h" + +static struct xe_gt * +huc_to_gt(struct xe_huc *huc) +{ + return container_of(huc, struct xe_gt, uc.huc); +} + +static struct xe_device * +huc_to_xe(struct xe_huc *huc) +{ + return gt_to_xe(huc_to_gt(huc)); +} + +static struct xe_guc * +huc_to_guc(struct xe_huc *huc) +{ + return &container_of(huc, struct xe_uc, huc)->guc; +} + +#define PXP43_HUC_AUTH_INOUT_SIZE SZ_4K +static int huc_alloc_gsc_pkt(struct xe_huc *huc) +{ + struct xe_gt *gt = huc_to_gt(huc); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *bo; + + /* we use a single object for both input and output */ + bo = xe_managed_bo_create_pin_map(xe, gt_to_tile(gt), + PXP43_HUC_AUTH_INOUT_SIZE * 2, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + huc->gsc_pkt = bo; + + return 0; +} + +int xe_huc_init(struct xe_huc *huc) +{ + struct xe_gt *gt = huc_to_gt(huc); + struct xe_device *xe = gt_to_xe(gt); + int ret; + + huc->fw.type = XE_UC_FW_TYPE_HUC; + + /* + * The HuC is only available on the media GT on most platforms. The + * exception to that rule are the old Xe1 platforms where there was + * no separate GT for media IP, so the HuC was part of the primary + * GT. Such platforms have graphics versions 12.55 and earlier. + */ + if (!xe_gt_is_media_type(gt) && GRAPHICS_VERx100(xe) > 1255) { + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_NOT_SUPPORTED); + return 0; + } + + ret = xe_uc_fw_init(&huc->fw); + if (ret) + goto out; + + if (!xe_uc_fw_is_enabled(&huc->fw)) + return 0; + + if (IS_SRIOV_VF(xe)) + return 0; + + if (huc->fw.has_gsc_headers) { + ret = huc_alloc_gsc_pkt(huc); + if (ret) + goto out; + } + + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_LOADABLE); + + return 0; + +out: + xe_gt_err(gt, "HuC: initialization failed: %pe\n", ERR_PTR(ret)); + return ret; +} + +int xe_huc_init_post_hwconfig(struct xe_huc *huc) +{ + struct xe_tile *tile = gt_to_tile(huc_to_gt(huc)); + struct xe_device *xe = huc_to_xe(huc); + int ret; + + if (!IS_DGFX(huc_to_xe(huc))) + return 0; + + if (!xe_uc_fw_is_loadable(&huc->fw)) + return 0; + + ret = xe_managed_bo_reinit_in_vram(xe, tile, &huc->fw.bo); + if (ret) + return ret; + + return 0; +} + +int xe_huc_upload(struct xe_huc *huc) +{ + if (!xe_uc_fw_is_loadable(&huc->fw)) + return 0; + return xe_uc_fw_upload(&huc->fw, 0, HUC_UKERNEL); +} + +#define huc_auth_msg_wr(xe_, map_, offset_, field_, val_) \ + xe_map_wr_field(xe_, map_, offset_, struct pxp43_new_huc_auth_in, field_, val_) +#define huc_auth_msg_rd(xe_, map_, offset_, field_) \ + xe_map_rd_field(xe_, map_, offset_, struct pxp43_huc_auth_out, field_) + +static u32 huc_emit_pxp_auth_msg(struct xe_device *xe, struct iosys_map *map, + u32 wr_offset, u32 huc_offset, u32 huc_size) +{ + xe_map_memset(xe, map, wr_offset, 0, sizeof(struct pxp43_new_huc_auth_in)); + + huc_auth_msg_wr(xe, map, wr_offset, header.api_version, PXP_APIVER(4, 3)); + huc_auth_msg_wr(xe, map, wr_offset, header.command_id, PXP43_CMDID_NEW_HUC_AUTH); + huc_auth_msg_wr(xe, map, wr_offset, header.status, 0); + huc_auth_msg_wr(xe, map, wr_offset, header.buffer_len, + sizeof(struct pxp43_new_huc_auth_in) - sizeof(struct pxp_cmd_header)); + huc_auth_msg_wr(xe, map, wr_offset, huc_base_address, huc_offset); + huc_auth_msg_wr(xe, map, wr_offset, huc_size, huc_size); + + return wr_offset + sizeof(struct pxp43_new_huc_auth_in); +} + +static int huc_auth_via_gsccs(struct xe_huc *huc) +{ + struct xe_gt *gt = huc_to_gt(huc); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *pkt = huc->gsc_pkt; + u32 wr_offset; + u32 rd_offset; + u64 ggtt_offset; + u32 out_status; + int retry = 5; + int err = 0; + + if (!pkt) + return -ENODEV; + + ggtt_offset = xe_bo_ggtt_addr(pkt); + + wr_offset = xe_gsc_emit_header(xe, &pkt->vmap, 0, HECI_MEADDRESS_PXP, 0, + sizeof(struct pxp43_new_huc_auth_in)); + wr_offset = huc_emit_pxp_auth_msg(xe, &pkt->vmap, wr_offset, + xe_bo_ggtt_addr(huc->fw.bo), + xe_bo_size(huc->fw.bo)); + do { + err = xe_gsc_pkt_submit_kernel(>->uc.gsc, ggtt_offset, wr_offset, + ggtt_offset + PXP43_HUC_AUTH_INOUT_SIZE, + PXP43_HUC_AUTH_INOUT_SIZE); + if (err) + break; + + if (xe_gsc_check_and_update_pending(xe, &pkt->vmap, 0, &pkt->vmap, + PXP43_HUC_AUTH_INOUT_SIZE)) { + err = -EBUSY; + msleep(50); + } + } while (--retry && err == -EBUSY); + + if (err) { + xe_gt_err(gt, "HuC: failed to submit GSC request to auth: %pe\n", ERR_PTR(err)); + return err; + } + + err = xe_gsc_read_out_header(xe, &pkt->vmap, PXP43_HUC_AUTH_INOUT_SIZE, + sizeof(struct pxp43_huc_auth_out), &rd_offset); + if (err) { + xe_gt_err(gt, "HuC: invalid GSC reply for auth: %pe\n", ERR_PTR(err)); + return err; + } + + /* + * The GSC will return PXP_STATUS_OP_NOT_PERMITTED if the HuC is already + * authenticated. If the same error is ever returned with HuC not loaded + * we'll still catch it when we check the authentication bit later. + */ + out_status = huc_auth_msg_rd(xe, &pkt->vmap, rd_offset, header.status); + if (out_status != PXP_STATUS_SUCCESS && out_status != PXP_STATUS_OP_NOT_PERMITTED) { + xe_gt_err(gt, "HuC: authentication failed with GSC error = %#x\n", out_status); + return -EIO; + } + + return 0; +} + +static const struct { + const char *name; + struct xe_reg reg; + u32 val; +} huc_auth_modes[XE_HUC_AUTH_TYPES_COUNT] = { + [XE_HUC_AUTH_VIA_GUC] = { "GuC", + HUC_KERNEL_LOAD_INFO, + HUC_LOAD_SUCCESSFUL }, + [XE_HUC_AUTH_VIA_GSC] = { "GSC", + HECI_FWSTS5(MTL_GSC_HECI1_BASE), + HECI1_FWSTS5_HUC_AUTH_DONE }, +}; + +bool xe_huc_is_authenticated(struct xe_huc *huc, enum xe_huc_auth_types type) +{ + struct xe_gt *gt = huc_to_gt(huc); + + return xe_mmio_read32(>->mmio, huc_auth_modes[type].reg) & huc_auth_modes[type].val; +} + +int xe_huc_auth(struct xe_huc *huc, enum xe_huc_auth_types type) +{ + struct xe_gt *gt = huc_to_gt(huc); + struct xe_guc *guc = huc_to_guc(huc); + int ret; + + if (!xe_uc_fw_is_loadable(&huc->fw)) + return 0; + + /* On newer platforms the HuC survives reset, so no need to re-auth */ + if (xe_huc_is_authenticated(huc, type)) { + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_RUNNING); + return 0; + } + + if (!xe_uc_fw_is_loaded(&huc->fw)) + return -ENOEXEC; + + switch (type) { + case XE_HUC_AUTH_VIA_GUC: + ret = xe_guc_auth_huc(guc, xe_bo_ggtt_addr(huc->fw.bo) + + xe_uc_fw_rsa_offset(&huc->fw)); + break; + case XE_HUC_AUTH_VIA_GSC: + ret = huc_auth_via_gsccs(huc); + break; + default: + XE_WARN_ON(type); + return -EINVAL; + } + if (ret) { + xe_gt_err(gt, "HuC: failed to trigger auth via %s: %pe\n", + huc_auth_modes[type].name, ERR_PTR(ret)); + goto fail; + } + + ret = xe_mmio_wait32(>->mmio, huc_auth_modes[type].reg, huc_auth_modes[type].val, + huc_auth_modes[type].val, 100000, NULL, false); + if (ret) { + xe_gt_err(gt, "HuC: firmware not verified: %pe\n", ERR_PTR(ret)); + goto fail; + } + + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_RUNNING); + xe_gt_dbg(gt, "HuC: authenticated via %s\n", huc_auth_modes[type].name); + + return 0; + +fail: + xe_gt_err(gt, "HuC: authentication via %s failed: %pe\n", + huc_auth_modes[type].name, ERR_PTR(ret)); + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_LOAD_FAIL); + + return ret; +} + +void xe_huc_sanitize(struct xe_huc *huc) +{ + xe_uc_fw_sanitize(&huc->fw); +} + +void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p) +{ + struct xe_gt *gt = huc_to_gt(huc); + + xe_uc_fw_print(&huc->fw, p); + + if (!xe_uc_fw_is_enabled(&huc->fw)) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return; + + drm_printf(p, "\nHuC status: 0x%08x\n", + xe_mmio_read32(>->mmio, HUC_KERNEL_LOAD_INFO)); +} diff --git a/drivers/gpu/drm/xe/xe_huc.h b/drivers/gpu/drm/xe/xe_huc.h new file mode 100644 index 000000000..fa1c45e70 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HUC_H_ +#define _XE_HUC_H_ + +#include + +struct drm_printer; +struct xe_huc; + +enum xe_huc_auth_types { + XE_HUC_AUTH_VIA_GUC = 0, + XE_HUC_AUTH_VIA_GSC, + XE_HUC_AUTH_TYPES_COUNT +}; + +int xe_huc_init(struct xe_huc *huc); +int xe_huc_init_post_hwconfig(struct xe_huc *huc); +int xe_huc_upload(struct xe_huc *huc); +int xe_huc_auth(struct xe_huc *huc, enum xe_huc_auth_types type); +bool xe_huc_is_authenticated(struct xe_huc *huc, enum xe_huc_auth_types type); +void xe_huc_sanitize(struct xe_huc *huc); +void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.c b/drivers/gpu/drm/xe/xe_huc_debugfs.c new file mode 100644 index 000000000..80829967b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc_debugfs.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_huc_debugfs.h" + +#include +#include +#include + +#include "xe_gt_types.h" +#include "xe_huc.h" +#include "xe_pm.h" + +static struct xe_gt * +huc_to_gt(struct xe_huc *huc) +{ + return container_of(huc, struct xe_gt, uc.huc); +} + +static struct xe_device * +huc_to_xe(struct xe_huc *huc) +{ + return gt_to_xe(huc_to_gt(huc)); +} + +static struct xe_huc *node_to_huc(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static int huc_info(struct seq_file *m, void *data) +{ + struct xe_huc *huc = node_to_huc(m->private); + struct xe_device *xe = huc_to_xe(huc); + struct drm_printer p = drm_seq_file_printer(m); + + guard(xe_pm_runtime)(xe); + xe_huc_print_info(huc, &p); + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"huc_info", huc_info, 0}, +}; + +void xe_huc_debugfs_register(struct xe_huc *huc, struct dentry *parent) +{ + struct drm_minor *minor = huc_to_xe(huc)->drm.primary; + struct drm_info_list *local; + int i; + +#define DEBUGFS_SIZE (ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list)) + local = drmm_kmalloc(&huc_to_xe(huc)->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) + return; + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = huc; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + parent, minor); +} diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.h b/drivers/gpu/drm/xe/xe_huc_debugfs.h new file mode 100644 index 000000000..ec58f1818 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HUC_DEBUGFS_H_ +#define _XE_HUC_DEBUGFS_H_ + +struct dentry; +struct xe_huc; + +void xe_huc_debugfs_register(struct xe_huc *huc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_huc_types.h b/drivers/gpu/drm/xe/xe_huc_types.h new file mode 100644 index 000000000..cfbaa5e0d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc_types.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HUC_TYPES_H_ +#define _XE_HUC_TYPES_H_ + +#include "xe_uc_fw_types.h" + +struct xe_bo; + +/** + * struct xe_huc - HuC + */ +struct xe_huc { + /** @fw: Generic uC firmware management */ + struct xe_uc_fw fw; + + /** @gsc_pkt: bo to store the packet for auth via GSC */ + struct xe_bo *gsc_pkt; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c new file mode 100644 index 000000000..010499766 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -0,0 +1,1177 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_hw_engine.h" + +#include + +#include +#include +#include +#include + +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_irq_regs.h" +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_execlist.h" +#include "xe_force_wake.h" +#include "xe_gsc.h" +#include "xe_gt.h" +#include "xe_gt_ccs_mode.h" +#include "xe_gt_clock.h" +#include "xe_gt_printk.h" +#include "xe_gt_mcr.h" +#include "xe_gt_topology.h" +#include "xe_guc_capture.h" +#include "xe_hw_engine_group.h" +#include "xe_hw_fence.h" +#include "xe_irq.h" +#include "xe_lrc.h" +#include "xe_mmio.h" +#include "xe_reg_sr.h" +#include "xe_reg_whitelist.h" +#include "xe_rtp.h" +#include "xe_sched_job.h" +#include "xe_sriov.h" +#include "xe_tuning.h" +#include "xe_uc_fw.h" +#include "xe_wa.h" + +#define MAX_MMIO_BASES 3 +struct engine_info { + const char *name; + unsigned int class : 8; + unsigned int instance : 8; + unsigned int irq_offset : 8; + enum xe_force_wake_domains domain; + u32 mmio_base; +}; + +static const struct engine_info engine_infos[] = { + [XE_HW_ENGINE_RCS0] = { + .name = "rcs0", + .class = XE_ENGINE_CLASS_RENDER, + .instance = 0, + .irq_offset = ilog2(INTR_RCS0), + .domain = XE_FW_RENDER, + .mmio_base = RENDER_RING_BASE, + }, + [XE_HW_ENGINE_BCS0] = { + .name = "bcs0", + .class = XE_ENGINE_CLASS_COPY, + .instance = 0, + .irq_offset = ilog2(INTR_BCS(0)), + .domain = XE_FW_GT, + .mmio_base = BLT_RING_BASE, + }, + [XE_HW_ENGINE_BCS1] = { + .name = "bcs1", + .class = XE_ENGINE_CLASS_COPY, + .instance = 1, + .irq_offset = ilog2(INTR_BCS(1)), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS1_RING_BASE, + }, + [XE_HW_ENGINE_BCS2] = { + .name = "bcs2", + .class = XE_ENGINE_CLASS_COPY, + .instance = 2, + .irq_offset = ilog2(INTR_BCS(2)), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS2_RING_BASE, + }, + [XE_HW_ENGINE_BCS3] = { + .name = "bcs3", + .class = XE_ENGINE_CLASS_COPY, + .instance = 3, + .irq_offset = ilog2(INTR_BCS(3)), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS3_RING_BASE, + }, + [XE_HW_ENGINE_BCS4] = { + .name = "bcs4", + .class = XE_ENGINE_CLASS_COPY, + .instance = 4, + .irq_offset = ilog2(INTR_BCS(4)), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS4_RING_BASE, + }, + [XE_HW_ENGINE_BCS5] = { + .name = "bcs5", + .class = XE_ENGINE_CLASS_COPY, + .instance = 5, + .irq_offset = ilog2(INTR_BCS(5)), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS5_RING_BASE, + }, + [XE_HW_ENGINE_BCS6] = { + .name = "bcs6", + .class = XE_ENGINE_CLASS_COPY, + .instance = 6, + .irq_offset = ilog2(INTR_BCS(6)), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS6_RING_BASE, + }, + [XE_HW_ENGINE_BCS7] = { + .name = "bcs7", + .class = XE_ENGINE_CLASS_COPY, + .irq_offset = ilog2(INTR_BCS(7)), + .instance = 7, + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS7_RING_BASE, + }, + [XE_HW_ENGINE_BCS8] = { + .name = "bcs8", + .class = XE_ENGINE_CLASS_COPY, + .instance = 8, + .irq_offset = ilog2(INTR_BCS8), + .domain = XE_FW_GT, + .mmio_base = XEHPC_BCS8_RING_BASE, + }, + + [XE_HW_ENGINE_VCS0] = { + .name = "vcs0", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 0, + .irq_offset = 32 + ilog2(INTR_VCS(0)), + .domain = XE_FW_MEDIA_VDBOX0, + .mmio_base = BSD_RING_BASE, + }, + [XE_HW_ENGINE_VCS1] = { + .name = "vcs1", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 1, + .irq_offset = 32 + ilog2(INTR_VCS(1)), + .domain = XE_FW_MEDIA_VDBOX1, + .mmio_base = BSD2_RING_BASE, + }, + [XE_HW_ENGINE_VCS2] = { + .name = "vcs2", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 2, + .irq_offset = 32 + ilog2(INTR_VCS(2)), + .domain = XE_FW_MEDIA_VDBOX2, + .mmio_base = BSD3_RING_BASE, + }, + [XE_HW_ENGINE_VCS3] = { + .name = "vcs3", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 3, + .irq_offset = 32 + ilog2(INTR_VCS(3)), + .domain = XE_FW_MEDIA_VDBOX3, + .mmio_base = BSD4_RING_BASE, + }, + [XE_HW_ENGINE_VCS4] = { + .name = "vcs4", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 4, + .irq_offset = 32 + ilog2(INTR_VCS(4)), + .domain = XE_FW_MEDIA_VDBOX4, + .mmio_base = XEHP_BSD5_RING_BASE, + }, + [XE_HW_ENGINE_VCS5] = { + .name = "vcs5", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 5, + .irq_offset = 32 + ilog2(INTR_VCS(5)), + .domain = XE_FW_MEDIA_VDBOX5, + .mmio_base = XEHP_BSD6_RING_BASE, + }, + [XE_HW_ENGINE_VCS6] = { + .name = "vcs6", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 6, + .irq_offset = 32 + ilog2(INTR_VCS(6)), + .domain = XE_FW_MEDIA_VDBOX6, + .mmio_base = XEHP_BSD7_RING_BASE, + }, + [XE_HW_ENGINE_VCS7] = { + .name = "vcs7", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 7, + .irq_offset = 32 + ilog2(INTR_VCS(7)), + .domain = XE_FW_MEDIA_VDBOX7, + .mmio_base = XEHP_BSD8_RING_BASE, + }, + [XE_HW_ENGINE_VECS0] = { + .name = "vecs0", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 0, + .irq_offset = 32 + ilog2(INTR_VECS(0)), + .domain = XE_FW_MEDIA_VEBOX0, + .mmio_base = VEBOX_RING_BASE, + }, + [XE_HW_ENGINE_VECS1] = { + .name = "vecs1", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 1, + .irq_offset = 32 + ilog2(INTR_VECS(1)), + .domain = XE_FW_MEDIA_VEBOX1, + .mmio_base = VEBOX2_RING_BASE, + }, + [XE_HW_ENGINE_VECS2] = { + .name = "vecs2", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 2, + .irq_offset = 32 + ilog2(INTR_VECS(2)), + .domain = XE_FW_MEDIA_VEBOX2, + .mmio_base = XEHP_VEBOX3_RING_BASE, + }, + [XE_HW_ENGINE_VECS3] = { + .name = "vecs3", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 3, + .irq_offset = 32 + ilog2(INTR_VECS(3)), + .domain = XE_FW_MEDIA_VEBOX3, + .mmio_base = XEHP_VEBOX4_RING_BASE, + }, + [XE_HW_ENGINE_CCS0] = { + .name = "ccs0", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 0, + .irq_offset = ilog2(INTR_CCS(0)), + .domain = XE_FW_RENDER, + .mmio_base = COMPUTE0_RING_BASE, + }, + [XE_HW_ENGINE_CCS1] = { + .name = "ccs1", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 1, + .irq_offset = ilog2(INTR_CCS(1)), + .domain = XE_FW_RENDER, + .mmio_base = COMPUTE1_RING_BASE, + }, + [XE_HW_ENGINE_CCS2] = { + .name = "ccs2", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 2, + .irq_offset = ilog2(INTR_CCS(2)), + .domain = XE_FW_RENDER, + .mmio_base = COMPUTE2_RING_BASE, + }, + [XE_HW_ENGINE_CCS3] = { + .name = "ccs3", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 3, + .irq_offset = ilog2(INTR_CCS(3)), + .domain = XE_FW_RENDER, + .mmio_base = COMPUTE3_RING_BASE, + }, + [XE_HW_ENGINE_GSCCS0] = { + .name = "gsccs0", + .class = XE_ENGINE_CLASS_OTHER, + .instance = OTHER_GSC_INSTANCE, + .domain = XE_FW_GSC, + .mmio_base = GSCCS_RING_BASE, + }, +}; + +static void hw_engine_fini(void *arg) +{ + struct xe_hw_engine *hwe = arg; + + if (hwe->exl_port) + xe_execlist_port_destroy(hwe->exl_port); + + hwe->gt = NULL; +} + +/** + * xe_hw_engine_mmio_read32() - Read engine register + * @hwe: engine + * @reg: register to read from + * + * This function will read from an engine specific register. + * Forcewake must be held by the caller. + * + * Return: value of the 32-bit register. + */ +u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg) +{ + xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base)); + xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); + + reg.addr += hwe->mmio_base; + + return xe_mmio_read32(&hwe->gt->mmio, reg); +} + +void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe) +{ + xe_mmio_write32(&hwe->gt->mmio, RING_HWS_PGA(hwe->mmio_base), + xe_bo_ggtt_addr(hwe->hwsp)); +} + +static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + /* + * Xe3p no longer supports load balance mode, so "fixed cslice" mode + * is automatic and no RCU_MODE programming is required. + */ + if (GRAPHICS_VER(gt_to_xe(gt)) >= 35) + return false; + + return xe_gt_ccs_mode_enabled(gt) && + xe_rtp_match_first_render_or_compute(xe, gt, hwe); +} + +static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + if (GRAPHICS_VER(xe) < 20) + return false; + + if (hwe->class != XE_ENGINE_CLASS_COMPUTE && + hwe->class != XE_ENGINE_CLASS_RENDER) + return false; + + return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE; +} + +static u32 blit_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + return REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, gt->mocs.uc_index) | + REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, gt->mocs.uc_index); +} + +static const struct xe_rtp_table_sr lrc_setup = XE_RTP_TABLE_SR( + /* + * Some blitter commands do not have a field for MOCS, those + * commands will use MOCS index pointed by BLIT_CCTL. + * BLIT_CCTL registers are needed to be programmed to un-cached. + */ + { XE_RTP_NAME("BLIT_CCTL_default_MOCS"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274), + ENGINE_CLASS(COPY)), + XE_RTP_ACTIONS(FIELD_SET_FUNC(BLIT_CCTL(0), + BLIT_CCTL_DST_MOCS_MASK | + BLIT_CCTL_SRC_MOCS_MASK, + blit_cctl_val, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + /* Disable WMTP if HW doesn't support it */ + { XE_RTP_NAME("DISABLE_WMTP_ON_UNSUPPORTED_HW"), + XE_RTP_RULES(FUNC(xe_rtp_cfeg_wmtp_disabled)), + XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(0), + PREEMPT_GPGPU_LEVEL_MASK, + PREEMPT_GPGPU_THREAD_GROUP_LEVEL)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE) + }, +); + +static void +hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + + xe_rtp_process_to_sr(&ctx, &lrc_setup, &hwe->reg_lrc, true); +} + +void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + + xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); + xe_wa_process_lrc(hwe); + hw_engine_setup_default_lrc_state(hwe); + xe_tuning_process_lrc(hwe); +} + +/* + * RING_CMD_CCTL specifies the default MOCS entry that will be + * used by the command streamer when executing commands that + * don't have a way to explicitly specify a MOCS setting. + * The default should usually reference whichever MOCS entry + * corresponds to uncached behavior, although use of a WB cached + * entry is recommended by the spec in certain circumstances on + * specific platforms. + * Bspec: 72161 + */ +static u32 ring_cmd_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + struct xe_device *xe = gt_to_xe(gt); + u8 mocs_read_idx = gt->mocs.uc_index; + + if (hwe->class == XE_ENGINE_CLASS_COMPUTE && IS_DGFX(xe) && + (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)) + mocs_read_idx = gt->mocs.wb_index; + + return REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, gt->mocs.uc_index) | + REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx); +} + +static const struct xe_rtp_table_sr engine_sr = XE_RTP_TABLE_SR( + { XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(FIELD_SET_FUNC(RING_CMD_CCTL(0), + CMD_CCTL_WRITE_OVERRIDE_MASK | + CMD_CCTL_READ_OVERRIDE_MASK, + ring_cmd_cctl_val, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("Disable HW status page updates for interrupts"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("Disable engine 'legacy' mode"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + /* + * To allow the GSC engine to go idle on MTL we need to enable + * idle messaging and set the hysteresis value (we use 0xA=5us + * as recommended in spec). On platforms after MTL this is + * enabled by default. + */ + { XE_RTP_NAME("MTL GSCCS IDLE MSG enable"), + XE_RTP_RULES(MEDIA_VERSION(1300), ENGINE_CLASS(OTHER)), + XE_RTP_ACTIONS(CLR(RING_PSMI_CTL(0), + IDLE_MSG_DISABLE, + XE_RTP_ACTION_FLAG(ENGINE_BASE)), + FIELD_SET(RING_PWRCTX_MAXCNT(0), + IDLE_WAIT_TIME, + 0xA, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + /* Enable Priority Mem Read */ + { XE_RTP_NAME("Priority_Mem_Read"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("Enable CCS Engine(s)"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE)) + }, + /* Use Fixed slice CCS mode */ + { XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"), + XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)), + XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE, + RCU_MODE_FIXED_SLICE_CCS_MODE)) + }, + { XE_RTP_NAME("Enable MSI-X interrupt support"), + XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)), + XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, +); + +static void +hw_engine_setup_default_state(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + + xe_rtp_process_to_sr(&ctx, &engine_sr, &hwe->reg_sr, false); +} + +static const struct engine_info *find_engine_info(enum xe_engine_class class, int instance) +{ + const struct engine_info *info; + enum xe_hw_engine_id id; + + for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { + info = &engine_infos[id]; + if (info->class == class && info->instance == instance) + return info; + } + + return NULL; +} + +static u16 get_msix_irq_offset(struct xe_gt *gt, enum xe_engine_class class) +{ + /* For MSI-X, hw engines report to offset of engine instance zero */ + const struct engine_info *info = find_engine_info(class, 0); + + xe_gt_assert(gt, info); + + return info ? info->irq_offset : 0; +} + +static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, + enum xe_hw_engine_id id) +{ + const struct engine_info *info; + + if (WARN_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name)) + return; + + if (!(gt->info.engine_mask & BIT(id))) + return; + + info = &engine_infos[id]; + + xe_gt_assert(gt, !hwe->gt); + + hwe->gt = gt; + hwe->class = info->class; + hwe->instance = info->instance; + hwe->mmio_base = info->mmio_base; + if (xe_device_has_msix(gt_to_xe(gt))) { + hwe->irq_offset = get_msix_irq_offset(gt, info->class); + hwe->irq_page = info->instance; + + } else { + hwe->irq_offset = info->irq_offset; + hwe->irq_page = 0; + } + hwe->domain = info->domain; + hwe->name = info->name; + hwe->fence_irq = >->fence_irq[info->class]; + hwe->engine_id = id; + + hwe->eclass = >->eclass[hwe->class]; + if (!hwe->eclass->sched_props.job_timeout_ms) { + hwe->eclass->sched_props.job_timeout_ms = 5 * 1000; + hwe->eclass->sched_props.job_timeout_min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; + hwe->eclass->sched_props.job_timeout_max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; + hwe->eclass->sched_props.timeslice_us = 1 * 1000; + hwe->eclass->sched_props.timeslice_min = XE_HW_ENGINE_TIMESLICE_MIN; + hwe->eclass->sched_props.timeslice_max = XE_HW_ENGINE_TIMESLICE_MAX; + hwe->eclass->sched_props.preempt_timeout_us = XE_HW_ENGINE_PREEMPT_TIMEOUT; + hwe->eclass->sched_props.preempt_timeout_min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; + hwe->eclass->sched_props.preempt_timeout_max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; + + /* + * The GSC engine can accept submissions while the GSC shim is + * being reset, during which time the submission is stalled. In + * the worst case, the shim reset can take up to the maximum GSC + * command execution time (250ms), so the request start can be + * delayed by that much; the request itself can take that long + * without being preemptible, which means worst case it can + * theoretically take up to 500ms for a preemption to go through + * on the GSC engine. Adding to that an extra 100ms as a safety + * margin, we get a minimum recommended timeout of 600ms. + * The preempt_timeout value can't be tuned for OTHER_CLASS + * because the class is reserved for kernel usage, so we just + * need to make sure that the starting value is above that + * threshold; since our default value (640ms) is greater than + * 600ms, the only way we can go below is via a kconfig setting. + * If that happens, log it in dmesg and update the value. + */ + if (hwe->class == XE_ENGINE_CLASS_OTHER) { + const u32 min_preempt_timeout = 600 * 1000; + if (hwe->eclass->sched_props.preempt_timeout_us < min_preempt_timeout) { + hwe->eclass->sched_props.preempt_timeout_us = min_preempt_timeout; + xe_gt_notice(gt, "Increasing preempt_timeout for GSC to 600ms\n"); + } + } + + /* Record default props */ + hwe->eclass->defaults = hwe->eclass->sched_props; + } + + xe_reg_sr_init(&hwe->reg_sr, hwe->name, gt_to_xe(gt)); + xe_tuning_process_engine(hwe); + xe_wa_process_engine(hwe); + hw_engine_setup_default_state(hwe); + + xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_sr_init(&hwe->oa_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_sr_init(&hwe->oa_sr, hwe->name, gt_to_xe(gt)); + xe_reg_whitelist_process_engine(hwe); +} + +static void adjust_idledly(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + u32 idledly, maxcnt; + u32 idledly_units_ps = 8 * gt->info.timestamp_base; + u32 maxcnt_units_ns = 640; + bool inhibit_switch = 0; + + if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) { + idledly = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); + maxcnt = xe_mmio_read32(>->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base)); + + inhibit_switch = idledly & INHIBIT_SWITCH_UNTIL_PREEMPTED; + idledly = REG_FIELD_GET(IDLE_DELAY, idledly); + idledly = DIV_ROUND_CLOSEST(idledly * idledly_units_ps, 1000); + maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt); + maxcnt *= maxcnt_units_ns; + + if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) { + idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000), + idledly_units_ps); + xe_mmio_write32(>->mmio, RING_IDLEDLY(hwe->mmio_base), idledly); + } + } +} + +static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, + enum xe_hw_engine_id id) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_tile *tile = gt_to_tile(gt); + int err; + + xe_gt_assert(gt, id < ARRAY_SIZE(engine_infos) && engine_infos[id].name); + xe_gt_assert(gt, gt->info.engine_mask & BIT(id)); + + xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); + + hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE); + if (IS_ERR(hwe->hwsp)) { + err = PTR_ERR(hwe->hwsp); + goto err_name; + } + + if (!xe_device_uc_enabled(xe)) { + hwe->exl_port = xe_execlist_port_create(xe, hwe); + if (IS_ERR(hwe->exl_port)) { + err = PTR_ERR(hwe->exl_port); + goto err_name; + } + } else { + /* GSCCS has a special interrupt for reset */ + if (hwe->class == XE_ENGINE_CLASS_OTHER) + hwe->irq_handler = xe_gsc_hwe_irq_handler; + + if (!IS_SRIOV_VF(xe)) + xe_hw_engine_enable_ring(hwe); + } + + /* Ensure IDLEDLY is lower than MAXCNT */ + adjust_idledly(hwe); + + return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe); + +err_name: + hwe->name = NULL; + + return err; +} + +static int hw_engine_setup_logical_and_paging_mapping(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int num_copy_engines = 0, num_paging_engines = 0; + unsigned int reserved_logical_bcs_start; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int class; + + for_each_hw_engine(hwe, gt, id) + if (hwe->class == XE_ENGINE_CLASS_COPY) + num_copy_engines++; + + if (num_copy_engines && xe->info.has_usm) + num_paging_engines = 1; + + if (IS_SRIOV_VF(xe)) { + u32 vf_num_paging_engines; + + /* + * PF could in theory reserve multiple paging engines, which + * internally the submission/scheduling backend can load balance + * from. Not something we currently expect, but we are at the + * mercy of the PF, so we just need try our best to mirror the + * paging configuration. + */ + vf_num_paging_engines = xe_gt_sriov_vf_paging_engines(gt); + if (vf_num_paging_engines) { + /* This should only be non-zero on NVL-S+ */ + if (xe_gt_WARN_ON(gt, xe->info.platform < XE_NOVALAKE_S)) + return -EINVAL; + + num_paging_engines = vf_num_paging_engines; + } + } + + if (xe_gt_WARN_ON(gt, num_paging_engines > num_copy_engines)) + return -EINVAL; + + /* + * On PF, we just reserve the highest BCS instance for USM. + * + * Note: This is now a requirement going forward. The PF must ALWAYS + * reserve BCS instances in top-down order, that way the VF has a chance + * of discovering the physical BCS instance mappings for paging engines, + * in conjunction with vf_num_paging_engines. In some places we might + * only have the physical instance, and from hw pov there is no such + * thing as a paging engine. For example, the page fault descriptor, + * which comes directly from the hw, will use the physical engine + * instance. + */ + reserved_logical_bcs_start = num_copy_engines - num_paging_engines; + + /* FIXME: Doing a simple logical mapping that works for most hardware */ + for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { + int logical_instance = 0; + + for_each_hw_engine(hwe, gt, id) { + if (hwe->class == class) { + hwe->logical_instance = logical_instance++; + + if (class == XE_ENGINE_CLASS_COPY && + hwe->logical_instance >= + reserved_logical_bcs_start) { + if (!gt->usm.paging_hwe0) + gt->usm.paging_hwe0 = hwe; + gt->usm.paging_logical_mask |= + BIT(hwe->logical_instance); + } + } + } + } + + return 0; +} + +static void read_media_fuses(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 media_fuse; + u16 vdbox_mask; + u16 vebox_mask; + int i, j; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + media_fuse = xe_mmio_read32(>->mmio, GT_VEBOX_VDBOX_DISABLE); + + /* + * Pre-Xe_HP platforms had register bits representing absent engines, + * whereas Xe_HP and beyond have bits representing present engines. + * Invert the polarity on old platforms so that we can use common + * handling below. + */ + if (GRAPHICS_VERx100(xe) < 1250) + media_fuse = ~media_fuse; + + vdbox_mask = REG_FIELD_GET(GT_VDBOX_DISABLE_MASK, media_fuse); + vebox_mask = REG_FIELD_GET(GT_VEBOX_DISABLE_MASK, media_fuse); + + for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!(BIT(j) & vdbox_mask)) { + gt->info.engine_mask &= ~BIT(i); + xe_gt_info(gt, "vcs%u fused off\n", j); + } + } + + for (i = XE_HW_ENGINE_VECS0, j = 0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!(BIT(j) & vebox_mask)) { + gt->info.engine_mask &= ~BIT(i); + xe_gt_info(gt, "vecs%u fused off\n", j); + } + } +} + +static u32 infer_svccopy_from_meml3(struct xe_gt *gt) +{ + u32 meml3 = REG_FIELD_GET(MEML3_EN_MASK, + xe_mmio_read32(>->mmio, MIRROR_FUSE3)); + u32 svccopy_mask = 0; + + /* + * Each of the four meml3 bits determines the fusing of two service + * copy engines. + */ + for (int i = 0; i < 4; i++) + svccopy_mask |= (meml3 & BIT(i)) ? 0b11 << 2 * i : 0; + + return svccopy_mask; +} + +static u32 read_svccopy_fuses(struct xe_gt *gt) +{ + return REG_FIELD_GET(FUSE_SERVICE_COPY_ENABLE_MASK, + xe_mmio_read32(>->mmio, SERVICE_COPY_ENABLE)); +} + +static void read_copy_fuses(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 bcs_mask; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + if (GRAPHICS_VER(xe) >= 35) + bcs_mask = read_svccopy_fuses(gt); + else if (GRAPHICS_VERx100(xe) == 1260) + bcs_mask = infer_svccopy_from_meml3(gt); + else + return; + + /* Only BCS1-BCS8 may be fused off */ + bcs_mask <<= XE_HW_ENGINE_BCS1; + for (int i = XE_HW_ENGINE_BCS1; i <= XE_HW_ENGINE_BCS8; ++i) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!(bcs_mask & BIT(i))) { + gt->info.engine_mask &= ~BIT(i); + xe_gt_info(gt, "bcs%u fused off\n", + i - XE_HW_ENGINE_BCS0); + } + } +} + +static void read_compute_fuses_from_dss(struct xe_gt *gt) +{ + /* + * CCS fusing based on DSS masks only applies to platforms that can + * have more than one CCS. + */ + if (hweight64(gt->info.engine_mask & + GENMASK_ULL(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0)) <= 1) + return; + + /* + * CCS availability on Xe_HP is inferred from the presence of DSS in + * each quadrant. + */ + for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!xe_gt_topology_has_dss_in_quadrant(gt, j)) { + gt->info.engine_mask &= ~BIT(i); + xe_gt_info(gt, "ccs%u fused off\n", j); + } + } +} + +static void read_compute_fuses_from_reg(struct xe_gt *gt) +{ + u32 ccs_mask; + + ccs_mask = xe_mmio_read32(>->mmio, XEHP_FUSE4); + ccs_mask = REG_FIELD_GET(CCS_EN_MASK, ccs_mask); + + for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if ((ccs_mask & BIT(j)) == 0) { + gt->info.engine_mask &= ~BIT(i); + xe_gt_info(gt, "ccs%u fused off\n", j); + } + } +} + +static void read_compute_fuses(struct xe_gt *gt) +{ + if (GRAPHICS_VER(gt_to_xe(gt)) >= 20) + read_compute_fuses_from_reg(gt); + else + read_compute_fuses_from_dss(gt); +} + +static void check_gsc_availability(struct xe_gt *gt) +{ + if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0))) + return; + + /* + * The GSCCS is only used to communicate with the GSC FW, so if we don't + * have the FW there is nothing we need the engine for and can therefore + * skip its initialization. + */ + if (!xe_uc_fw_is_available(>->uc.gsc.fw)) { + gt->info.engine_mask &= ~BIT(XE_HW_ENGINE_GSCCS0); + + /* interrupts where previously enabled, so turn them off */ + xe_mmio_write32(>->mmio, GUNIT_GSC_INTR_ENABLE, 0); + xe_mmio_write32(>->mmio, GUNIT_GSC_INTR_MASK, ~0); + + xe_gt_dbg(gt, "GSC FW not used, disabling gsccs\n"); + } +} + +static void check_sw_disable(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u64 sw_allowed = xe_configfs_get_engines_allowed(to_pci_dev(xe->drm.dev)); + enum xe_hw_engine_id id; + + for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { + if (!(gt->info.engine_mask & BIT(id))) + continue; + + if (!(sw_allowed & BIT(id))) { + gt->info.engine_mask &= ~BIT(id); + xe_gt_info(gt, "%s disabled via configfs\n", + engine_infos[id].name); + } + } +} + +int xe_hw_engines_init_early(struct xe_gt *gt) +{ + int i; + + read_media_fuses(gt); + read_copy_fuses(gt); + read_compute_fuses(gt); + check_gsc_availability(gt); + check_sw_disable(gt); + + BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT < XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN); + BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT > XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX); + + for (i = 0; i < ARRAY_SIZE(gt->hw_engines); i++) + hw_engine_init_early(gt, >->hw_engines[i], i); + + return 0; +} + +int xe_hw_engines_init(struct xe_gt *gt) +{ + int err; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) { + err = hw_engine_init(gt, hwe, id); + if (err) + return err; + } + + err = hw_engine_setup_logical_and_paging_mapping(gt); + if (err) + return err; + + err = xe_hw_engine_setup_groups(gt); + if (err) + return err; + + return 0; +} + +void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec) +{ + wake_up_all(>_to_xe(hwe->gt)->ufence_wq); + + if (hwe->irq_handler) + hwe->irq_handler(hwe, intr_vec); + + if (intr_vec & GT_MI_USER_INTERRUPT) + xe_hw_fence_irq_run(hwe->fence_irq); +} + +/** + * xe_hw_engine_snapshot_capture - Take a quick snapshot of the HW Engine. + * @hwe: Xe HW Engine. + * @q: The exec queue object. + * + * This can be printed out in a later stage like during dev_coredump + * analysis. + * + * Returns: a Xe HW Engine snapshot object that must be freed by the + * caller, using `xe_hw_engine_snapshot_free`. + */ +struct xe_hw_engine_snapshot * +xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q) +{ + struct xe_hw_engine_snapshot *snapshot; + struct __guc_capture_parsed_output *node; + + if (!xe_hw_engine_is_valid(hwe)) + return NULL; + + snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC); + + if (!snapshot) + return NULL; + + snapshot->name = kstrdup(hwe->name, GFP_ATOMIC); + snapshot->hwe = hwe; + snapshot->logical_instance = hwe->logical_instance; + snapshot->forcewake.domain = hwe->domain; + snapshot->forcewake.ref = xe_force_wake_ref(gt_to_fw(hwe->gt), + hwe->domain); + snapshot->mmio_base = hwe->mmio_base; + snapshot->kernel_reserved = xe_hw_engine_is_reserved(hwe); + + /* no more VF accessible data below this point */ + if (IS_SRIOV_VF(gt_to_xe(hwe->gt))) + return snapshot; + + if (q) { + /* If got guc capture, set source to GuC */ + node = xe_guc_capture_get_matching_and_lock(q); + if (node) { + struct xe_device *xe = gt_to_xe(hwe->gt); + struct xe_devcoredump *coredump = &xe->devcoredump; + + coredump->snapshot.matched_node = node; + xe_gt_dbg(hwe->gt, "Found and locked GuC-err-capture node"); + return snapshot; + } + } + + /* otherwise, do manual capture */ + xe_engine_manual_capture(hwe, snapshot); + xe_gt_dbg(hwe->gt, "Proceeding with manual engine snapshot"); + + return snapshot; +} + +/** + * xe_hw_engine_snapshot_free - Free all allocated objects for a given snapshot. + * @snapshot: Xe HW Engine snapshot object. + * + * This function free all the memory that needed to be allocated at capture + * time. + */ +void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot) +{ + struct xe_gt *gt; + if (!snapshot) + return; + + gt = snapshot->hwe->gt; + /* + * xe_guc_capture_put_matched_nodes is called here and from + * xe_devcoredump_snapshot_free, to cover the 2 calling paths + * of hw_engines - debugfs and devcoredump free. + */ + xe_guc_capture_put_matched_nodes(>->uc.guc); + + kfree(snapshot->name); + kfree(snapshot); +} + +/** + * xe_hw_engine_print - Xe HW Engine Print. + * @hwe: Hardware Engine. + * @p: drm_printer. + * + * This function quickly capture a snapshot and immediately print it out. + */ +void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p) +{ + struct xe_hw_engine_snapshot *snapshot; + + snapshot = xe_hw_engine_snapshot_capture(hwe, NULL); + xe_engine_snapshot_print(snapshot, p); + xe_hw_engine_snapshot_free(snapshot); +} + +u32 xe_hw_engine_mask_per_class(struct xe_gt *gt, + enum xe_engine_class engine_class) +{ + u32 mask = 0; + enum xe_hw_engine_id id; + + for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { + if (engine_infos[id].class == engine_class && + gt->info.engine_mask & BIT(id)) + mask |= BIT(engine_infos[id].instance); + } + return mask; +} + +bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + + if (xe_device_is_admin_only(xe)) + return true; + + if (hwe->class == XE_ENGINE_CLASS_OTHER) + return true; + + /* Check for engines disabled by ccs_mode setting */ + if (xe_gt_ccs_mode_enabled(gt) && + hwe->class == XE_ENGINE_CLASS_COMPUTE && + hwe->logical_instance >= gt->ccs_mode) + return true; + + return xe_gt_is_usm_hwe(gt, hwe); +} + +const char *xe_hw_engine_class_to_str(enum xe_engine_class class) +{ + switch (class) { + case XE_ENGINE_CLASS_RENDER: + return "rcs"; + case XE_ENGINE_CLASS_VIDEO_DECODE: + return "vcs"; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + return "vecs"; + case XE_ENGINE_CLASS_COPY: + return "bcs"; + case XE_ENGINE_CLASS_OTHER: + return "other"; + case XE_ENGINE_CLASS_COMPUTE: + return "ccs"; + case XE_ENGINE_CLASS_MAX: + break; + } + + return NULL; +} + +u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe) +{ + return xe_mmio_read64_2x32(&hwe->gt->mmio, RING_TIMESTAMP(hwe->mmio_base)); +} + +enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe) +{ + return engine_infos[hwe->engine_id].domain; +} + +static const enum xe_engine_class user_to_xe_engine_class[] = { + [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, + [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, + [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, + [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, + [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, +}; + +/** + * xe_hw_engine_lookup() - Lookup hardware engine for class:instance + * @xe: xe device + * @eci: engine class and instance + * + * This function will find a hardware engine for given engine + * class and instance. + * + * Return: If found xe_hw_engine pointer, NULL otherwise. + */ +struct xe_hw_engine * +xe_hw_engine_lookup(struct xe_device *xe, + struct drm_xe_engine_class_instance eci) +{ + struct xe_gt *gt = xe_device_get_gt(xe, eci.gt_id); + unsigned int idx; + + if (eci.engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) + return NULL; + + if (!gt) + return NULL; + + idx = array_index_nospec(eci.engine_class, + ARRAY_SIZE(user_to_xe_engine_class)); + + return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), + user_to_xe_engine_class[idx], + eci.engine_instance, true); +} diff --git a/drivers/gpu/drm/xe/xe_hw_engine.h b/drivers/gpu/drm/xe/xe_hw_engine.h new file mode 100644 index 000000000..c3ee37f8c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_H_ +#define _XE_HW_ENGINE_H_ + +#include "xe_hw_engine_types.h" + +struct drm_printer; +struct drm_xe_engine_class_instance; +struct xe_device; +struct xe_exec_queue; + +#ifdef CONFIG_DRM_XE_JOB_TIMEOUT_MIN +#define XE_HW_ENGINE_JOB_TIMEOUT_MIN CONFIG_DRM_XE_JOB_TIMEOUT_MIN +#else +#define XE_HW_ENGINE_JOB_TIMEOUT_MIN 1 +#endif +#ifdef CONFIG_DRM_XE_JOB_TIMEOUT_MAX +#define XE_HW_ENGINE_JOB_TIMEOUT_MAX CONFIG_DRM_XE_JOB_TIMEOUT_MAX +#else +#define XE_HW_ENGINE_JOB_TIMEOUT_MAX (10 * 1000) +#endif +#ifdef CONFIG_DRM_XE_TIMESLICE_MIN +#define XE_HW_ENGINE_TIMESLICE_MIN CONFIG_DRM_XE_TIMESLICE_MIN +#else +#define XE_HW_ENGINE_TIMESLICE_MIN 1 +#endif +#ifdef CONFIG_DRM_XE_TIMESLICE_MAX +#define XE_HW_ENGINE_TIMESLICE_MAX CONFIG_DRM_XE_TIMESLICE_MAX +#else +#define XE_HW_ENGINE_TIMESLICE_MAX (10 * 1000 * 1000) +#endif +#ifdef CONFIG_DRM_XE_PREEMPT_TIMEOUT +#define XE_HW_ENGINE_PREEMPT_TIMEOUT CONFIG_DRM_XE_PREEMPT_TIMEOUT +#else +#define XE_HW_ENGINE_PREEMPT_TIMEOUT (640 * 1000) +#endif +#ifdef CONFIG_DRM_XE_PREEMPT_TIMEOUT_MIN +#define XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN CONFIG_DRM_XE_PREEMPT_TIMEOUT_MIN +#else +#define XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN 1 +#endif +#ifdef CONFIG_DRM_XE_PREEMPT_TIMEOUT_MAX +#define XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX CONFIG_DRM_XE_PREEMPT_TIMEOUT_MAX +#else +#define XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX (10 * 1000 * 1000) +#endif + +int xe_hw_engines_init_early(struct xe_gt *gt); +int xe_hw_engines_init(struct xe_gt *gt); +void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec); +void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe); +u32 xe_hw_engine_mask_per_class(struct xe_gt *gt, + enum xe_engine_class engine_class); +struct xe_hw_engine_snapshot * +xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q); +void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot); +void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p); +void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe); + +bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe); + +struct xe_hw_engine * +xe_hw_engine_lookup(struct xe_device *xe, + struct drm_xe_engine_class_instance eci); + +static inline bool xe_hw_engine_is_valid(struct xe_hw_engine *hwe) +{ + return hwe->name; +} + +const char *xe_hw_engine_class_to_str(enum xe_engine_class class); +u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe); +enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe); + +u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg); + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c new file mode 100644 index 000000000..d42e263e0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_gt.h" +#include "xe_hw_engine_class_sysfs.h" +#include "xe_pm.h" + +#define MAX_ENGINE_CLASS_NAME_LEN 16 +static int xe_add_hw_engine_class_defaults(struct xe_device *xe, + struct kobject *parent); + +/** + * xe_hw_engine_timeout_in_range - Helper to check if timeout is in range + * @timeout: timeout to validate + * @min: min value of valid range + * @max: max value of valid range + * + * This helper helps to validate if timeout is in min-max range of HW engine + * scheduler. + * + * Returns: Returns false value for failure and true for success. + */ +bool xe_hw_engine_timeout_in_range(u64 timeout, u64 min, u64 max) +{ + return timeout >= min && timeout <= max; +} + +static void xe_hw_engine_sysfs_kobj_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static ssize_t xe_hw_engine_class_sysfs_attr_show(struct kobject *kobj, + struct attribute *attr, + char *buf) +{ + struct xe_device *xe = kobj_to_xe(kobj); + struct kobj_attribute *kattr; + + kattr = container_of(attr, struct kobj_attribute, attr); + if (kattr->show) { + guard(xe_pm_runtime)(xe); + return kattr->show(kobj, kattr, buf); + } + + return -EIO; +} + +static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, + size_t count) +{ + struct xe_device *xe = kobj_to_xe(kobj); + struct kobj_attribute *kattr; + + kattr = container_of(attr, struct kobj_attribute, attr); + if (kattr->store) { + guard(xe_pm_runtime)(xe); + return kattr->store(kobj, kattr, buf, count); + } + + return -EIO; +} + +static const struct sysfs_ops xe_hw_engine_class_sysfs_ops = { + .show = xe_hw_engine_class_sysfs_attr_show, + .store = xe_hw_engine_class_sysfs_attr_store, +}; + +static const struct kobj_type kobj_xe_hw_engine_type = { + .release = xe_hw_engine_sysfs_kobj_release, + .sysfs_ops = &xe_hw_engine_class_sysfs_ops, +}; + +static const struct kobj_type kobj_xe_hw_engine_type_def = { + .release = xe_hw_engine_sysfs_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, +}; + +static ssize_t job_timeout_max_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 timeout; + int err; + + err = kstrtou32(buf, 0, &timeout); + if (err) + return err; + + if (timeout < eclass->sched_props.job_timeout_min) + return -EINVAL; + + if (!xe_hw_engine_timeout_in_range(timeout, + XE_HW_ENGINE_JOB_TIMEOUT_MIN, + XE_HW_ENGINE_JOB_TIMEOUT_MAX)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.job_timeout_max, timeout); + + return count; +} + +static ssize_t job_timeout_max_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.job_timeout_max); +} + +static const struct kobj_attribute job_timeout_max_attr = +__ATTR(job_timeout_max, 0644, job_timeout_max_show, job_timeout_max_store); + +static ssize_t job_timeout_min_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 timeout; + int err; + + err = kstrtou32(buf, 0, &timeout); + if (err) + return err; + + if (timeout > eclass->sched_props.job_timeout_max) + return -EINVAL; + + if (!xe_hw_engine_timeout_in_range(timeout, + XE_HW_ENGINE_JOB_TIMEOUT_MIN, + XE_HW_ENGINE_JOB_TIMEOUT_MAX)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.job_timeout_min, timeout); + + return count; +} + +static ssize_t job_timeout_min_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.job_timeout_min); +} + +static const struct kobj_attribute job_timeout_min_attr = +__ATTR(job_timeout_min, 0644, job_timeout_min_show, job_timeout_min_store); + +static ssize_t job_timeout_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 min = eclass->sched_props.job_timeout_min; + u32 max = eclass->sched_props.job_timeout_max; + u32 timeout; + int err; + + err = kstrtou32(buf, 0, &timeout); + if (err) + return err; + + if (!xe_hw_engine_timeout_in_range(timeout, min, max)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.job_timeout_ms, timeout); + + return count; +} + +static ssize_t job_timeout_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.job_timeout_ms); +} + +static const struct kobj_attribute job_timeout_attr = +__ATTR(job_timeout_ms, 0644, job_timeout_show, job_timeout_store); + +static ssize_t job_timeout_default(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.job_timeout_ms); +} + +static const struct kobj_attribute job_timeout_def = +__ATTR(job_timeout_ms, 0444, job_timeout_default, NULL); + +static ssize_t job_timeout_min_default(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.job_timeout_min); +} + +static const struct kobj_attribute job_timeout_min_def = +__ATTR(job_timeout_min, 0444, job_timeout_min_default, NULL); + +static ssize_t job_timeout_max_default(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.job_timeout_max); +} + +static const struct kobj_attribute job_timeout_max_def = +__ATTR(job_timeout_max, 0444, job_timeout_max_default, NULL); + +static ssize_t timeslice_duration_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 min = eclass->sched_props.timeslice_min; + u32 max = eclass->sched_props.timeslice_max; + u32 duration; + int err; + + err = kstrtou32(buf, 0, &duration); + if (err) + return err; + + if (!xe_hw_engine_timeout_in_range(duration, min, max)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.timeslice_us, duration); + + return count; +} + +static ssize_t timeslice_duration_max_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 duration; + int err; + + err = kstrtou32(buf, 0, &duration); + if (err) + return err; + + if (duration < eclass->sched_props.timeslice_min) + return -EINVAL; + + if (!xe_hw_engine_timeout_in_range(duration, + XE_HW_ENGINE_TIMESLICE_MIN, + XE_HW_ENGINE_TIMESLICE_MAX)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.timeslice_max, duration); + + return count; +} + +static ssize_t timeslice_duration_max_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.timeslice_max); +} + +static const struct kobj_attribute timeslice_duration_max_attr = + __ATTR(timeslice_duration_max, 0644, timeslice_duration_max_show, + timeslice_duration_max_store); + +static ssize_t timeslice_duration_min_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 duration; + int err; + + err = kstrtou32(buf, 0, &duration); + if (err) + return err; + + if (duration > eclass->sched_props.timeslice_max) + return -EINVAL; + + if (!xe_hw_engine_timeout_in_range(duration, + XE_HW_ENGINE_TIMESLICE_MIN, + XE_HW_ENGINE_TIMESLICE_MAX)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.timeslice_min, duration); + + return count; +} + +static ssize_t timeslice_duration_min_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.timeslice_min); +} + +static const struct kobj_attribute timeslice_duration_min_attr = + __ATTR(timeslice_duration_min, 0644, timeslice_duration_min_show, + timeslice_duration_min_store); + +static ssize_t timeslice_duration_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.timeslice_us); +} + +static const struct kobj_attribute timeslice_duration_attr = + __ATTR(timeslice_duration_us, 0644, timeslice_duration_show, + timeslice_duration_store); + +static ssize_t timeslice_default(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.timeslice_us); +} + +static const struct kobj_attribute timeslice_duration_def = +__ATTR(timeslice_duration_us, 0444, timeslice_default, NULL); + +static ssize_t timeslice_min_default(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.timeslice_min); +} + +static const struct kobj_attribute timeslice_duration_min_def = +__ATTR(timeslice_duration_min, 0444, timeslice_min_default, NULL); + +static ssize_t timeslice_max_default(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.timeslice_max); +} + +static const struct kobj_attribute timeslice_duration_max_def = +__ATTR(timeslice_duration_max, 0444, timeslice_max_default, NULL); + +static ssize_t preempt_timeout_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 min = eclass->sched_props.preempt_timeout_min; + u32 max = eclass->sched_props.preempt_timeout_max; + u32 timeout; + int err; + + err = kstrtou32(buf, 0, &timeout); + if (err) + return err; + + if (!xe_hw_engine_timeout_in_range(timeout, min, max)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.preempt_timeout_us, timeout); + + return count; +} + +static ssize_t preempt_timeout_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.preempt_timeout_us); +} + +static const struct kobj_attribute preempt_timeout_attr = +__ATTR(preempt_timeout_us, 0644, preempt_timeout_show, preempt_timeout_store); + +static ssize_t preempt_timeout_default(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.preempt_timeout_us); +} + +static const struct kobj_attribute preempt_timeout_def = +__ATTR(preempt_timeout_us, 0444, preempt_timeout_default, NULL); + +static ssize_t preempt_timeout_min_default(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.preempt_timeout_min); +} + +static const struct kobj_attribute preempt_timeout_min_def = +__ATTR(preempt_timeout_min, 0444, preempt_timeout_min_default, NULL); + +static ssize_t preempt_timeout_max_default(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent); + + return sysfs_emit(buf, "%u\n", eclass->defaults.preempt_timeout_max); +} + +static const struct kobj_attribute preempt_timeout_max_def = +__ATTR(preempt_timeout_max, 0444, preempt_timeout_max_default, NULL); + +static ssize_t preempt_timeout_max_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 timeout; + int err; + + err = kstrtou32(buf, 0, &timeout); + if (err) + return err; + + if (timeout < eclass->sched_props.preempt_timeout_min) + return -EINVAL; + + if (!xe_hw_engine_timeout_in_range(timeout, + XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN, + XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.preempt_timeout_max, timeout); + + return count; +} + +static ssize_t preempt_timeout_max_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.preempt_timeout_max); +} + +static const struct kobj_attribute preempt_timeout_max_attr = + __ATTR(preempt_timeout_max, 0644, preempt_timeout_max_show, + preempt_timeout_max_store); + +static ssize_t preempt_timeout_min_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + u32 timeout; + int err; + + err = kstrtou32(buf, 0, &timeout); + if (err) + return err; + + if (timeout > eclass->sched_props.preempt_timeout_max) + return -EINVAL; + + if (!xe_hw_engine_timeout_in_range(timeout, + XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN, + XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX)) + return -EINVAL; + + WRITE_ONCE(eclass->sched_props.preempt_timeout_min, timeout); + + return count; +} + +static ssize_t preempt_timeout_min_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj); + + return sysfs_emit(buf, "%u\n", eclass->sched_props.preempt_timeout_min); +} + +static const struct kobj_attribute preempt_timeout_min_attr = + __ATTR(preempt_timeout_min, 0644, preempt_timeout_min_show, + preempt_timeout_min_store); + +static const struct attribute *defaults[] = { + &job_timeout_def.attr, + &job_timeout_min_def.attr, + &job_timeout_max_def.attr, + ×lice_duration_def.attr, + ×lice_duration_min_def.attr, + ×lice_duration_max_def.attr, + &preempt_timeout_def.attr, + &preempt_timeout_min_def.attr, + &preempt_timeout_max_def.attr, + NULL +}; + +static const struct attribute * const files[] = { + &job_timeout_attr.attr, + &job_timeout_min_attr.attr, + &job_timeout_max_attr.attr, + ×lice_duration_attr.attr, + ×lice_duration_min_attr.attr, + ×lice_duration_max_attr.attr, + &preempt_timeout_attr.attr, + &preempt_timeout_min_attr.attr, + &preempt_timeout_max_attr.attr, + NULL +}; + +static void kobj_xe_hw_engine_class_fini(void *arg) +{ + struct kobject *kobj = arg; + + sysfs_remove_files(kobj, files); + kobject_put(kobj); +} + +static struct kobj_eclass * +kobj_xe_hw_engine_class(struct xe_device *xe, struct kobject *parent, const char *name) +{ + struct kobj_eclass *keclass; + int err = 0; + + keclass = kzalloc_obj(*keclass); + if (!keclass) + return NULL; + + kobject_init(&keclass->base, &kobj_xe_hw_engine_type); + if (kobject_add(&keclass->base, parent, "%s", name)) { + kobject_put(&keclass->base); + return NULL; + } + keclass->xe = xe; + + err = devm_add_action_or_reset(xe->drm.dev, kobj_xe_hw_engine_class_fini, + &keclass->base); + if (err) + return NULL; + + return keclass; +} + +static void hw_engine_class_defaults_fini(void *arg) +{ + struct kobject *kobj = arg; + + sysfs_remove_files(kobj, defaults); + kobject_put(kobj); +} + +static int xe_add_hw_engine_class_defaults(struct xe_device *xe, + struct kobject *parent) +{ + struct kobject *kobj; + int err = 0; + + kobj = kzalloc_obj(*kobj); + if (!kobj) + return -ENOMEM; + + kobject_init(kobj, &kobj_xe_hw_engine_type_def); + err = kobject_add(kobj, parent, "%s", ".defaults"); + if (err) + goto err_object; + + err = sysfs_create_files(kobj, defaults); + if (err) + goto err_object; + + return devm_add_action_or_reset(xe->drm.dev, hw_engine_class_defaults_fini, kobj); + +err_object: + kobject_put(kobj); + return err; +} +ALLOW_ERROR_INJECTION(xe_add_hw_engine_class_defaults, ERRNO); /* See xe_pci_probe() */ + + +static void hw_engine_class_sysfs_fini(void *arg) +{ + struct kobject *kobj = arg; + + kobject_put(kobj); +} + +/** + * xe_hw_engine_class_sysfs_init - Init HW engine classes on GT. + * @gt: Xe GT. + * + * This routine creates sysfs for HW engine classes and adds methods + * to get/set different scheduling properties for HW engines class. + * + * Returns: Returns error value for failure and 0 for success. + */ +int xe_hw_engine_class_sysfs_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct kobject *kobj; + u16 class_mask = 0; + int err = 0; + + kobj = kzalloc_obj(*kobj); + if (!kobj) + return -ENOMEM; + + kobject_init(kobj, &kobj_xe_hw_engine_type); + + err = kobject_add(kobj, gt->sysfs, "engines"); + if (err) + goto err_object; + + for_each_hw_engine(hwe, gt, id) { + const char *name; + struct kobj_eclass *keclass; + + if (hwe->class == XE_ENGINE_CLASS_OTHER || + hwe->class == XE_ENGINE_CLASS_MAX) + continue; + + if ((class_mask >> hwe->class) & 1) + continue; + + class_mask |= 1 << hwe->class; + name = xe_hw_engine_class_to_str(hwe->class); + if (!name) { + err = -EINVAL; + goto err_object; + } + + keclass = kobj_xe_hw_engine_class(xe, kobj, name); + if (!keclass) { + err = -EINVAL; + goto err_object; + } + + keclass->eclass = hwe->eclass; + err = xe_add_hw_engine_class_defaults(xe, &keclass->base); + if (err) + goto err_object; + + err = sysfs_create_files(&keclass->base, files); + if (err) + goto err_object; + } + + return devm_add_action_or_reset(xe->drm.dev, hw_engine_class_sysfs_fini, kobj); + +err_object: + kobject_put(kobj); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h new file mode 100644 index 000000000..37225c1ae --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_CLASS_SYSFS_H_ +#define _XE_HW_ENGINE_CLASS_SYSFS_H_ + +#include + +struct xe_gt; +struct xe_hw_engine_class_intf; + +int xe_hw_engine_class_sysfs_init(struct xe_gt *gt); +bool xe_hw_engine_timeout_in_range(u64 timeout, u64 min, u64 max); + +/** + * struct kobj_eclass - A eclass's kobject struct that connects the kobject and the + * eclass. + * + * When dealing with multiple eclass, this struct helps to understand which eclass + * needs to be addressed on a given sysfs call. + */ +struct kobj_eclass { + /** @base: The actual kobject */ + struct kobject base; + /** @eclass: A pointer to the hw engine class interface */ + struct xe_hw_engine_class_intf *eclass; + /** @xe: A pointer to the xe device */ + struct xe_device *xe; +}; + +static inline struct xe_hw_engine_class_intf *kobj_to_eclass(struct kobject *kobj) +{ + return container_of(kobj, struct kobj_eclass, base)->eclass; +} + +static inline struct xe_device *kobj_to_xe(struct kobject *kobj) +{ + return container_of(kobj, struct kobj_eclass, base)->xe; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c new file mode 100644 index 000000000..0804b426b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include "xe_assert.h" +#include "xe_device_types.h" +#include "xe_exec_queue.h" +#include "xe_gt.h" +#include "xe_gt_stats.h" +#include "xe_hw_engine_group.h" +#include "xe_sync.h" +#include "xe_vm.h" + +static void +hw_engine_group_resume_lr_jobs_func(struct work_struct *w) +{ + struct xe_exec_queue *q; + struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work); + int err; + enum xe_hw_engine_group_execution_mode previous_mode; + + err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode, + NULL, 0); + if (err) + return; + + if (previous_mode == EXEC_MODE_LR) + goto put; + + list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { + if (!xe_vm_in_fault_mode(q->vm)) + continue; + + /* + * Only resume queues that were actually suspended. A queue whose + * suspend() failed (e.g. killed/banned/wedged) was never + * suspended, so it must not be resumed. + */ + if (!READ_ONCE(q->lr.suspended)) + continue; + + WRITE_ONCE(q->lr.suspended, false); + q->ops->resume(q); + } + +put: + xe_hw_engine_group_put(group); +} + +static struct xe_hw_engine_group * +hw_engine_group_alloc(struct xe_device *xe) +{ + struct xe_hw_engine_group *group; + int err; + + group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL); + if (!group) + return ERR_PTR(-ENOMEM); + + group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU, + 0); + if (!group->resume_wq) + return ERR_PTR(-ENOMEM); + + err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq); + if (err) + return ERR_PTR(err); + + init_rwsem(&group->mode_sem); + INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func); + INIT_LIST_HEAD(&group->exec_queue_list); + + return group; +} + +/** + * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt + * @gt: The gt for which groups are setup + * + * Return: 0 on success, negative error code on error. + */ +int xe_hw_engine_setup_groups(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs; + struct xe_device *xe = gt_to_xe(gt); + + group_rcs_ccs = hw_engine_group_alloc(xe); + if (IS_ERR(group_rcs_ccs)) + return PTR_ERR(group_rcs_ccs); + + group_bcs = hw_engine_group_alloc(xe); + if (IS_ERR(group_bcs)) + return PTR_ERR(group_bcs); + + group_vcs_vecs = hw_engine_group_alloc(xe); + if (IS_ERR(group_vcs_vecs)) + return PTR_ERR(group_vcs_vecs); + + for_each_hw_engine(hwe, gt, id) { + switch (hwe->class) { + case XE_ENGINE_CLASS_COPY: + hwe->hw_engine_group = group_bcs; + break; + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + hwe->hw_engine_group = group_rcs_ccs; + break; + case XE_ENGINE_CLASS_VIDEO_DECODE: + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + hwe->hw_engine_group = group_vcs_vecs; + break; + case XE_ENGINE_CLASS_OTHER: + break; + case XE_ENGINE_CLASS_MAX: + xe_gt_assert(gt, false); + } + } + + return 0; +} + +/** + * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group + * @group: The hw engine group + * @q: The exec_queue + * + * Return: 0 on success, + * -EINTR if the lock could not be acquired + */ +int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) +{ + int err; + struct xe_device *xe = gt_to_xe(q->gt); + + xe_assert(xe, group); + xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); + xe_assert(xe, q->vm); + + if (xe_vm_in_preempt_fence_mode(q->vm)) + return 0; + + err = down_write_killable(&group->mode_sem); + if (err) + return err; + + if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) { + /* + * suspend() can fail (e.g. killed/banned/wedged), leaving the + * queue un-suspended. Propagate the failure so the queue is not + * added; on failure nothing was suspended, so there is nothing to + * undo. Only record the queue as suspended (and later resume it) + * once suspend() has succeeded. + */ + err = q->ops->suspend(q); + if (err) + goto err_suspend; + + WRITE_ONCE(q->lr.suspended, true); + err = q->ops->suspend_wait(q); + if (err) + goto err_suspend; + + xe_hw_engine_group_resume_faulting_lr_jobs(group); + } + + list_add(&q->hw_engine_group_link, &group->exec_queue_list); + up_write(&group->mode_sem); + + return 0; + +err_suspend: + up_write(&group->mode_sem); + return err; +} +ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO); + +/** + * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group + * @group: The hw engine group + * @q: The exec_queue + */ +void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) +{ + struct xe_device *xe = gt_to_xe(q->gt); + + xe_assert(xe, group); + xe_assert(xe, q->vm); + + down_write(&group->mode_sem); + + if (!list_empty(&q->hw_engine_group_link)) + list_del(&q->hw_engine_group_link); + + up_write(&group->mode_sem); +} + +/** + * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's + * faulting LR jobs + * @group: The hw engine group + */ +void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group) +{ + queue_work(group->resume_wq, &group->resume_work); +} + +/** + * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group + * @group: The hw engine group + * @has_deps: dma-fence job triggering suspend has dependencies + * + * Return: 0 on success, negative error code on error. + */ +static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group, + bool has_deps) +{ + int err; + struct xe_exec_queue *q; + struct xe_gt *gt = NULL; + bool need_resume = false; + ktime_t start = xe_gt_stats_ktime_get(); + + lockdep_assert_held_write(&group->mode_sem); + + list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { + + if (!xe_vm_in_fault_mode(q->vm)) + continue; + + if (has_deps) + return -EAGAIN; + + xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); + /* + * suspend() only fails when the queue is killed/banned/wedged. + * Such a queue is being torn down (its removal from HW is handled + * by the kill/ban teardown), so it is not a live fault-mode + * context the mode switch must preempt. Skip it rather than + * failing the switch, otherwise one dying sibling would block a + * dma-fence submission on the healthy queues in the group. Only + * queues recorded as suspended below are later waited on and + * resumed. + */ + err = q->ops->suspend(q); + if (err) + continue; + + WRITE_ONCE(q->lr.suspended, true); + need_resume = true; + gt = q->gt; + } + + list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { + if (!xe_vm_in_fault_mode(q->vm)) + continue; + + /* Only wait on queues that were actually suspended above. */ + if (!READ_ONCE(q->lr.suspended)) + continue; + + err = q->ops->suspend_wait(q); + if (err) + goto err_resume; + } + + if (gt) { + xe_gt_stats_incr(gt, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, + xe_gt_stats_ktime_us_delta(start)); + } + + if (need_resume) + xe_hw_engine_group_resume_faulting_lr_jobs(group); + + return 0; + +err_resume: + /* + * A suspend_wait() failed partway through the mode switch. Resume the + * sibling queues that were already suspended in this call so they are + * not left suspended forever. + * + * resume() requires the suspend to have completed (suspend_pending + * cleared) or it trips the !suspend_pending assert. So skip the resume + * when either: + * - suspend_wait_blocking() fails: on a GuC timeout it bans the queue + * and triggers cleanup, so the queue is being torn down; or + * - reset_status() is true: the queue was reset/killed/banned/wedged. + * suspend_wait() can return success in this case via its killed/ + * stopped wait condition while suspend_pending is still set, and the + * queue is being torn down anyway, so its state is resolved by + * teardown rather than by a resume here. + * In either case leave the queue marked suspended. + * + * Use the *blocking* (uninterruptible) wait here: the queues resumed on + * this path may belong to a different process than the one that + * triggered the mode switch. An interruptible suspend_wait() would + * return -ERESTARTSYS if the triggering task is signalled, skip the + * resume, and leave the other process's queue suspended forever + * (cross-process DoS). + */ + list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { + if (!xe_vm_in_fault_mode(q->vm)) + continue; + + if (!READ_ONCE(q->lr.suspended)) + continue; + + if (q->ops->suspend_wait_blocking(q) || q->ops->reset_status(q)) + continue; + + WRITE_ONCE(q->lr.suspended, false); + q->ops->resume(q); + } + + return err; +} + +/** + * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete + * @group: The hw engine group + * + * This function is not meant to be called directly from a user IOCTL as dma_fence_wait() + * is not interruptible. + * + * Return: 0 on success, + * -ETIME if waiting for one job failed + */ +static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group) +{ + long timeout; + struct xe_exec_queue *q; + struct xe_gt *gt = NULL; + struct dma_fence *fence; + ktime_t start = xe_gt_stats_ktime_get(); + + lockdep_assert_held_write(&group->mode_sem); + + list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { + if (xe_vm_in_lr_mode(q->vm)) + continue; + + xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1); + fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); + timeout = dma_fence_wait(fence, false); + dma_fence_put(fence); + gt = q->gt; + + if (timeout < 0) + return -ETIME; + } + + if (gt) { + xe_gt_stats_incr(gt, + XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, + xe_gt_stats_ktime_us_delta(start)); + } + + return 0; +} + +static int switch_mode(struct xe_hw_engine_group *group, bool has_deps) +{ + int err = 0; + enum xe_hw_engine_group_execution_mode new_mode; + + lockdep_assert_held_write(&group->mode_sem); + + switch (group->cur_mode) { + case EXEC_MODE_LR: + new_mode = EXEC_MODE_DMA_FENCE; + err = xe_hw_engine_group_suspend_faulting_lr_jobs(group, + has_deps); + break; + case EXEC_MODE_DMA_FENCE: + new_mode = EXEC_MODE_LR; + err = xe_hw_engine_group_wait_for_dma_fence_jobs(group); + break; + } + + if (err) + return err; + + group->cur_mode = new_mode; + + return 0; +} + +static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs) +{ + int err, i; + + for (i = 0; i < num_syncs; ++i) { + err = xe_sync_entry_wait(syncs + i); + if (err) + return err; + } + + return 0; +} + +/** + * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode + * @group: The hw engine group + * @new_mode: The new execution mode + * @previous_mode: Pointer to the previous mode provided for use by caller + * @syncs: Syncs from exec IOCTL + * @num_syncs: Number of syncs from exec IOCTL + * + * Return: 0 if successful, -EINTR if locking failed. + */ +int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, + enum xe_hw_engine_group_execution_mode new_mode, + enum xe_hw_engine_group_execution_mode *previous_mode, + struct xe_sync_entry *syncs, int num_syncs) +__acquires(&group->mode_sem) +{ + bool has_deps = !!num_syncs; + int err = down_read_interruptible(&group->mode_sem); + + if (err) + return err; + + *previous_mode = group->cur_mode; + + if (new_mode != group->cur_mode) { + up_read(&group->mode_sem); +retry: + err = down_write_killable(&group->mode_sem); + if (err) + return err; + + if (new_mode != group->cur_mode) { + err = switch_mode(group, has_deps); + if (err) { + up_write(&group->mode_sem); + + if (err != -EAGAIN) + return err; + + err = wait_syncs(syncs, num_syncs); + if (err) + return err; + + has_deps = false; + goto retry; + } + } + downgrade_write(&group->mode_sem); + } + + return err; +} + +/** + * xe_hw_engine_group_put() - Put the group + * @group: The hw engine group + */ +void xe_hw_engine_group_put(struct xe_hw_engine_group *group) +__releases(&group->mode_sem) +{ + up_read(&group->mode_sem); +} + +/** + * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue + * @q: The exec_queue + */ +enum xe_hw_engine_group_execution_mode +xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q) +{ + if (xe_vm_in_fault_mode(q->vm)) + return EXEC_MODE_LR; + else + return EXEC_MODE_DMA_FENCE; +} diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.h b/drivers/gpu/drm/xe/xe_hw_engine_group.h new file mode 100644 index 000000000..8b17ccd30 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_GROUP_H_ +#define _XE_HW_ENGINE_GROUP_H_ + +#include "xe_hw_engine_group_types.h" + +struct drm_device; +struct xe_exec_queue; +struct xe_gt; +struct xe_sync_entry; + +int xe_hw_engine_setup_groups(struct xe_gt *gt); + +int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q); +void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q); + +int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, + enum xe_hw_engine_group_execution_mode new_mode, + enum xe_hw_engine_group_execution_mode *previous_mode, + struct xe_sync_entry *syncs, int num_syncs); +void xe_hw_engine_group_put(struct xe_hw_engine_group *group); + +enum xe_hw_engine_group_execution_mode +xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q); +void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group); + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group_types.h b/drivers/gpu/drm/xe/xe_hw_engine_group_types.h new file mode 100644 index 000000000..92b6e0712 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_group_types.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_GROUP_TYPES_H_ +#define _XE_HW_ENGINE_GROUP_TYPES_H_ + +#include "xe_force_wake_types.h" +#include "xe_lrc_types.h" +#include "xe_reg_sr_types.h" + +/** + * enum xe_hw_engine_group_execution_mode - possible execution modes of a hw + * engine group + * + * @EXEC_MODE_LR: execution in long-running mode + * @EXEC_MODE_DMA_FENCE: execution in dma fence mode + */ +enum xe_hw_engine_group_execution_mode { + EXEC_MODE_LR, + EXEC_MODE_DMA_FENCE, +}; + +/** + * struct xe_hw_engine_group - Hardware engine group + * + * hw engines belong to the same group if they share hardware resources in a way + * that prevents them from making progress when one is stuck on a page fault. + */ +struct xe_hw_engine_group { + /** + * @exec_queue_list: list of exec queues attached to this + * xe_hw_engine_group + */ + struct list_head exec_queue_list; + /** @resume_work: worker to resume faulting LR exec queues */ + struct work_struct resume_work; + /** @resume_wq: workqueue to resume faulting LR exec queues */ + struct workqueue_struct *resume_wq; + /** + * @mode_sem: used to protect this group's hardware resources and ensure + * mutual exclusion between execution only in faulting LR mode and + * execution only in DMA_FENCE mode + */ + struct rw_semaphore mode_sem; + /** @cur_mode: current execution mode of this hw engine group */ + enum xe_hw_engine_group_execution_mode cur_mode; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h new file mode 100644 index 000000000..ff115ab42 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_TYPES_H_ +#define _XE_HW_ENGINE_TYPES_H_ + +#include "xe_force_wake_types.h" +#include "xe_lrc_types.h" +#include "xe_reg_sr_types.h" + +/* See "Engine ID Definition" struct in the Icelake PRM */ +enum xe_engine_class { + XE_ENGINE_CLASS_RENDER = 0, + XE_ENGINE_CLASS_VIDEO_DECODE = 1, + XE_ENGINE_CLASS_VIDEO_ENHANCE = 2, + XE_ENGINE_CLASS_COPY = 3, + XE_ENGINE_CLASS_OTHER = 4, + XE_ENGINE_CLASS_COMPUTE = 5, + XE_ENGINE_CLASS_MAX = 6, +}; + +enum xe_hw_engine_id { + XE_HW_ENGINE_RCS0, +#define XE_HW_ENGINE_RCS_MASK GENMASK_ULL(XE_HW_ENGINE_RCS0, XE_HW_ENGINE_RCS0) + XE_HW_ENGINE_BCS0, + XE_HW_ENGINE_BCS1, + XE_HW_ENGINE_BCS2, + XE_HW_ENGINE_BCS3, + XE_HW_ENGINE_BCS4, + XE_HW_ENGINE_BCS5, + XE_HW_ENGINE_BCS6, + XE_HW_ENGINE_BCS7, + XE_HW_ENGINE_BCS8, +#define XE_HW_ENGINE_BCS_MASK GENMASK_ULL(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS0) + XE_HW_ENGINE_VCS0, + XE_HW_ENGINE_VCS1, + XE_HW_ENGINE_VCS2, + XE_HW_ENGINE_VCS3, + XE_HW_ENGINE_VCS4, + XE_HW_ENGINE_VCS5, + XE_HW_ENGINE_VCS6, + XE_HW_ENGINE_VCS7, +#define XE_HW_ENGINE_VCS_MASK GENMASK_ULL(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) + XE_HW_ENGINE_VECS0, + XE_HW_ENGINE_VECS1, + XE_HW_ENGINE_VECS2, + XE_HW_ENGINE_VECS3, +#define XE_HW_ENGINE_VECS_MASK GENMASK_ULL(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) + XE_HW_ENGINE_CCS0, + XE_HW_ENGINE_CCS1, + XE_HW_ENGINE_CCS2, + XE_HW_ENGINE_CCS3, +#define XE_HW_ENGINE_CCS_MASK GENMASK_ULL(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0) + XE_HW_ENGINE_GSCCS0, +#define XE_HW_ENGINE_GSCCS_MASK GENMASK_ULL(XE_HW_ENGINE_GSCCS0, XE_HW_ENGINE_GSCCS0) + XE_NUM_HW_ENGINES, +}; + +/* FIXME: s/XE_HW_ENGINE_MAX_INSTANCE/XE_HW_ENGINE_MAX_COUNT */ +#define XE_HW_ENGINE_MAX_INSTANCE 9 + +struct xe_bo; +struct xe_execlist_port; +struct xe_gt; + +/** + * struct xe_hw_engine_class_intf - per hw engine class struct interface + * + * Contains all the hw engine properties per engine class. + * + * @sched_props: scheduling properties + * @defaults: default scheduling properties + */ +struct xe_hw_engine_class_intf { + /** + * @sched_props: scheduling properties + * @defaults: default scheduling properties + */ + struct { + /** @sched_props.job_timeout_ms: Set job timeout in ms for engine */ + u32 job_timeout_ms; + /** @sched_props.job_timeout_min: Min job timeout in ms for engine */ + u32 job_timeout_min; + /** @sched_props.job_timeout_max: Max job timeout in ms for engine */ + u32 job_timeout_max; + /** @sched_props.timeslice_us: timeslice period in micro-seconds */ + u32 timeslice_us; + /** @sched_props.timeslice_min: min timeslice period in micro-seconds */ + u32 timeslice_min; + /** @sched_props.timeslice_max: max timeslice period in micro-seconds */ + u32 timeslice_max; + /** @sched_props.preempt_timeout_us: preemption timeout in micro-seconds */ + u32 preempt_timeout_us; + /** @sched_props.preempt_timeout_min: min preemption timeout in micro-seconds */ + u32 preempt_timeout_min; + /** @sched_props.preempt_timeout_max: max preemption timeout in micro-seconds */ + u32 preempt_timeout_max; + } sched_props, defaults; +}; + +/** + * struct xe_hw_engine - Hardware engine + * + * Contains all the hardware engine state for physical instances. + */ +struct xe_hw_engine { + /** @gt: GT structure this hw engine belongs to */ + struct xe_gt *gt; + /** @name: name of this hw engine */ + const char *name; + /** @class: class of this hw engine */ + enum xe_engine_class class; + /** @instance: physical instance of this hw engine */ + u16 instance; + /** + * @logical_instance: logical instance of this hw engine. + * + * Note: For GuC usage, always use xe_hwe_guc_logical_instance(). + * For GuC usage, we should no longer use the raw logical instance. + */ + u16 logical_instance; + /** @irq_offset: IRQ offset of this hw engine */ + u16 irq_offset; + /** @irq_page: MEMIRQ page used by this HW engine */ + u16 irq_page; + /** @mmio_base: MMIO base address of this hw engine*/ + u32 mmio_base; + /** + * @reg_sr: table with registers to be restored on GT init/resume/reset + */ + struct xe_reg_sr reg_sr; + /** + * @reg_whitelist: table with registers to be whitelisted + */ + struct xe_reg_sr reg_whitelist; + /** + * @oa_whitelist: oa registers to be whitelisted + */ + struct xe_reg_sr oa_whitelist; + /** + * @oa_sr: oa nonpriv whitelist registers, changed on oa stream open/close + */ + struct xe_reg_sr oa_sr; + /** + * @reg_lrc: LRC workaround registers + */ + struct xe_reg_sr reg_lrc; + /** @domain: force wake domain of this hw engine */ + enum xe_force_wake_domains domain; + /** @hwsp: hardware status page buffer object */ + struct xe_bo *hwsp; + /** @exl_port: execlists port */ + struct xe_execlist_port *exl_port; + /** @fence_irq: fence IRQ to run when a hw engine IRQ is received */ + struct xe_hw_fence_irq *fence_irq; + /** @irq_handler: IRQ handler to run when hw engine IRQ is received */ + void (*irq_handler)(struct xe_hw_engine *hwe, u16 intr_vec); + /** @engine_id: id for this hw engine */ + enum xe_hw_engine_id engine_id; + /** @eclass: pointer to per hw engine class interface */ + struct xe_hw_engine_class_intf *eclass; + /** @oa_unit: oa unit for this hw engine */ + struct xe_oa_unit *oa_unit; + /** @hw_engine_group: the group of hw engines this one belongs to */ + struct xe_hw_engine_group *hw_engine_group; +}; + +enum xe_hw_engine_snapshot_source_id { + XE_ENGINE_CAPTURE_SOURCE_MANUAL, + XE_ENGINE_CAPTURE_SOURCE_GUC +}; + +/** + * struct xe_hw_engine_snapshot - Hardware engine snapshot + * + * Contains the snapshot of useful hardware engine info and registers. + */ +struct xe_hw_engine_snapshot { + /** @name: name of the hw engine */ + char *name; + /** @hwe: hw engine */ + struct xe_hw_engine *hwe; + /** @logical_instance: logical instance of this hw engine */ + u16 logical_instance; + /** @forcewake: Force Wake information snapshot */ + struct { + /** @forcewake.domain: force wake domain of this hw engine */ + enum xe_force_wake_domains domain; + /** @forcewake.ref: Forcewake ref for the above domain */ + int ref; + } forcewake; + /** @mmio_base: MMIO base address of this hw engine*/ + u32 mmio_base; + /** @kernel_reserved: Engine reserved, can't be used by userspace */ + bool kernel_reserved; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c new file mode 100644 index 000000000..5f2abc948 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -0,0 +1,555 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "regs/xe_gsc_regs.h" +#include "regs/xe_hw_error_regs.h" +#include "regs/xe_irq_regs.h" + +#include "xe_debugfs.h" +#include "xe_device.h" +#include "xe_drm_ras.h" +#include "xe_hw_error.h" +#include "xe_mmio.h" +#include "xe_survivability_mode.h" + +#define GT_HW_ERROR_MAX_ERR_BITS 16 +#define HEC_UNCORR_FW_ERR_BITS 4 +#define XE_RAS_REG_SIZE 32 +#define XE_SOC_NUM_IEH 2 + +#define PVC_ERROR_MASK_SET(hw_err, err_bit) ((hw_err == HARDWARE_ERROR_CORRECTABLE) ? \ + (PVC_COR_ERR_MASK & REG_BIT(err_bit)) : \ + (PVC_FAT_ERR_MASK & REG_BIT(err_bit))) + +static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES; + +static const char * const hec_uncorrected_fw_errors[] = { + "Fatal", + "CSE Disabled", + "FD Corruption", + "Data Corruption" +}; + +enum gt_vector_regs { + ERR_STAT_GT_VECTOR0 = 0, + ERR_STAT_GT_VECTOR1, + ERR_STAT_GT_VECTOR2, + ERR_STAT_GT_VECTOR3, + ERR_STAT_GT_VECTOR4, + ERR_STAT_GT_VECTOR5, + ERR_STAT_GT_VECTOR6, + ERR_STAT_GT_VECTOR7, + ERR_STAT_GT_VECTOR_MAX +}; + +#define PVC_GT_VECTOR_LEN(hw_err) ((hw_err == HARDWARE_ERROR_CORRECTABLE) ? \ + ERR_STAT_GT_VECTOR4 : ERR_STAT_GT_VECTOR_MAX) + +static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_error hw_err) +{ + if (hw_err == HARDWARE_ERROR_CORRECTABLE) + return DRM_XE_RAS_ERR_SEV_CORRECTABLE; + + /* Uncorrectable errors comprise of both fatal and non-fatal errors */ + return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE; +} + +static inline u32 err_src_to_id(u32 err_bit) +{ + switch (err_bit) { + case XE_GT_ERROR: + return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; + case XE_SOC_ERROR: + return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL; + default: + return 0; + } +} + +static const char * const pvc_master_global_err_reg[] = { + [0 ... 1] = "Undefined", + [2] = "HBM SS0: Channel0", + [3] = "HBM SS0: Channel1", + [4] = "HBM SS0: Channel2", + [5] = "HBM SS0: Channel3", + [6] = "HBM SS0: Channel4", + [7] = "HBM SS0: Channel5", + [8] = "HBM SS0: Channel6", + [9] = "HBM SS0: Channel7", + [10] = "HBM SS1: Channel0", + [11] = "HBM SS1: Channel1", + [12] = "HBM SS1: Channel2", + [13] = "HBM SS1: Channel3", + [14] = "HBM SS1: Channel4", + [15] = "HBM SS1: Channel5", + [16] = "HBM SS1: Channel6", + [17] = "HBM SS1: Channel7", + [18 ... 31] = "Undefined", +}; +static_assert(ARRAY_SIZE(pvc_master_global_err_reg) == XE_RAS_REG_SIZE); + +static const char * const pvc_slave_global_err_reg[] = { + [0] = "Undefined", + [1] = "HBM SS2: Channel0", + [2] = "HBM SS2: Channel1", + [3] = "HBM SS2: Channel2", + [4] = "HBM SS2: Channel3", + [5] = "HBM SS2: Channel4", + [6] = "HBM SS2: Channel5", + [7] = "HBM SS2: Channel6", + [8] = "HBM SS2: Channel7", + [9] = "HBM SS3: Channel0", + [10] = "HBM SS3: Channel1", + [11] = "HBM SS3: Channel2", + [12] = "HBM SS3: Channel3", + [13] = "HBM SS3: Channel4", + [14] = "HBM SS3: Channel5", + [15] = "HBM SS3: Channel6", + [16] = "HBM SS3: Channel7", + [17] = "Undefined", + [18] = "ANR MDFI", + [19 ... 31] = "Undefined", +}; +static_assert(ARRAY_SIZE(pvc_slave_global_err_reg) == XE_RAS_REG_SIZE); + +static const char * const pvc_slave_local_fatal_err_reg[] = { + [0] = "Local IEH: Malformed PCIe AER", + [1] = "Local IEH: Malformed PCIe ERR", + [2] = "Local IEH: UR conditions in IEH", + [3] = "Local IEH: From SERR Sources", + [4 ... 19] = "Undefined", + [20] = "Malformed MCA error packet (HBM/Punit)", + [21 ... 31] = "Undefined", +}; +static_assert(ARRAY_SIZE(pvc_slave_local_fatal_err_reg) == XE_RAS_REG_SIZE); + +static const char * const pvc_master_local_fatal_err_reg[] = { + [0] = "Local IEH: Malformed IOSF PCIe AER", + [1] = "Local IEH: Malformed IOSF PCIe ERR", + [2] = "Local IEH: UR RESPONSE", + [3] = "Local IEH: From SERR SPI controller", + [4] = "Base Die MDFI T2T", + [5] = "Undefined", + [6] = "Base Die MDFI T2C", + [7] = "Undefined", + [8] = "Invalid CSC PSF Command Parity", + [9] = "Invalid CSC PSF Unexpected Completion", + [10] = "Invalid CSC PSF Unsupported Request", + [11] = "Invalid PCIe PSF Command Parity", + [12] = "PCIe PSF Unexpected Completion", + [13] = "PCIe PSF Unsupported Request", + [14 ... 19] = "Undefined", + [20] = "Malformed MCA error packet (HBM/Punit)", + [21 ... 31] = "Undefined", +}; +static_assert(ARRAY_SIZE(pvc_master_local_fatal_err_reg) == XE_RAS_REG_SIZE); + +static const char * const pvc_master_local_nonfatal_err_reg[] = { + [0 ... 3] = "Undefined", + [4] = "Base Die MDFI T2T", + [5] = "Undefined", + [6] = "Base Die MDFI T2C", + [7] = "Undefined", + [8] = "Invalid CSC PSF Command Parity", + [9] = "Invalid CSC PSF Unexpected Completion", + [10] = "Invalid PCIe PSF Command Parity", + [11 ... 31] = "Undefined", +}; +static_assert(ARRAY_SIZE(pvc_master_local_nonfatal_err_reg) == XE_RAS_REG_SIZE); + +#define PVC_MASTER_LOCAL_REG_INFO(hw_err) ((hw_err == HARDWARE_ERROR_FATAL) ? \ + pvc_master_local_fatal_err_reg : \ + pvc_master_local_nonfatal_err_reg) + +static void csc_hw_error_work(struct work_struct *work) +{ + struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work); + struct xe_device *xe = tile_to_xe(tile); + + xe_survivability_mode_runtime_enable(xe); +} + +static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err) +{ + const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err); + const char *severity_str = error_severity[severity]; + struct xe_device *xe = tile_to_xe(tile); + struct xe_mmio *mmio = &tile->mmio; + u32 base, err_bit, err_src; + unsigned long fw_err; + + if (xe->info.platform != XE_BATTLEMAGE) + return; + + base = BMG_GSC_HECI1_BASE; + lockdep_assert_held(&xe->irq.lock); + err_src = xe_mmio_read32(mmio, HEC_UNCORR_ERR_STATUS(base)); + if (!err_src) { + drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported %s HEC_ERR_STATUS register blank\n", + tile->id, severity_str); + return; + } + + if (err_src & UNCORR_FW_REPORTED_ERR) { + fw_err = xe_mmio_read32(mmio, HEC_UNCORR_FW_ERR_DW0(base)); + for_each_set_bit(err_bit, &fw_err, HEC_UNCORR_FW_ERR_BITS) { + drm_err_ratelimited(&xe->drm, HW_ERR + "HEC FW %s %s reported, bit[%d] is set\n", + hec_uncorrected_fw_errors[err_bit], severity_str, + err_bit); + + schedule_work(&tile->csc_hw_error_work); + } + } + + xe_mmio_write32(mmio, HEC_UNCORR_ERR_STATUS(base), err_src); +} + +static void log_hw_error(struct xe_tile *tile, const char *name, + const enum drm_xe_ras_error_severity severity) +{ + const char *severity_str = error_severity[severity]; + struct xe_device *xe = tile_to_xe(tile); + + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) + drm_warn(&xe->drm, HW_ERR "%s %s detected\n", name, severity_str); + else + drm_err_ratelimited(&xe->drm, HW_ERR "%s %s detected\n", name, severity_str); +} + +static void log_gt_err(struct xe_tile *tile, const char *name, int i, u32 err, + const enum drm_xe_ras_error_severity severity) +{ + const char *severity_str = error_severity[severity]; + struct xe_device *xe = tile_to_xe(tile); + + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) + drm_warn(&xe->drm, HW_ERR "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n", + name, severity_str, i, err); + else + drm_err_ratelimited(&xe->drm, HW_ERR "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n", + name, severity_str, i, err); +} + +static void log_soc_error(struct xe_tile *tile, const char * const *reg_info, + const enum drm_xe_ras_error_severity severity, u32 err_bit, u32 index) +{ + const char *severity_str = error_severity[severity]; + struct xe_device *xe = tile_to_xe(tile); + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[severity]; + const char *name; + + name = reg_info[err_bit]; + + if (strcmp(name, "Undefined")) { + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) + drm_warn(&xe->drm, HW_ERR "%s SOC %s detected", name, severity_str); + else + drm_err_ratelimited(&xe->drm, HW_ERR "%s SOC %s detected", name, severity_str); + atomic_inc(&info[index].counter); + } +} + +static void gt_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err, + u32 error_id) +{ + const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err); + struct xe_device *xe = tile_to_xe(tile); + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[severity]; + struct xe_mmio *mmio = &tile->mmio; + unsigned long err_stat = 0; + int i; + + if (xe->info.platform != XE_PVC) + return; + + if (hw_err == HARDWARE_ERROR_NONFATAL) { + atomic_inc(&info[error_id].counter); + log_hw_error(tile, info[error_id].name, severity); + return; + } + + for (i = 0; i < PVC_GT_VECTOR_LEN(hw_err); i++) { + u32 vector, val; + + vector = xe_mmio_read32(mmio, ERR_STAT_GT_VECTOR_REG(hw_err, i)); + if (!vector) + continue; + + switch (i) { + case ERR_STAT_GT_VECTOR0: + case ERR_STAT_GT_VECTOR1: { + u32 errbit; + + val = hweight32(vector); + atomic_add(val, &info[error_id].counter); + log_gt_err(tile, "Subslice", i, vector, severity); + + /* + * Error status register is only populated once per error. + * Read the register and clear once. + */ + if (err_stat) + break; + + err_stat = xe_mmio_read32(mmio, ERR_STAT_GT_REG(hw_err)); + for_each_set_bit(errbit, &err_stat, GT_HW_ERROR_MAX_ERR_BITS) { + if (PVC_ERROR_MASK_SET(hw_err, errbit)) + atomic_inc(&info[error_id].counter); + } + if (err_stat) + xe_mmio_write32(mmio, ERR_STAT_GT_REG(hw_err), err_stat); + break; + } + case ERR_STAT_GT_VECTOR2: + case ERR_STAT_GT_VECTOR3: + val = hweight32(vector); + atomic_add(val, &info[error_id].counter); + log_gt_err(tile, "L3 BANK", i, vector, severity); + break; + case ERR_STAT_GT_VECTOR6: + val = hweight32(vector); + atomic_add(val, &info[error_id].counter); + log_gt_err(tile, "TLB", i, vector, severity); + break; + case ERR_STAT_GT_VECTOR7: + val = hweight32(vector); + atomic_add(val, &info[error_id].counter); + log_gt_err(tile, "L3 Fabric", i, vector, severity); + break; + default: + log_gt_err(tile, "Undefined", i, vector, severity); + } + + xe_mmio_write32(mmio, ERR_STAT_GT_VECTOR_REG(hw_err, i), vector); + } +} + +static void soc_slave_ieh_handler(struct xe_tile *tile, const enum hardware_error hw_err, u32 error_id) +{ + const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err); + unsigned long slave_global_errstat, slave_local_errstat; + struct xe_mmio *mmio = &tile->mmio; + u32 regbit, slave; + + slave = SOC_PVC_SLAVE_BASE; + slave_global_errstat = xe_mmio_read32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err)); + + if (slave_global_errstat & SOC_IEH1_LOCAL_ERR_STATUS) { + slave_local_errstat = xe_mmio_read32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err)); + + if (hw_err == HARDWARE_ERROR_FATAL) { + for_each_set_bit(regbit, &slave_local_errstat, XE_RAS_REG_SIZE) + log_soc_error(tile, pvc_slave_local_fatal_err_reg, severity, + regbit, error_id); + } + + xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err), + slave_local_errstat); + } + + for_each_set_bit(regbit, &slave_global_errstat, XE_RAS_REG_SIZE) + log_soc_error(tile, pvc_slave_global_err_reg, severity, regbit, error_id); + + xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err), slave_global_errstat); +} + +static void soc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err, + u32 error_id) +{ + const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err); + struct xe_device *xe = tile_to_xe(tile); + struct xe_mmio *mmio = &tile->mmio; + unsigned long master_global_errstat, master_local_errstat; + u32 master, slave, regbit; + int i; + + if (xe->info.platform != XE_PVC) + return; + + master = SOC_PVC_MASTER_BASE; + slave = SOC_PVC_SLAVE_BASE; + + /* Mask error type in GSYSEVTCTL so that no new errors of the type will be reported */ + for (i = 0; i < XE_SOC_NUM_IEH; i++) + xe_mmio_write32(mmio, SOC_GSYSEVTCTL_REG(master, slave, i), ~REG_BIT(hw_err)); + + if (hw_err == HARDWARE_ERROR_CORRECTABLE) { + xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err), REG_GENMASK(31, 0)); + xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err), REG_GENMASK(31, 0)); + xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err), REG_GENMASK(31, 0)); + xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err), REG_GENMASK(31, 0)); + goto unmask_gsysevtctl; + } + + /* + * Read the master global IEH error register, if BIT(1) is set then process + * the slave IEH first. If BIT(0) in global error register is set then process + * the corresponding local error registers. + */ + master_global_errstat = xe_mmio_read32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err)); + if (master_global_errstat & SOC_SLAVE_IEH) + soc_slave_ieh_handler(tile, hw_err, error_id); + + if (master_global_errstat & SOC_IEH0_LOCAL_ERR_STATUS) { + master_local_errstat = xe_mmio_read32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err)); + + for_each_set_bit(regbit, &master_local_errstat, XE_RAS_REG_SIZE) + log_soc_error(tile, PVC_MASTER_LOCAL_REG_INFO(hw_err), severity, regbit, error_id); + + xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err), master_local_errstat); + } + + for_each_set_bit(regbit, &master_global_errstat, XE_RAS_REG_SIZE) + log_soc_error(tile, pvc_master_global_err_reg, severity, regbit, error_id); + + xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err), master_global_errstat); + +unmask_gsysevtctl: + for (i = 0; i < XE_SOC_NUM_IEH; i++) + xe_mmio_write32(mmio, SOC_GSYSEVTCTL_REG(master, slave, i), + (HARDWARE_ERROR_MAX << 1) + 1); +} + +static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_error hw_err) +{ + const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err); + const char *severity_str = error_severity[severity]; + struct xe_device *xe = tile_to_xe(tile); + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[severity]; + unsigned long flags, err_src; + u32 err_bit; + + if (!IS_DGFX(xe)) + return; + + /* + * Hardware errors are reported through System Controller on the platforms that + * support it, and never routed as direct IRQ to SGUnit. So we should never be + * here for those platforms. + */ + if (xe->info.has_sysctrl) { + drm_err_ratelimited(&xe->drm, HW_ERR "Invalid error routing\n"); + return; + } + + spin_lock_irqsave(&xe->irq.lock, flags); + err_src = xe_mmio_read32(&tile->mmio, DEV_ERR_STAT_REG(hw_err)); + if (!err_src) { + drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported %s DEV_ERR_STAT register blank!\n", + tile->id, severity_str); + goto unlock; + } + + /* + * On encountering CSC firmware errors, the graphics device becomes unrecoverable + * so return immediately on error. The only way to recover from these errors is + * firmware flash. The device will enter Runtime Survivability mode when such + * errors are detected. + */ + if (err_src & REG_BIT(XE_CSC_ERROR)) { + csc_hw_error_handler(tile, hw_err); + goto clear_reg; + } + + if (!info) + goto clear_reg; + + for_each_set_bit(err_bit, &err_src, XE_RAS_REG_SIZE) { + const char *name; + u32 error_id; + + error_id = err_src_to_id(err_bit); + if (!error_id) + continue; + + name = info[error_id].name; + if (!name) + continue; + + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) { + drm_warn(&xe->drm, HW_ERR + "TILE%d reported %s %s, bit[%d] is set\n", + tile->id, name, severity_str, err_bit); + } else { + drm_err_ratelimited(&xe->drm, HW_ERR + "TILE%d reported %s %s, bit[%d] is set\n", + tile->id, name, severity_str, err_bit); + } + + if (err_bit == XE_GT_ERROR) + gt_hw_error_handler(tile, hw_err, error_id); + if (err_bit == XE_SOC_ERROR) + soc_hw_error_handler(tile, hw_err, error_id); + } + +clear_reg: + xe_mmio_write32(&tile->mmio, DEV_ERR_STAT_REG(hw_err), err_src); +unlock: + spin_unlock_irqrestore(&xe->irq.lock, flags); +} + +/** + * xe_hw_error_irq_handler - irq handling for hw errors + * @tile: tile instance + * @master_ctl: value read from master interrupt register + * + * Xe platforms add three error bits to the master interrupt register to support error handling. + * These three bits are used to convey the class of error FATAL, NONFATAL, or CORRECTABLE. + * To process the interrupt, determine the source of error by reading the Device Error Source + * Register that corresponds to the class of error being serviced. + */ +void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl) +{ + enum hardware_error hw_err; + + if (xe_fault_csc_hw_error()) + schedule_work(&tile->csc_hw_error_work); + + for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++) { + if (master_ctl & ERROR_IRQ(hw_err)) + hw_error_source_handler(tile, hw_err); + } +} + +/* + * Process hardware errors during boot + */ +static void process_hw_errors(struct xe_device *xe) +{ + struct xe_tile *tile; + u32 master_ctl; + u8 id; + + for_each_tile(tile, xe, id) { + master_ctl = xe_mmio_read32(&tile->mmio, GFX_MSTR_IRQ); + xe_hw_error_irq_handler(tile, master_ctl); + xe_mmio_write32(&tile->mmio, GFX_MSTR_IRQ, master_ctl); + } +} + +/** + * xe_hw_error_init - Initialize hw errors + * @xe: xe device instance + * + * Initialize and check for errors that occurred during boot + * prior to driver load + */ +void xe_hw_error_init(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) + return; + + INIT_WORK(&tile->csc_hw_error_work, csc_hw_error_work); + + process_hw_errors(xe); +} diff --git a/drivers/gpu/drm/xe/xe_hw_error.h b/drivers/gpu/drm/xe/xe_hw_error.h new file mode 100644 index 000000000..5e3a11424 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_error.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ +#ifndef _XE_HW_ERROR_H_ +#define _XE_HW_ERROR_H_ + +#include + +struct xe_tile; +struct xe_device; + +void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl); +void xe_hw_error_init(struct xe_device *xe); +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_fence.c b/drivers/gpu/drm/xe/xe_hw_fence.c new file mode 100644 index 000000000..14720623a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_fence.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_hw_fence.h" + +#include +#include + +#include "xe_device_types.h" +#include "xe_hw_engine.h" +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_trace.h" + +static struct kmem_cache *xe_hw_fence_slab; + +int __init xe_hw_fence_module_init(void) +{ + xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", + sizeof(struct xe_hw_fence), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_hw_fence_slab) + return -ENOMEM; + + return 0; +} + +void xe_hw_fence_module_exit(void) +{ + rcu_barrier(); + kmem_cache_destroy(xe_hw_fence_slab); +} + +static struct xe_hw_fence *fence_alloc(void) +{ + return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); +} + +static void fence_free(struct rcu_head *rcu) +{ + struct xe_hw_fence *fence = + container_of(rcu, struct xe_hw_fence, dma.rcu); + + if (!WARN_ON_ONCE(!fence)) + kmem_cache_free(xe_hw_fence_slab, fence); +} + +static void hw_fence_irq_run_cb(struct irq_work *work) +{ + struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); + struct xe_hw_fence *fence, *next; + bool tmp; + + tmp = dma_fence_begin_signalling(); + spin_lock(&irq->lock); + if (irq->enabled) { + list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { + struct dma_fence *dma_fence = &fence->dma; + + trace_xe_hw_fence_try_signal(fence); + if (dma_fence_is_signaled_locked(dma_fence)) { + trace_xe_hw_fence_signal(fence); + list_del_init(&fence->irq_link); + dma_fence_put(dma_fence); + } + } + } + spin_unlock(&irq->lock); + dma_fence_end_signalling(tmp); +} + +void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) +{ + spin_lock_init(&irq->lock); + init_irq_work(&irq->work, hw_fence_irq_run_cb); + INIT_LIST_HEAD(&irq->pending); + irq->enabled = true; +} + +void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) +{ + struct xe_hw_fence *fence, *next; + unsigned long flags; + bool tmp; + + if (XE_WARN_ON(!list_empty(&irq->pending))) { + tmp = dma_fence_begin_signalling(); + spin_lock_irqsave(&irq->lock, flags); + list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { + list_del_init(&fence->irq_link); + XE_WARN_ON(dma_fence_check_and_signal_locked(&fence->dma)); + dma_fence_put(&fence->dma); + } + spin_unlock_irqrestore(&irq->lock, flags); + dma_fence_end_signalling(tmp); + } + + /* Safe release of the irq->lock used in dma_fence_init. */ + synchronize_rcu(); +} + +void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) +{ + irq_work_queue(&irq->work); +} + +void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, + struct xe_hw_fence_irq *irq, const char *name) +{ + ctx->gt = gt; + ctx->irq = irq; + ctx->dma_fence_ctx = dma_fence_context_alloc(1); + ctx->next_seqno = XE_FENCE_INITIAL_SEQNO; + snprintf(ctx->name, sizeof(ctx->name), "%s", name); +} + +void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) +{ +} + +static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); + +static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) +{ + return container_of(fence->dma.extern_lock, struct xe_hw_fence_irq, + lock); +} + +static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + + return dev_name(fence->xe->drm.dev); +} + +static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + + return fence->name; +} + +static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + struct xe_device *xe = fence->xe; + u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); + + return dma_fence->error || + !__dma_fence_is_later(dma_fence, dma_fence->seqno, seqno); +} + +static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); + + dma_fence_get(dma_fence); + list_add_tail(&fence->irq_link, &irq->pending); + + /* SW completed (no HW IRQ) so kick handler to signal fence */ + if (xe_hw_fence_signaled(dma_fence)) + xe_hw_fence_irq_run(irq); + + return true; +} + +static void xe_hw_fence_release(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + + XE_WARN_ON(!list_empty(&fence->irq_link)); + call_rcu(&dma_fence->rcu, fence_free); +} + +static const struct dma_fence_ops xe_hw_fence_ops = { + .get_driver_name = xe_hw_fence_get_driver_name, + .get_timeline_name = xe_hw_fence_get_timeline_name, + .enable_signaling = xe_hw_fence_enable_signaling, + .signaled = xe_hw_fence_signaled, + .release = xe_hw_fence_release, +}; + +static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) +{ + if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) + return NULL; + + return container_of(fence, struct xe_hw_fence, dma); +} + +/** + * xe_hw_fence_alloc() - Allocate an hw fence. + * + * Allocate but don't initialize an hw fence. + * + * Return: Pointer to the allocated fence or + * negative error pointer on error. + */ +struct dma_fence *xe_hw_fence_alloc(void) +{ + struct xe_hw_fence *hw_fence = fence_alloc(); + + if (!hw_fence) + return ERR_PTR(-ENOMEM); + + return &hw_fence->dma; +} + +/** + * xe_hw_fence_free() - Free an hw fence. + * @fence: Pointer to the fence to free. + * + * Frees an hw fence that hasn't yet been + * initialized. + */ +void xe_hw_fence_free(struct dma_fence *fence) +{ + fence_free(&fence->rcu); +} + +/** + * xe_hw_fence_init() - Initialize an hw fence. + * @fence: Pointer to the fence to initialize. + * @ctx: Pointer to the struct xe_hw_fence_ctx fence context. + * @seqno_map: Pointer to the map into where the seqno is blitted. + * + * Initializes a pre-allocated hw fence. + * After initialization, the fence is subject to normal + * dma-fence refcounting. + */ +void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx, + struct iosys_map seqno_map) +{ + struct xe_hw_fence *hw_fence = + container_of(fence, typeof(*hw_fence), dma); + + hw_fence->xe = gt_to_xe(ctx->gt); + snprintf(hw_fence->name, sizeof(hw_fence->name), "%s", ctx->name); + hw_fence->seqno_map = seqno_map; + INIT_LIST_HEAD(&hw_fence->irq_link); + + dma_fence_init(fence, &xe_hw_fence_ops, &ctx->irq->lock, + ctx->dma_fence_ctx, ctx->next_seqno++); + + trace_xe_hw_fence_create(hw_fence); +} diff --git a/drivers/gpu/drm/xe/xe_hw_fence.h b/drivers/gpu/drm/xe/xe_hw_fence.h new file mode 100644 index 000000000..599492c13 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_fence.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_HW_FENCE_H_ +#define _XE_HW_FENCE_H_ + +#include "xe_hw_fence_types.h" + +/* Cause an early wrap to catch wrapping errors */ +#define XE_FENCE_INITIAL_SEQNO (-127) + +int xe_hw_fence_module_init(void); +void xe_hw_fence_module_exit(void); + +void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq); +void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq); +void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq); + +void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, + struct xe_hw_fence_irq *irq, const char *name); +void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx); + +struct dma_fence *xe_hw_fence_alloc(void); + +void xe_hw_fence_free(struct dma_fence *fence); + +void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx, + struct iosys_map seqno_map); +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_fence_types.h b/drivers/gpu/drm/xe/xe_hw_fence_types.h new file mode 100644 index 000000000..58a8d09af --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_fence_types.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HW_FENCE_TYPES_H_ +#define _XE_HW_FENCE_TYPES_H_ + +#include +#include +#include +#include +#include + +struct xe_device; +struct xe_gt; + +/** + * struct xe_hw_fence_irq - hardware fence IRQ handler + * + * One per engine class, signals completed xe_hw_fences, triggered via hw engine + * interrupt. On each trigger, search list of pending fences and signal. + */ +struct xe_hw_fence_irq { + /** @lock: protects all xe_hw_fences + pending list */ + spinlock_t lock; + /** @work: IRQ worker run to signal the fences */ + struct irq_work work; + /** @pending: list of pending xe_hw_fences */ + struct list_head pending; + /** @enabled: fence signaling enabled */ + bool enabled; +}; + +#define MAX_FENCE_NAME_LEN 16 + +/** + * struct xe_hw_fence_ctx - hardware fence context + * + * The context for a hardware fence. 1 to 1 relationship with xe_engine. Points + * to a xe_hw_fence_irq, maintains serial seqno. + */ +struct xe_hw_fence_ctx { + /** @gt: GT structure of hardware fence context */ + struct xe_gt *gt; + /** @irq: fence irq handler */ + struct xe_hw_fence_irq *irq; + /** @dma_fence_ctx: dma fence context for hardware fence */ + u64 dma_fence_ctx; + /** @next_seqno: next seqno for hardware fence */ + u32 next_seqno; + /** @name: name of hardware fence context */ + char name[MAX_FENCE_NAME_LEN]; +}; + +/** + * struct xe_hw_fence - hardware fence + * + * Used to indicate a xe_sched_job is complete via a seqno written to memory. + * Signals on error or seqno past. + */ +struct xe_hw_fence { + /** @dma: base dma fence for hardware fence context */ + struct dma_fence dma; + /** @xe: Xe device for hw fence driver name */ + struct xe_device *xe; + /** @name: name of hardware fence context */ + char name[MAX_FENCE_NAME_LEN]; + /** @seqno_map: I/O map for seqno */ + struct iosys_map seqno_map; + /** @irq_link: Link in struct xe_hw_fence_irq.pending */ + struct list_head irq_link; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c new file mode 100644 index 000000000..de3f2aeff --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -0,0 +1,1585 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include "regs/xe_gt_regs.h" +#include "regs/xe_mchbar_regs.h" +#include "regs/xe_pcode_regs.h" +#include "xe_device.h" +#include "xe_hwmon.h" +#include "xe_mmio.h" +#include "xe_pcode.h" +#include "xe_pcode_api.h" +#include "xe_sriov.h" +#include "xe_pm.h" +#include "xe_vsec.h" +#include "regs/xe_pmt.h" + +enum xe_hwmon_reg { + REG_TEMP, + REG_PKG_RAPL_LIMIT, + REG_PKG_POWER_SKU, + REG_PKG_POWER_SKU_UNIT, + REG_GT_PERF_STATUS, + REG_PKG_ENERGY_STATUS, + REG_FAN_SPEED, +}; + +enum xe_hwmon_reg_operation { + REG_READ32, + REG_RMW32, + REG_READ64, +}; + +#define MAX_VRAM_CHANNELS (16) + +enum xe_hwmon_channel { + CHANNEL_CARD, + CHANNEL_PKG, + CHANNEL_VRAM, + CHANNEL_MCTRL, + CHANNEL_PCIE, + CHANNEL_VRAM_N, + CHANNEL_VRAM_N_MAX = CHANNEL_VRAM_N + MAX_VRAM_CHANNELS - 1, + CHANNEL_MAX, +}; + +enum xe_fan_channel { + FAN_1, + FAN_2, + FAN_3, + FAN_MAX, +}; + +enum xe_temp_limit { + TEMP_LIMIT_PKG_SHUTDOWN, + TEMP_LIMIT_PKG_CRIT, + TEMP_LIMIT_MEM_SHUTDOWN, + TEMP_LIMIT_PKG_MAX, + TEMP_LIMIT_MEM_CRIT, + TEMP_LIMIT_MAX +}; + +/* Attribute index for powerX_xxx_interval sysfs entries */ +enum sensor_attr_power { + SENSOR_INDEX_PSYS_PL1, + SENSOR_INDEX_PKG_PL1, + SENSOR_INDEX_PSYS_PL2, + SENSOR_INDEX_PKG_PL2, +}; + +/* + * For platforms that support mailbox commands for power limits, REG_PKG_POWER_SKU_UNIT is + * not supported and below are SKU units to be used. + */ +#define PWR_UNIT 0x3 +#define ENERGY_UNIT 0xe +#define TIME_UNIT 0xa + +/* + * SF_* - scale factors for particular quantities according to hwmon spec. + */ +#define SF_POWER 1000000 /* microwatts */ +#define SF_CURR 1000 /* milliamperes */ +#define SF_VOLTAGE 1000 /* millivolts */ +#define SF_ENERGY 1000000 /* microjoules */ +#define SF_TIME 1000 /* milliseconds */ + +/* + * PL*_HWMON_ATTR - mapping of hardware power limits to corresponding hwmon power attribute. + */ +#define PL1_HWMON_ATTR hwmon_power_max +#define PL2_HWMON_ATTR hwmon_power_cap + +#define PWR_ATTR_TO_STR(attr) (((attr) == hwmon_power_max) ? "PL1" : "PL2") + +/* + * Timeout for power limit write mailbox command. + */ +#define PL_WRITE_MBX_TIMEOUT_MS (1) + +/* Index of memory controller in READ_THERMAL_DATA output */ +#define TEMP_INDEX_MCTRL 2 + +/* Maximum characters in hwmon label name */ +#define MAX_LABEL_SIZE 16 + +/** + * struct xe_hwmon_energy_info - to accumulate energy + */ +struct xe_hwmon_energy_info { + /** @reg_val_prev: previous energy reg val */ + u32 reg_val_prev; + /** @accum_energy: accumulated energy */ + long accum_energy; +}; + +/** + * struct xe_hwmon_fan_info - to cache previous fan reading + */ +struct xe_hwmon_fan_info { + /** @reg_val_prev: previous fan reg val */ + u32 reg_val_prev; + /** @time_prev: previous timestamp */ + u64 time_prev; +}; + +/** + * struct xe_hwmon_thermal_info - to store temperature data + */ +struct xe_hwmon_thermal_info { + union { + /** @limit: temperatures limits */ + u8 limit[TEMP_LIMIT_MAX]; + /** @data: temperature limits in dwords */ + u32 data[DIV_ROUND_UP(TEMP_LIMIT_MAX, sizeof(u32))]; + }; + /** @count: no of temperature sensors available for the platform */ + u8 count; + /** @value: signed value from each sensor */ + s8 value[U8_MAX]; + /** @vram_label: vram label names */ + char vram_label[MAX_VRAM_CHANNELS][MAX_LABEL_SIZE]; +}; + +/** + * struct xe_hwmon - xe hwmon data structure + */ +struct xe_hwmon { + /** @hwmon_dev: hwmon device for xe */ + struct device *hwmon_dev; + /** @xe: Xe device */ + struct xe_device *xe; + /** @hwmon_lock: lock for rw attributes*/ + struct mutex hwmon_lock; + /** @scl_shift_power: pkg power unit */ + int scl_shift_power; + /** @scl_shift_energy: pkg energy unit */ + int scl_shift_energy; + /** @scl_shift_time: pkg time unit */ + int scl_shift_time; + /** @ei: Energy info for energyN_input */ + struct xe_hwmon_energy_info ei[CHANNEL_MAX]; + /** @fi: Fan info for fanN_input */ + struct xe_hwmon_fan_info fi[FAN_MAX]; + /** @boot_power_limit_read: is boot power limits read */ + bool boot_power_limit_read; + /** @pl1_on_boot: power limit PL1 on boot */ + u32 pl1_on_boot[CHANNEL_MAX]; + /** @pl2_on_boot: power limit PL2 on boot */ + u32 pl2_on_boot[CHANNEL_MAX]; + /** @temp: Temperature info */ + struct xe_hwmon_thermal_info temp; +}; + +static inline int prepare_power_limit_param2(const struct xe_hwmon *hwmon) +{ + if (hwmon->boot_power_limit_read) { + if (hwmon->xe->info.platform >= XE_CRESCENTISLAND) + return READ_PL_ACCEPTED; + else + return READ_PL_FROM_PCODE; + } else { + return READ_PL_FROM_FW; + } +} + +static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 attr, int channel, + u32 *uval) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + u32 val0 = 0, val1 = 0; + int ret = 0; + + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, + (channel == CHANNEL_CARD) ? + READ_PSYSGPU_POWER_LIMIT : + READ_PACKAGE_POWER_LIMIT, + prepare_power_limit_param2(hwmon)), &val0, &val1); + + if (ret) { + drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", + channel, val0, val1, ret); + *uval = 0; + return ret; + } + + /* return the value only if limit is enabled */ + if (attr == PL1_HWMON_ATTR) + *uval = (val0 & PWR_LIM_EN) ? val0 : 0; + else if (attr == PL2_HWMON_ATTR) + *uval = (val1 & PWR_LIM_EN) ? val1 : 0; + else if (attr == hwmon_power_label) + *uval = (val0 & PWR_LIM_EN) ? 1 : (val1 & PWR_LIM_EN) ? 1 : 0; + else + *uval = 0; + + return ret; +} + +static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr, u8 channel, + u32 clr, u32 set) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + u32 val0 = 0, val1 = 0; + int ret = 0; + + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, + (channel == CHANNEL_CARD) ? + READ_PSYSGPU_POWER_LIMIT : + READ_PACKAGE_POWER_LIMIT, + prepare_power_limit_param2(hwmon)), &val0, &val1); + if (ret) + drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", + channel, val0, val1, ret); + + if (attr == PL1_HWMON_ATTR) + val0 = (val0 & ~clr) | set; + else if (attr == PL2_HWMON_ATTR) + val1 = (val1 & ~clr) | set; + else + return -EIO; + + ret = xe_pcode_write64_timeout(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, + (channel == CHANNEL_CARD) ? + WRITE_PSYSGPU_POWER_LIMIT : + WRITE_PACKAGE_POWER_LIMIT, 0), + val0, val1, PL_WRITE_MBX_TIMEOUT_MS); + if (ret) + drm_dbg(&hwmon->xe->drm, "write failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", + channel, val0, val1, ret); + return ret; +} + +static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg, + int channel) +{ + struct xe_device *xe = hwmon->xe; + + switch (hwmon_reg) { + case REG_TEMP: + if (xe->info.platform == XE_BATTLEMAGE) { + if (channel == CHANNEL_PKG) + return BMG_PACKAGE_TEMPERATURE; + else if (channel == CHANNEL_VRAM) + return BMG_VRAM_TEMPERATURE; + else if (in_range(channel, CHANNEL_VRAM_N, MAX_VRAM_CHANNELS)) + return BMG_VRAM_TEMPERATURE_N(channel - CHANNEL_VRAM_N); + } else if (xe->info.platform == XE_DG2) { + if (channel == CHANNEL_PKG) + return PCU_CR_PACKAGE_TEMPERATURE; + else if (channel == CHANNEL_VRAM) + return BMG_VRAM_TEMPERATURE; + } + break; + case REG_PKG_RAPL_LIMIT: + if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) + return PVC_GT0_PACKAGE_RAPL_LIMIT; + else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG)) + return PCU_CR_PACKAGE_RAPL_LIMIT; + break; + case REG_PKG_POWER_SKU: + if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) + return PVC_GT0_PACKAGE_POWER_SKU; + else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG)) + return PCU_CR_PACKAGE_POWER_SKU; + break; + case REG_PKG_POWER_SKU_UNIT: + if (xe->info.platform == XE_PVC) + return PVC_GT0_PACKAGE_POWER_SKU_UNIT; + else if (xe->info.platform == XE_DG2) + return PCU_CR_PACKAGE_POWER_SKU_UNIT; + break; + case REG_GT_PERF_STATUS: + if (xe->info.platform == XE_DG2 && channel == CHANNEL_PKG) + return GT_PERF_STATUS; + break; + case REG_PKG_ENERGY_STATUS: + if (xe->info.platform == XE_CRESCENTISLAND) { + if (channel == CHANNEL_CARD) + return CRI_PLATFORM_ENERGY_STATUS; + else if (channel == CHANNEL_PKG) + return CRI_PACKAGE_ENERGY_STATUS; + } else if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) { + return PVC_GT0_PLATFORM_ENERGY_STATUS; + } else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG)) { + return PCU_CR_PACKAGE_ENERGY_STATUS; + } + break; + case REG_FAN_SPEED: + if (channel == FAN_1) + return BMG_FAN_1_SPEED; + else if (channel == FAN_2) + return BMG_FAN_2_SPEED; + else if (channel == FAN_3) + return BMG_FAN_3_SPEED; + break; + default: + drm_warn(&xe->drm, "Unknown xe hwmon reg id: %d\n", hwmon_reg); + break; + } + + return XE_REG(0); +} + +#define PL_DISABLE 0 + +/* + * HW allows arbitrary PL1 limits to be set but silently clamps these values to + * "typical but not guaranteed" min/max values in REG_PKG_POWER_SKU. Follow the + * same pattern for sysfs, allow arbitrary PL1 limits to be set but display + * clamped values when read. + */ +static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *value) +{ + u32 reg_val = 0; + struct xe_device *xe = hwmon->xe; + struct xe_reg rapl_limit, pkg_power_sku; + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + mutex_lock(&hwmon->hwmon_lock); + + if (hwmon->xe->info.has_mbx_power_limits) { + xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, ®_val); + } else { + rapl_limit = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); + pkg_power_sku = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); + reg_val = xe_mmio_read32(mmio, rapl_limit); + } + + /* Check if PL limits are disabled. */ + if (!(reg_val & PWR_LIM_EN)) { + *value = PL_DISABLE; + drm_info(&hwmon->xe->drm, "%s disabled for channel %d, val 0x%08x\n", + PWR_ATTR_TO_STR(attr), channel, reg_val); + goto unlock; + } + + reg_val = REG_FIELD_GET(PWR_LIM_VAL, reg_val); + *value = mul_u32_u32(reg_val, SF_POWER) >> hwmon->scl_shift_power; + + /* For platforms with mailbox power limit support clamping would be done by pcode. */ + if (!hwmon->xe->info.has_mbx_power_limits) { + u64 pkg_pwr, min, max; + + pkg_pwr = xe_mmio_read64_2x32(mmio, pkg_power_sku); + min = REG_FIELD_GET(PKG_MIN_PWR, pkg_pwr); + max = REG_FIELD_GET(PKG_MAX_PWR, pkg_pwr); + min = mul_u64_u32_shr(min, SF_POWER, hwmon->scl_shift_power); + max = mul_u64_u32_shr(max, SF_POWER, hwmon->scl_shift_power); + if (min && max) + *value = clamp_t(u64, *value, min, max); + } +unlock: + mutex_unlock(&hwmon->hwmon_lock); +} + +static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, u32 attr, int channel, long value) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + int ret = 0; + u32 reg_val, max; + struct xe_reg rapl_limit; + u64 max_supp_power_limit = 0; + + mutex_lock(&hwmon->hwmon_lock); + + rapl_limit = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); + + /* Disable Power Limit and verify, as limit cannot be disabled on all platforms. */ + if (value == PL_DISABLE) { + if (hwmon->xe->info.has_mbx_power_limits) { + drm_dbg(&hwmon->xe->drm, "disabling %s on channel %d\n", + PWR_ATTR_TO_STR(attr), channel); + xe_hwmon_pcode_rmw_power_limit(hwmon, attr, channel, PWR_LIM_EN, 0); + xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, ®_val); + } else { + reg_val = xe_mmio_rmw32(mmio, rapl_limit, PWR_LIM_EN, 0); + reg_val = xe_mmio_read32(mmio, rapl_limit); + } + + if (reg_val & PWR_LIM_EN) { + drm_warn(&hwmon->xe->drm, "Power limit disable is not supported!\n"); + ret = -EOPNOTSUPP; + } + goto unlock; + } + + /* + * If the sysfs value exceeds the maximum pcode supported power limit value, clamp it to + * the supported maximum (U12.3 format). + * This is to avoid truncation during reg_val calculation below and ensure the valid + * power limit is sent for pcode which would clamp it to card-supported value. + */ + max_supp_power_limit = ((PWR_LIM_VAL) >> hwmon->scl_shift_power) * SF_POWER; + if (value > max_supp_power_limit) { + value = max_supp_power_limit; + drm_info(&hwmon->xe->drm, + "Power limit clamped as selected %s exceeds channel %d limit\n", + PWR_ATTR_TO_STR(attr), channel); + } + + /* Computation in 64-bits to avoid overflow. Round to nearest. */ + reg_val = DIV_ROUND_CLOSEST_ULL((u64)value << hwmon->scl_shift_power, SF_POWER); + + /* + * Clamp power limit to GPU firmware default as maximum, as an additional protection to + * pcode clamp. + */ + if (hwmon->xe->info.has_mbx_power_limits) { + max = (attr == PL1_HWMON_ATTR) ? + hwmon->pl1_on_boot[channel] : hwmon->pl2_on_boot[channel]; + max = REG_FIELD_PREP(PWR_LIM_VAL, max); + if (reg_val > max) { + reg_val = max; + drm_dbg(&hwmon->xe->drm, + "Clamping power limit to GPU firmware default 0x%x\n", + reg_val); + } + } + + reg_val = PWR_LIM_EN | REG_FIELD_PREP(PWR_LIM_VAL, reg_val); + + if (hwmon->xe->info.has_mbx_power_limits) + ret = xe_hwmon_pcode_rmw_power_limit(hwmon, attr, channel, PWR_LIM, reg_val); + else + reg_val = xe_mmio_rmw32(mmio, rapl_limit, PWR_LIM, reg_val); +unlock: + mutex_unlock(&hwmon->hwmon_lock); + return ret; +} + +static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, u32 attr, int channel, + long *value) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + u32 reg_val; + + if (hwmon->xe->info.has_mbx_power_limits) { + /* PL1 is rated max if supported. */ + xe_hwmon_pcode_read_power_limit(hwmon, PL1_HWMON_ATTR, channel, ®_val); + } else { + /* + * This sysfs file won't be visible if REG_PKG_POWER_SKU is invalid, so valid check + * for this register can be skipped. + * See xe_hwmon_power_is_visible. + */ + struct xe_reg reg = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); + + reg_val = xe_mmio_read32(mmio, reg); + } + + reg_val = REG_FIELD_GET(PKG_TDP, reg_val); + *value = mul_u64_u32_shr(reg_val, SF_POWER, hwmon->scl_shift_power); +} + +/* + * xe_hwmon_energy_get - Obtain energy value + * + * The underlying energy hardware register is 32-bits and is subject to + * overflow. How long before overflow? For example, with an example + * scaling bit shift of 14 bits (see register *PACKAGE_POWER_SKU_UNIT) and + * a power draw of 1000 watts, the 32-bit counter will overflow in + * approximately 4.36 minutes. + * + * Examples: + * 1 watt: (2^32 >> 14) / 1 W / (60 * 60 * 24) secs/day -> 3 days + * 1000 watts: (2^32 >> 14) / 1000 W / 60 secs/min -> 4.36 minutes + * + * The function significantly increases overflow duration (from 4.36 + * minutes) by accumulating the energy register into a 'long' as allowed by + * the hwmon API. Using x86_64 128 bit arithmetic (see mul_u64_u32_shr()), + * a 'long' of 63 bits, SF_ENERGY of 1e6 (~20 bits) and + * hwmon->scl_shift_energy of 14 bits we have 57 (63 - 20 + 14) bits before + * energyN_input overflows. This at 1000 W is an overflow duration of 278 years. + */ +static void +xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + struct xe_hwmon_energy_info *ei = &hwmon->ei[channel]; + u32 reg_val; + int ret = 0; + + /* Energy is supported only for card and pkg */ + if (channel > CHANNEL_PKG) { + *energy = 0; + return; + } + + if (hwmon->xe->info.platform == XE_BATTLEMAGE) { + u64 pmt_val; + + ret = xe_pmt_telem_read(hwmon->xe->drm.dev, + xe_mmio_read32(mmio, PUNIT_TELEMETRY_GUID), + &pmt_val, BMG_ENERGY_STATUS_PMT_OFFSET, sizeof(pmt_val)); + if (ret != sizeof(pmt_val)) { + drm_warn(&hwmon->xe->drm, "energy read from pmt failed, ret %d\n", ret); + *energy = 0; + return; + } + + if (channel == CHANNEL_PKG) + reg_val = REG_FIELD_GET64(ENERGY_PKG, pmt_val); + else + reg_val = REG_FIELD_GET64(ENERGY_CARD, pmt_val); + } else { + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_ENERGY_STATUS, + channel)); + } + + ei->accum_energy += reg_val - ei->reg_val_prev; + ei->reg_val_prev = reg_val; + + *energy = mul_u64_u32_shr(ei->accum_energy, SF_ENERGY, + hwmon->scl_shift_energy); +} + +static ssize_t +xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct xe_hwmon *hwmon = dev_get_drvdata(dev); + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + u32 reg_val, x, y, x_w = 2; /* 2 bits */ + u64 tau4, out; + int channel = (to_sensor_dev_attr(attr)->index % 2) ? CHANNEL_PKG : CHANNEL_CARD; + u32 power_attr = (to_sensor_dev_attr(attr)->index > 1) ? PL2_HWMON_ATTR : PL1_HWMON_ATTR; + + int ret = 0; + + guard(xe_pm_runtime)(hwmon->xe); + + mutex_lock(&hwmon->hwmon_lock); + + if (hwmon->xe->info.has_mbx_power_limits) { + ret = xe_hwmon_pcode_read_power_limit(hwmon, power_attr, channel, ®_val); + if (ret) { + drm_err(&hwmon->xe->drm, + "power interval read fail, ch %d, attr %d, val 0x%08x, ret %d\n", + channel, power_attr, reg_val, ret); + reg_val = 0; + } + } else { + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, + channel)); + } + + mutex_unlock(&hwmon->hwmon_lock); + + x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val); + y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val); + + /* + * tau = (1 + (x / 4)) * power(2,y), x = bits(23:22), y = bits(21:17) + * = (4 | x) << (y - 2) + * + * Here (y - 2) ensures a 1.x fixed point representation of 1.x + * As x is 2 bits so 1.x can be 1.0, 1.25, 1.50, 1.75 + * + * As y can be < 2, we compute tau4 = (4 | x) << y + * and then add 2 when doing the final right shift to account for units + */ + tau4 = (u64)((1 << x_w) | x) << y; + + /* val in hwmon interface units (millisec) */ + out = mul_u64_u32_shr(tau4, SF_TIME, hwmon->scl_shift_time + x_w); + + return sysfs_emit(buf, "%llu\n", out); +} + +static ssize_t +xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct xe_hwmon *hwmon = dev_get_drvdata(dev); + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + u32 x, y, rxy, x_w = 2; /* 2 bits */ + u64 tau4, r, max_win; + unsigned long val; + int channel = (to_sensor_dev_attr(attr)->index % 2) ? CHANNEL_PKG : CHANNEL_CARD; + u32 power_attr = (to_sensor_dev_attr(attr)->index > 1) ? PL2_HWMON_ATTR : PL1_HWMON_ATTR; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret) + return ret; + + /* + * Max HW supported tau in '(1 + (x / 4)) * power(2,y)' format, x = 0, y = 0x12. + * The hwmon->scl_shift_time default of 0xa results in a max tau of 256 seconds. + * + * The ideal scenario is for PKG_MAX_WIN to be read from the PKG_PWR_SKU register. + * However, it is observed that existing discrete GPUs does not provide correct + * PKG_MAX_WIN value, therefore a using default constant value. For future discrete GPUs + * this may get resolved, in which case PKG_MAX_WIN should be obtained from PKG_PWR_SKU. + */ +#define PKG_MAX_WIN_DEFAULT 0x12ull + + /* + * val must be < max in hwmon interface units. The steps below are + * explained in xe_hwmon_power_max_interval_show() + */ + r = FIELD_PREP(PKG_MAX_WIN, PKG_MAX_WIN_DEFAULT); + x = REG_FIELD_GET(PKG_MAX_WIN_X, r); + y = REG_FIELD_GET(PKG_MAX_WIN_Y, r); + tau4 = (u64)((1 << x_w) | x) << y; + max_win = mul_u64_u32_shr(tau4, SF_TIME, hwmon->scl_shift_time + x_w); + + if (val > max_win) + return -EINVAL; + + /* val in hw units */ + val = DIV_ROUND_CLOSEST_ULL((u64)val << hwmon->scl_shift_time, SF_TIME) + 1; + + /* + * Convert val to 1.x * power(2,y) + * y = ilog2(val) + * x = (val - (1 << y)) >> (y - 2) + */ + if (!val) { + y = 0; + x = 0; + } else { + y = ilog2(val); + x = (val - (1ul << y)) << x_w >> y; + } + + rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) | + REG_FIELD_PREP(PWR_LIM_TIME_Y, y); + + guard(xe_pm_runtime)(hwmon->xe); + + mutex_lock(&hwmon->hwmon_lock); + + if (hwmon->xe->info.has_mbx_power_limits) + xe_hwmon_pcode_rmw_power_limit(hwmon, power_attr, channel, PWR_LIM_TIME, rxy); + else + r = xe_mmio_rmw32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel), + PWR_LIM_TIME, rxy); + + mutex_unlock(&hwmon->hwmon_lock); + + return count; +} + +/* PSYS PL1 */ +static SENSOR_DEVICE_ATTR(power1_max_interval, 0664, + xe_hwmon_power_max_interval_show, + xe_hwmon_power_max_interval_store, SENSOR_INDEX_PSYS_PL1); +/* PKG PL1 */ +static SENSOR_DEVICE_ATTR(power2_max_interval, 0664, + xe_hwmon_power_max_interval_show, + xe_hwmon_power_max_interval_store, SENSOR_INDEX_PKG_PL1); +/* PSYS PL2 */ +static SENSOR_DEVICE_ATTR(power1_cap_interval, 0664, + xe_hwmon_power_max_interval_show, + xe_hwmon_power_max_interval_store, SENSOR_INDEX_PSYS_PL2); +/* PKG PL2 */ +static SENSOR_DEVICE_ATTR(power2_cap_interval, 0664, + xe_hwmon_power_max_interval_show, + xe_hwmon_power_max_interval_store, SENSOR_INDEX_PKG_PL2); + +static struct attribute *hwmon_attributes[] = { + &sensor_dev_attr_power1_max_interval.dev_attr.attr, + &sensor_dev_attr_power2_max_interval.dev_attr.attr, + &sensor_dev_attr_power1_cap_interval.dev_attr.attr, + &sensor_dev_attr_power2_cap_interval.dev_attr.attr, + NULL +}; + +static umode_t xe_hwmon_attributes_visible(struct kobject *kobj, + struct attribute *attr, int index) +{ + struct device *dev = kobj_to_dev(kobj); + struct xe_hwmon *hwmon = dev_get_drvdata(dev); + int ret = 0; + int channel = (index % 2) ? CHANNEL_PKG : CHANNEL_CARD; + u32 power_attr = (index > 1) ? PL2_HWMON_ATTR : PL1_HWMON_ATTR; + u32 uval = 0; + struct xe_reg rapl_limit; + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + + if (hwmon->xe->info.has_mbx_power_limits) { + xe_hwmon_pcode_read_power_limit(hwmon, power_attr, channel, &uval); + } else if (power_attr != PL2_HWMON_ATTR) { + rapl_limit = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); + if (xe_reg_is_valid(rapl_limit)) + uval = xe_mmio_read32(mmio, rapl_limit); + } + ret = (uval & PWR_LIM_EN) ? attr->mode : 0; + + return ret; +} + +static const struct attribute_group hwmon_attrgroup = { + .attrs = hwmon_attributes, + .is_visible = xe_hwmon_attributes_visible, +}; + +static const struct attribute_group *hwmon_groups[] = { + &hwmon_attrgroup, + NULL +}; + +static const struct hwmon_channel_info * const hwmon_info[] = { + HWMON_CHANNEL_INFO(temp, + HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL | + HWMON_T_MAX, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL), + HWMON_CHANNEL_INFO(power, HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_LABEL | HWMON_P_CRIT | + HWMON_P_CAP, + HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_LABEL | HWMON_P_CAP), + HWMON_CHANNEL_INFO(curr, HWMON_C_LABEL, HWMON_C_CRIT | HWMON_C_LABEL), + HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL), + HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT | HWMON_E_LABEL, HWMON_E_INPUT | HWMON_E_LABEL), + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT, HWMON_F_INPUT, HWMON_F_INPUT), + NULL +}; + +static int xe_hwmon_pcode_read_thermal_info(struct xe_hwmon *hwmon) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + u32 config = 0; + int ret; + + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_LIMITS, 0), + &hwmon->temp.data[0], &hwmon->temp.data[1]); + if (ret) + return ret; + + drm_dbg(&hwmon->xe->drm, "thermal info read val 0x%x val1 0x%x\n", + hwmon->temp.data[0], hwmon->temp.data[1]); + + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_CONFIG, 0), + &config, NULL); + if (ret) + return ret; + + drm_dbg(&hwmon->xe->drm, "thermal config count 0x%x\n", config); + hwmon->temp.count = REG_FIELD_GET(TEMP_MASK, config); + + return ret; +} + +static int get_mc_temp(struct xe_hwmon *hwmon, long *val) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + u32 *dword = (u32 *)hwmon->temp.value; + s32 average = 0; + int ret, i; + + for (i = 0; i < DIV_ROUND_UP(TEMP_LIMIT_MAX, sizeof(u32)); i++) { + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_DATA, i), + (dword + i), NULL); + if (ret) + return ret; + drm_dbg(&hwmon->xe->drm, "thermal data for group %d val 0x%x\n", i, dword[i]); + } + + for (i = TEMP_INDEX_MCTRL; i < hwmon->temp.count - 1; i++) + average += hwmon->temp.value[i]; + + average /= (hwmon->temp.count - TEMP_INDEX_MCTRL - 1); + *val = average * MILLIDEGREE_PER_DEGREE; + return 0; +} + +static int get_pcie_temp(struct xe_hwmon *hwmon, long *val) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + u32 data = 0; + int ret; + + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_DATA, + PCIE_SENSOR_GROUP_ID), &data, NULL); + if (ret) + return ret; + + /* Sensor offset is different for G21 */ + if (hwmon->xe->info.subplatform != XE_SUBPLATFORM_BATTLEMAGE_G21) + data = REG_FIELD_GET(PCIE_SENSOR_MASK, data); + + data = REG_FIELD_GET(TEMP_MASK, data); + *val = (s8)data * MILLIDEGREE_PER_DEGREE; + + return 0; +} + +/* I1 is exposed as power_crit or as curr_crit depending on bit 31 */ +static int xe_hwmon_pcode_read_i1(const struct xe_hwmon *hwmon, u32 *uval) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + + /* Avoid Illegal Subcommand error */ + if (hwmon->xe->info.platform == XE_DG2) + return -ENXIO; + + return xe_pcode_read(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, + POWER_SETUP_SUBCOMMAND_READ_I1, 0), + uval, NULL); +} + +static int xe_hwmon_pcode_write_i1(const struct xe_hwmon *hwmon, u32 uval) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + + return xe_pcode_write(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, + POWER_SETUP_SUBCOMMAND_WRITE_I1, 0), + (uval & POWER_SETUP_I1_DATA_MASK)); +} + +static int xe_hwmon_pcode_read_fan_control(const struct xe_hwmon *hwmon, u32 subcmd, u32 *uval) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + + /* Platforms that don't return correct value */ + if (hwmon->xe->info.platform == XE_DG2 && subcmd == FSC_READ_NUM_FANS) { + *uval = 2; + return 0; + } + + return xe_pcode_read(root_tile, PCODE_MBOX(FAN_SPEED_CONTROL, subcmd, 0), uval, NULL); +} + +static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, int channel, + long *value, u32 scale_factor) +{ + int ret; + u32 uval = 0; + + mutex_lock(&hwmon->hwmon_lock); + + ret = xe_hwmon_pcode_read_i1(hwmon, &uval); + if (ret) + goto unlock; + + *value = mul_u64_u32_shr(REG_FIELD_GET(POWER_SETUP_I1_DATA_MASK, uval), + scale_factor, POWER_SETUP_I1_SHIFT); +unlock: + mutex_unlock(&hwmon->hwmon_lock); + return ret; +} + +static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, int channel, + long value, u32 scale_factor) +{ + int ret; + u32 uval; + u64 max_crit_power_curr = 0; + + mutex_lock(&hwmon->hwmon_lock); + + /* + * If the sysfs value exceeds the pcode mailbox cmd POWER_SETUP_SUBCOMMAND_WRITE_I1 + * max supported value, clamp it to the command's max (U10.6 format). + * This is to avoid truncation during uval calculation below and ensure the valid power + * limit is sent for pcode which would clamp it to card-supported value. + */ + max_crit_power_curr = (POWER_SETUP_I1_DATA_MASK >> POWER_SETUP_I1_SHIFT) * scale_factor; + if (value > max_crit_power_curr) { + value = max_crit_power_curr; + drm_info(&hwmon->xe->drm, + "Power limit clamped as selected exceeds channel %d limit\n", + channel); + } + uval = DIV_ROUND_CLOSEST_ULL(value << POWER_SETUP_I1_SHIFT, scale_factor); + ret = xe_hwmon_pcode_write_i1(hwmon, uval); + + mutex_unlock(&hwmon->hwmon_lock); + return ret; +} + +static void xe_hwmon_get_voltage(struct xe_hwmon *hwmon, int channel, long *value) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + u64 reg_val; + + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_GT_PERF_STATUS, channel)); + /* HW register value in units of 2.5 millivolt */ + *value = DIV_ROUND_CLOSEST(REG_FIELD_GET(VOLTAGE_MASK, reg_val) * 2500, SF_VOLTAGE); +} + +static inline bool is_vram_ch_available(struct xe_hwmon *hwmon, int channel) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + int vram_id = channel - CHANNEL_VRAM_N; + struct xe_reg vram_reg; + + vram_reg = xe_hwmon_get_reg(hwmon, REG_TEMP, channel); + if (!xe_reg_is_valid(vram_reg) || !xe_mmio_read32(mmio, vram_reg)) + return false; + + /* Create label only for available vram channel */ + sprintf(hwmon->temp.vram_label[vram_id], "vram_ch_%d", vram_id); + return true; +} + +static umode_t +xe_hwmon_temp_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) +{ + switch (attr) { + case hwmon_temp_emergency: + switch (channel) { + case CHANNEL_PKG: + return hwmon->temp.limit[TEMP_LIMIT_PKG_SHUTDOWN] ? 0444 : 0; + case CHANNEL_VRAM: + return hwmon->temp.limit[TEMP_LIMIT_MEM_SHUTDOWN] ? 0444 : 0; + case CHANNEL_MCTRL: + case CHANNEL_PCIE: + return hwmon->temp.count ? 0444 : 0; + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + return (is_vram_ch_available(hwmon, channel) && + hwmon->temp.limit[TEMP_LIMIT_MEM_SHUTDOWN]) ? 0444 : 0; + default: + return 0; + } + case hwmon_temp_crit: + switch (channel) { + case CHANNEL_PKG: + return hwmon->temp.limit[TEMP_LIMIT_PKG_CRIT] ? 0444 : 0; + case CHANNEL_VRAM: + return hwmon->temp.limit[TEMP_LIMIT_MEM_CRIT] ? 0444 : 0; + case CHANNEL_MCTRL: + case CHANNEL_PCIE: + return hwmon->temp.count ? 0444 : 0; + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + return (is_vram_ch_available(hwmon, channel) && + hwmon->temp.limit[TEMP_LIMIT_MEM_CRIT]) ? 0444 : 0; + default: + return 0; + } + case hwmon_temp_max: + switch (channel) { + case CHANNEL_PKG: + return hwmon->temp.limit[TEMP_LIMIT_PKG_MAX] ? 0444 : 0; + default: + return 0; + } + case hwmon_temp_input: + case hwmon_temp_label: + switch (channel) { + case CHANNEL_PKG: + case CHANNEL_VRAM: + return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_TEMP, + channel)) ? 0444 : 0; + case CHANNEL_MCTRL: + case CHANNEL_PCIE: + return hwmon->temp.count ? 0444 : 0; + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + return is_vram_ch_available(hwmon, channel) ? 0444 : 0; + default: + return 0; + } + default: + return 0; + } +} + +static int +xe_hwmon_temp_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + u64 reg_val; + + switch (attr) { + case hwmon_temp_input: + switch (channel) { + case CHANNEL_PKG: + case CHANNEL_VRAM: + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_TEMP, channel)); + + /* HW register value is in degrees Celsius, convert to millidegrees. */ + *val = REG_FIELD_GET(TEMP_MASK, reg_val) * MILLIDEGREE_PER_DEGREE; + return 0; + case CHANNEL_MCTRL: + return get_mc_temp(hwmon, val); + case CHANNEL_PCIE: + return get_pcie_temp(hwmon, val); + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_TEMP, channel)); + /* + * This temperature format is 24 bit [31:8] signed integer and 8 bit + * [7:0] fraction. + */ + *val = (s32)(REG_FIELD_GET(TEMP_MASK_VRAM_N, reg_val)) * + (REG_FIELD_GET(TEMP_SIGN_MASK, reg_val) ? -1 : 1) * + MILLIDEGREE_PER_DEGREE; + return 0; + default: + return -EOPNOTSUPP; + } + case hwmon_temp_emergency: + switch (channel) { + case CHANNEL_PKG: + case CHANNEL_MCTRL: + case CHANNEL_PCIE: + *val = hwmon->temp.limit[TEMP_LIMIT_PKG_SHUTDOWN] * MILLIDEGREE_PER_DEGREE; + return 0; + case CHANNEL_VRAM: + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + *val = hwmon->temp.limit[TEMP_LIMIT_MEM_SHUTDOWN] * MILLIDEGREE_PER_DEGREE; + return 0; + default: + return -EOPNOTSUPP; + } + case hwmon_temp_crit: + switch (channel) { + case CHANNEL_PKG: + case CHANNEL_MCTRL: + case CHANNEL_PCIE: + *val = hwmon->temp.limit[TEMP_LIMIT_PKG_CRIT] * MILLIDEGREE_PER_DEGREE; + return 0; + case CHANNEL_VRAM: + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + *val = hwmon->temp.limit[TEMP_LIMIT_MEM_CRIT] * MILLIDEGREE_PER_DEGREE; + return 0; + default: + return -EOPNOTSUPP; + } + case hwmon_temp_max: + switch (channel) { + case CHANNEL_PKG: + *val = hwmon->temp.limit[TEMP_LIMIT_PKG_MAX] * MILLIDEGREE_PER_DEGREE; + return 0; + default: + return -EOPNOTSUPP; + } + default: + return -EOPNOTSUPP; + } +} + +static umode_t +xe_hwmon_power_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) +{ + u32 uval = 0; + struct xe_reg reg; + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + + switch (attr) { + case hwmon_power_max: + case hwmon_power_cap: + if (hwmon->xe->info.has_mbx_power_limits) { + xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, &uval); + } else if (attr != PL2_HWMON_ATTR) { + reg = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); + if (xe_reg_is_valid(reg)) + uval = xe_mmio_read32(mmio, reg); + } + if (uval & PWR_LIM_EN) { + drm_info(&hwmon->xe->drm, "%s is supported on channel %d\n", + PWR_ATTR_TO_STR(attr), channel); + return 0664; + } + drm_dbg(&hwmon->xe->drm, "%s is unsupported on channel %d\n", + PWR_ATTR_TO_STR(attr), channel); + return 0; + case hwmon_power_rated_max: + if (hwmon->xe->info.has_mbx_power_limits) { + return 0; + } else { + reg = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); + if (xe_reg_is_valid(reg)) + uval = xe_mmio_read32(mmio, reg); + return uval ? 0444 : 0; + } + case hwmon_power_crit: + if (channel == CHANNEL_CARD) { + xe_hwmon_pcode_read_i1(hwmon, &uval); + return (uval & POWER_SETUP_I1_WATTS) ? 0644 : 0; + } + break; + case hwmon_power_label: + if (hwmon->xe->info.has_mbx_power_limits) { + xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, &uval); + } else { + reg = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); + if (xe_reg_is_valid(reg)) + uval = xe_mmio_read32(mmio, reg); + + if (!uval) { + reg = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); + if (xe_reg_is_valid(reg)) + uval = xe_mmio_read32(mmio, reg); + } + } + if ((!(uval & PWR_LIM_EN)) && channel == CHANNEL_CARD) { + xe_hwmon_pcode_read_i1(hwmon, &uval); + return (uval & POWER_SETUP_I1_WATTS) ? 0444 : 0; + } + return (uval) ? 0444 : 0; + default: + return 0; + } + return 0; +} + +static int +xe_hwmon_power_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + switch (attr) { + case hwmon_power_max: + case hwmon_power_cap: + xe_hwmon_power_max_read(hwmon, attr, channel, val); + return 0; + case hwmon_power_rated_max: + xe_hwmon_power_rated_max_read(hwmon, attr, channel, val); + return 0; + case hwmon_power_crit: + return xe_hwmon_power_curr_crit_read(hwmon, channel, val, SF_POWER); + default: + return -EOPNOTSUPP; + } +} + +static int +xe_hwmon_power_write(struct xe_hwmon *hwmon, u32 attr, int channel, long val) +{ + switch (attr) { + case hwmon_power_cap: + case hwmon_power_max: + return xe_hwmon_power_max_write(hwmon, attr, channel, val); + case hwmon_power_crit: + return xe_hwmon_power_curr_crit_write(hwmon, channel, val, SF_POWER); + default: + return -EOPNOTSUPP; + } +} + +static umode_t +xe_hwmon_curr_is_visible(const struct xe_hwmon *hwmon, u32 attr, int channel) +{ + u32 uval = 0; + + /* hwmon sysfs attribute of current available only for package */ + if (channel != CHANNEL_PKG) + return 0; + + switch (attr) { + case hwmon_curr_crit: + return (xe_hwmon_pcode_read_i1(hwmon, &uval) || + (uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644; + case hwmon_curr_label: + return (xe_hwmon_pcode_read_i1(hwmon, &uval) || + (uval & POWER_SETUP_I1_WATTS)) ? 0 : 0444; + break; + default: + return 0; + } + return 0; +} + +static int +xe_hwmon_curr_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + switch (attr) { + case hwmon_curr_crit: + return xe_hwmon_power_curr_crit_read(hwmon, channel, val, SF_CURR); + default: + return -EOPNOTSUPP; + } +} + +static int +xe_hwmon_curr_write(struct xe_hwmon *hwmon, u32 attr, int channel, long val) +{ + switch (attr) { + case hwmon_curr_crit: + return xe_hwmon_power_curr_crit_write(hwmon, channel, val, SF_CURR); + default: + return -EOPNOTSUPP; + } +} + +static umode_t +xe_hwmon_in_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) +{ + switch (attr) { + case hwmon_in_input: + case hwmon_in_label: + return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_GT_PERF_STATUS, + channel)) ? 0444 : 0; + default: + return 0; + } +} + +static int +xe_hwmon_in_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + switch (attr) { + case hwmon_in_input: + xe_hwmon_get_voltage(hwmon, channel, val); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static umode_t +xe_hwmon_energy_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) +{ + long energy = 0; + + switch (attr) { + case hwmon_energy_input: + case hwmon_energy_label: + if (hwmon->xe->info.platform == XE_BATTLEMAGE) { + xe_hwmon_energy_get(hwmon, channel, &energy); + return energy ? 0444 : 0; + } else { + return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_ENERGY_STATUS, + channel)) ? 0444 : 0; + } + default: + return 0; + } +} + +static int +xe_hwmon_energy_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + switch (attr) { + case hwmon_energy_input: + xe_hwmon_energy_get(hwmon, channel, val); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static umode_t +xe_hwmon_fan_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) +{ + u32 uval = 0; + + if (!hwmon->xe->info.has_fan_control) + return 0; + + switch (attr) { + case hwmon_fan_input: + if (xe_hwmon_pcode_read_fan_control(hwmon, FSC_READ_NUM_FANS, &uval)) + return 0; + + return channel < uval ? 0444 : 0; + default: + return 0; + } +} + +static int +xe_hwmon_fan_input_read(struct xe_hwmon *hwmon, int channel, long *val) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + struct xe_hwmon_fan_info *fi = &hwmon->fi[channel]; + u64 rotations, time_now, time; + u32 reg_val; + int ret = 0; + + mutex_lock(&hwmon->hwmon_lock); + + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_FAN_SPEED, channel)); + time_now = get_jiffies_64(); + + /* + * HW register value is accumulated count of pulses from PWM fan with the scale + * of 2 pulses per rotation. + */ + rotations = (reg_val - fi->reg_val_prev) / 2; + + time = jiffies_delta_to_msecs(time_now - fi->time_prev); + if (unlikely(!time)) { + ret = -EAGAIN; + goto unlock; + } + + /* + * Calculate fan speed in RPM by time averaging two subsequent readings in minutes. + * RPM = number of rotations * msecs per minute / time in msecs + */ + *val = DIV_ROUND_UP_ULL(rotations * (MSEC_PER_SEC * 60), time); + + fi->reg_val_prev = reg_val; + fi->time_prev = time_now; +unlock: + mutex_unlock(&hwmon->hwmon_lock); + return ret; +} + +static int +xe_hwmon_fan_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + switch (attr) { + case hwmon_fan_input: + return xe_hwmon_fan_input_read(hwmon, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static umode_t +xe_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type, + u32 attr, int channel) +{ + struct xe_hwmon *hwmon = (struct xe_hwmon *)drvdata; + int ret; + + switch (type) { + case hwmon_temp: + ret = xe_hwmon_temp_is_visible(hwmon, attr, channel); + break; + case hwmon_power: + ret = xe_hwmon_power_is_visible(hwmon, attr, channel); + break; + case hwmon_curr: + ret = xe_hwmon_curr_is_visible(hwmon, attr, channel); + break; + case hwmon_in: + ret = xe_hwmon_in_is_visible(hwmon, attr, channel); + break; + case hwmon_energy: + ret = xe_hwmon_energy_is_visible(hwmon, attr, channel); + break; + case hwmon_fan: + ret = xe_hwmon_fan_is_visible(hwmon, attr, channel); + break; + default: + ret = 0; + break; + } + + return ret; +} + +static int +xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct xe_hwmon *hwmon = dev_get_drvdata(dev); + + guard(xe_pm_runtime)(hwmon->xe); + + switch (type) { + case hwmon_temp: + return xe_hwmon_temp_read(hwmon, attr, channel, val); + case hwmon_power: + return xe_hwmon_power_read(hwmon, attr, channel, val); + case hwmon_curr: + return xe_hwmon_curr_read(hwmon, attr, channel, val); + case hwmon_in: + return xe_hwmon_in_read(hwmon, attr, channel, val); + case hwmon_energy: + return xe_hwmon_energy_read(hwmon, attr, channel, val); + case hwmon_fan: + return xe_hwmon_fan_read(hwmon, attr, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static int +xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, long val) +{ + struct xe_hwmon *hwmon = dev_get_drvdata(dev); + + guard(xe_pm_runtime)(hwmon->xe); + + switch (type) { + case hwmon_power: + return xe_hwmon_power_write(hwmon, attr, channel, val); + case hwmon_curr: + return xe_hwmon_curr_write(hwmon, attr, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static int xe_hwmon_read_label(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + struct xe_hwmon *hwmon = dev_get_drvdata(dev); + + switch (type) { + case hwmon_temp: + if (channel == CHANNEL_PKG) + *str = "pkg"; + else if (channel == CHANNEL_VRAM) + *str = "vram"; + else if (channel == CHANNEL_MCTRL) + *str = "mctrl"; + else if (channel == CHANNEL_PCIE) + *str = "pcie"; + else if (in_range(channel, CHANNEL_VRAM_N, MAX_VRAM_CHANNELS)) + *str = hwmon->temp.vram_label[channel - CHANNEL_VRAM_N]; + return 0; + case hwmon_power: + case hwmon_energy: + case hwmon_curr: + case hwmon_in: + if (channel == CHANNEL_CARD) + *str = "card"; + else if (channel == CHANNEL_PKG) + *str = "pkg"; + return 0; + default: + return -EOPNOTSUPP; + } +} + +static const struct hwmon_ops hwmon_ops = { + .is_visible = xe_hwmon_is_visible, + .read = xe_hwmon_read, + .write = xe_hwmon_write, + .read_string = xe_hwmon_read_label, +}; + +static const struct hwmon_chip_info hwmon_chip_info = { + .ops = &hwmon_ops, + .info = hwmon_info, +}; + +static void +xe_hwmon_get_preregistration_info(struct xe_hwmon *hwmon) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + long energy, fan_speed; + u64 val_sku_unit = 0; + int channel; + struct xe_reg pkg_power_sku_unit; + + if (hwmon->xe->info.has_mbx_power_limits) { + /* Check if GPU firmware support mailbox power limits commands. */ + if (xe_hwmon_pcode_read_power_limit(hwmon, PL1_HWMON_ATTR, CHANNEL_CARD, + &hwmon->pl1_on_boot[CHANNEL_CARD]) | + xe_hwmon_pcode_read_power_limit(hwmon, PL1_HWMON_ATTR, CHANNEL_PKG, + &hwmon->pl1_on_boot[CHANNEL_PKG]) | + xe_hwmon_pcode_read_power_limit(hwmon, PL2_HWMON_ATTR, CHANNEL_CARD, + &hwmon->pl2_on_boot[CHANNEL_CARD]) | + xe_hwmon_pcode_read_power_limit(hwmon, PL2_HWMON_ATTR, CHANNEL_PKG, + &hwmon->pl2_on_boot[CHANNEL_PKG])) { + drm_warn(&hwmon->xe->drm, + "Failed to read power limits, check GPU firmware !\n"); + } else { + drm_info(&hwmon->xe->drm, "Using mailbox commands for power limits\n"); + /* Write default limits to read from pcode from now on. */ + xe_hwmon_pcode_rmw_power_limit(hwmon, PL1_HWMON_ATTR, + CHANNEL_CARD, PWR_LIM | PWR_LIM_TIME, + hwmon->pl1_on_boot[CHANNEL_CARD]); + xe_hwmon_pcode_rmw_power_limit(hwmon, PL1_HWMON_ATTR, + CHANNEL_PKG, PWR_LIM | PWR_LIM_TIME, + hwmon->pl1_on_boot[CHANNEL_PKG]); + xe_hwmon_pcode_rmw_power_limit(hwmon, PL2_HWMON_ATTR, + CHANNEL_CARD, PWR_LIM | PWR_LIM_TIME, + hwmon->pl2_on_boot[CHANNEL_CARD]); + xe_hwmon_pcode_rmw_power_limit(hwmon, PL2_HWMON_ATTR, + CHANNEL_PKG, PWR_LIM | PWR_LIM_TIME, + hwmon->pl2_on_boot[CHANNEL_PKG]); + hwmon->scl_shift_power = PWR_UNIT; + hwmon->scl_shift_energy = ENERGY_UNIT; + hwmon->scl_shift_time = TIME_UNIT; + hwmon->boot_power_limit_read = true; + } + } else { + drm_info(&hwmon->xe->drm, "Using register for power limits\n"); + /* + * The contents of register PKG_POWER_SKU_UNIT do not change, + * so read it once and store the shift values. + */ + pkg_power_sku_unit = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU_UNIT, 0); + if (xe_reg_is_valid(pkg_power_sku_unit)) { + val_sku_unit = xe_mmio_read32(mmio, pkg_power_sku_unit); + hwmon->scl_shift_power = REG_FIELD_GET(PKG_PWR_UNIT, val_sku_unit); + hwmon->scl_shift_energy = REG_FIELD_GET(PKG_ENERGY_UNIT, val_sku_unit); + hwmon->scl_shift_time = REG_FIELD_GET(PKG_TIME_UNIT, val_sku_unit); + } + } + /* + * Initialize 'struct xe_hwmon_energy_info', i.e. set fields to the + * first value of the energy register read + */ + for (channel = 0; channel < CHANNEL_MAX; channel++) + if (xe_hwmon_is_visible(hwmon, hwmon_energy, hwmon_energy_input, channel)) + xe_hwmon_energy_get(hwmon, channel, &energy); + + /* Initialize 'struct xe_hwmon_fan_info' with initial fan register reading. */ + for (channel = 0; channel < FAN_MAX; channel++) + if (xe_hwmon_is_visible(hwmon, hwmon_fan, hwmon_fan_input, channel)) + xe_hwmon_fan_input_read(hwmon, channel, &fan_speed); + + if (hwmon->xe->info.has_mbx_thermal_info && xe_hwmon_pcode_read_thermal_info(hwmon)) + drm_warn(&hwmon->xe->drm, "Thermal mailbox not supported by card firmware\n"); +} + +int xe_hwmon_register(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + struct xe_hwmon *hwmon; + int ret; + + /* hwmon is available only for dGfx */ + if (!IS_DGFX(xe)) + return 0; + + /* hwmon is not available on VFs */ + if (IS_SRIOV_VF(xe)) + return 0; + + hwmon = devm_kzalloc(dev, sizeof(*hwmon), GFP_KERNEL); + if (!hwmon) + return -ENOMEM; + + ret = devm_mutex_init(dev, &hwmon->hwmon_lock); + if (ret) + return ret; + + /* There's only one instance of hwmon per device */ + hwmon->xe = xe; + xe->hwmon = hwmon; + + xe_hwmon_get_preregistration_info(hwmon); + + drm_dbg(&xe->drm, "Register xe hwmon interface\n"); + + /* hwmon_dev points to device hwmon */ + hwmon->hwmon_dev = devm_hwmon_device_register_with_info(dev, "xe", hwmon, + &hwmon_chip_info, + hwmon_groups); + if (IS_ERR(hwmon->hwmon_dev)) { + drm_err(&xe->drm, "Failed to register xe hwmon (%pe)\n", hwmon->hwmon_dev); + xe->hwmon = NULL; + return PTR_ERR(hwmon->hwmon_dev); + } + + return 0; +} +MODULE_IMPORT_NS("INTEL_PMT_TELEMETRY"); diff --git a/drivers/gpu/drm/xe/xe_hwmon.h b/drivers/gpu/drm/xe/xe_hwmon.h new file mode 100644 index 000000000..d02c1bfe8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hwmon.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_HWMON_H_ +#define _XE_HWMON_H_ + +#include + +struct xe_device; + +#if IS_REACHABLE(CONFIG_HWMON) +int xe_hwmon_register(struct xe_device *xe); +#else +static inline int xe_hwmon_register(struct xe_device *xe) { return 0; }; +#endif + +#endif /* _XE_HWMON_H_ */ diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c new file mode 100644 index 000000000..399099ff0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Intel Xe I2C attached Microcontroller Units (MCU) + * + * Copyright (C) 2025 Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "regs/xe_i2c_regs.h" +#include "regs/xe_irq_regs.h" + +#include "xe_amc.h" +#include "xe_device.h" +#include "xe_i2c.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_survivability_mode.h" + +/** + * DOC: Xe I2C devices + * + * Register a platform device for the I2C host controller (Synpsys DesignWare + * I2C) if the registers of that controller are mapped to the MMIO, and also the + * I2C client device for the Add-In Management Controller (the MCU) attached to + * the host controller. + * + * See drivers/i2c/busses/i2c-designware-* for more information on the I2C host + * controller. + */ + +static const char adapter_name[] = "i2c_designware"; + +static const struct property_entry xe_i2c_adapter_properties[] = { + PROPERTY_ENTRY_STRING("compatible", "intel,xe-i2c"), + PROPERTY_ENTRY_U32("clock-frequency", I2C_MAX_FAST_MODE_PLUS_FREQ), + { } +}; + +static inline void xe_i2c_read_endpoint(struct xe_mmio *mmio, void *ep) +{ + u32 *val = ep; + + val[0] = xe_mmio_read32(mmio, REG_SG_REMAP_ADDR_PREFIX); + val[1] = xe_mmio_read32(mmio, REG_SG_REMAP_ADDR_POSTFIX); +} + +static void xe_i2c_handle_smbus_alert(struct xe_i2c *i2c) +{ + u32 stat; + + stat = xe_mmio_read32(i2c->mmio, I2C_REG(DW_IC_SMBUS_INTR_STAT)); + if (!stat) + return; + + xe_mmio_write32(i2c->mmio, I2C_REG(DW_IC_CLR_SMBUS_INTR), stat); + + if (stat & DW_IC_SMBUS_INTR_ALERT && i2c->amc) + xe_amc_handle_alert(i2c); + else + xe_mmio_rmw32(i2c->mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); +} + +static void xe_i2c_client_work(struct work_struct *work) +{ + struct xe_i2c *i2c = container_of(work, struct xe_i2c, work); + struct i2c_board_info info = { + .type = "amc", + .flags = I2C_CLIENT_HOST_NOTIFY, + .addr = i2c->ep.addr[XE_I2C_CLIENT_AMC], + }; + + i2c->client[XE_I2C_CLIENT_AMC] = i2c_new_client_device(i2c->adapter, &info); +} + +static int xe_i2c_notifier(struct notifier_block *nb, unsigned long action, void *data) +{ + struct xe_i2c *i2c = container_of(nb, struct xe_i2c, bus_notifier); + struct i2c_adapter *adapter = i2c_verify_adapter(data); + struct device *dev = data; + + if (action == BUS_NOTIFY_ADD_DEVICE && + adapter && dev->parent == &i2c->pdev->dev) { + i2c->adapter = adapter; + schedule_work(&i2c->work); + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + +static int xe_i2c_register_adapter(struct xe_i2c *i2c) +{ + struct pci_dev *pci = to_pci_dev(i2c->drm_dev); + struct platform_device *pdev; + int ret; + u32 id; + + id = (pci_domain_nr(pci->bus) << 16) | pci_dev_id(pci); + + /* + * Not using platform_device_register_full() here because we don't have + * a handle to the platform_device before it returns. xe_i2c_notifier() + * uses that handle, but it may be called before + * platform_device_register_full() is done. + */ + pdev = platform_device_alloc(adapter_name, id); + if (!pdev) + return -ENOMEM; + + ret = device_create_managed_software_node(&pdev->dev, + xe_i2c_adapter_properties, + NULL); + if (ret) + goto err_pdev_put; + + pdev->dev.parent = i2c->drm_dev; + i2c->pdev = pdev; + + ret = platform_device_add(pdev); + if (ret) + goto err_pdev_put; + + return 0; + +err_pdev_put: + platform_device_put(pdev); + + return ret; +} + +static void xe_i2c_unregister_adapter(struct xe_i2c *i2c) +{ + platform_device_unregister(i2c->pdev); +} + +/** + * xe_i2c_present - I2C controller is present and functional + * @xe: xe device instance + * + * Check whether the I2C controller is present and functioning with valid + * endpoint cookie. + * + * Return: %true if present, %false otherwise. + */ +bool xe_i2c_present(struct xe_device *xe) +{ + return xe->i2c && xe->i2c->ep.cookie == XE_I2C_EP_COOKIE_DEVICE; +} + +static bool xe_i2c_irq_present(struct xe_device *xe) +{ + return xe->i2c && xe->i2c->ep.capabilities & XE_I2C_EP_CAP_IRQ && + !xe_survivability_mode_is_boot_enabled(xe); +} + +/** + * xe_i2c_irq_handler: Handler for I2C interrupts + * @xe: xe device instance + * @master_ctl: interrupt register + * + * Forward interrupts generated by the I2C host adapter to the I2C host adapter + * driver. + */ +void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + if (!(master_ctl & I2C_IRQ) || !xe_i2c_irq_present(xe)) + return; + + xe_i2c_handle_smbus_alert(xe->i2c); +} + +void xe_i2c_irq_reset(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + if (!xe_i2c_irq_present(xe)) + return; + + xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_INTX_DISABLE); + xe_mmio_rmw32(mmio, I2C_BRIDGE_PCICFGCTL, ACPI_INTR_EN, 0); +} + +void xe_i2c_irq_postinstall(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + if (!xe_i2c_irq_present(xe)) + return; + + xe_mmio_rmw32(mmio, I2C_BRIDGE_PCICFGCTL, 0, ACPI_INTR_EN); + xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); +} + +/* See "Disabling DW_apb_i2c" in the DesignWare DW_abp_i2c databook. */ +static void xe_i2c_disable(struct xe_i2c *i2c) +{ + int timeout = 100; + u32 status; + + xe_mmio_rmw32(i2c->mmio, I2C_REG(DW_IC_ENABLE), DW_IC_ENABLE_ENABLE, 0); + + do { + status = xe_mmio_read32(i2c->mmio, I2C_REG(DW_IC_ENABLE_STATUS)); + if (!(status & DW_IC_ENABLE_ENABLE)) + return; + /* Can't sleep here. */ + udelay(25); + } while (timeout--); + + dev_warn(i2c->drm_dev, "timeout in disabling i2c adapter\n"); +} + +static int xe_i2c_read(void *context, unsigned int reg, unsigned int *val) +{ + struct xe_i2c *i2c = context; + + *val = xe_mmio_read32(i2c->mmio, I2C_REG(reg)); + + switch (reg) { + case DW_IC_ENABLE: + case DW_IC_ENABLE_STATUS: + FIELD_MODIFY(DW_IC_ENABLE_ENABLE, val, + i2c->ic_enable & DW_IC_ENABLE_ENABLE); + break; + default: + break; + } + + return 0; +} + +static int xe_i2c_write(void *context, unsigned int reg, unsigned int val) +{ + struct xe_i2c *i2c = context; + + switch (reg) { + case DW_IC_CON: + case DW_IC_TAR: + case DW_IC_SAR: + /* Disable the controller. */ + xe_i2c_disable(i2c); + + /* Write the register. */ + xe_mmio_write32(i2c->mmio, I2C_REG(reg), val); + + /* Enable the controller. */ + xe_mmio_rmw32(i2c->mmio, I2C_REG(DW_IC_ENABLE), 0, DW_IC_ENABLE_ENABLE); + return 0; + case DW_IC_ENABLE: + i2c->ic_enable = val; + /* Other fields can be updated except the enable bit. */ + val |= DW_IC_ENABLE_ENABLE; + break; + case DW_IC_SMBUS_INTR_MASK: + /* Make sure the Alert is never masked. */ + val |= DW_IC_SMBUS_INTR_ALERT; + break; + default: + break; + } + + xe_mmio_write32(i2c->mmio, I2C_REG(reg), val); + return 0; +} + +static const struct regmap_config i2c_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_read = xe_i2c_read, + .reg_write = xe_i2c_write, + .fast_io = true, +}; + +void xe_i2c_pm_suspend(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + if (!xe_i2c_present(xe)) + return; + + xe_mmio_rmw32(mmio, I2C_CONFIG_PMCSR, PCI_PM_CTRL_STATE_MASK, (__force u32)PCI_D3hot); + drm_dbg(&xe->drm, "pmcsr: 0x%08x\n", xe_mmio_read32(mmio, I2C_CONFIG_PMCSR)); +} + +void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + if (!xe_i2c_present(xe)) + return; + + if (d3cold) + xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + + xe_mmio_rmw32(mmio, I2C_CONFIG_PMCSR, PCI_PM_CTRL_STATE_MASK, (__force u32)PCI_D0); + drm_dbg(&xe->drm, "pmcsr: 0x%08x\n", xe_mmio_read32(mmio, I2C_CONFIG_PMCSR)); +} + +static void xe_i2c_remove(void *data) +{ + struct xe_i2c *i2c = data; + unsigned int i; + + xe_amc_exit(i2c); + + for (i = 0; i < XE_I2C_MAX_CLIENTS; i++) { + i2c_unregister_device(i2c->client[i]); + i2c->client[i] = NULL; + } + + bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); + xe_i2c_unregister_adapter(i2c); +} + +/** + * xe_i2c_probe: Probe the I2C host adapter and the I2C clients attached to it + * @xe: xe device instance + * + * Register all the I2C devices described in the I2C Endpoint data structure. + * + * Return: 0 on success, error code on failure + */ +int xe_i2c_probe(struct xe_device *xe) +{ + struct device *drm_dev = xe->drm.dev; + struct xe_i2c_endpoint ep; + struct regmap *regmap; + struct xe_i2c *i2c; + int ret; + + if (!xe->info.has_i2c) + return 0; + + xe_i2c_read_endpoint(xe_root_tile_mmio(xe), &ep); + if (ep.cookie != XE_I2C_EP_COOKIE_DEVICE) + return 0; + + i2c = devm_kzalloc(drm_dev, sizeof(*i2c), GFP_KERNEL); + if (!i2c) + return -ENOMEM; + + INIT_WORK(&i2c->work, xe_i2c_client_work); + i2c->mmio = xe_root_tile_mmio(xe); + i2c->drm_dev = drm_dev; + i2c->ep = ep; + xe->i2c = i2c; + + /* PCI PM isn't aware of this device, bring it up and match it with SGUnit state. */ + xe_i2c_pm_resume(xe, true); + + regmap = devm_regmap_init(drm_dev, NULL, i2c, &i2c_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + i2c->bus_notifier.notifier_call = xe_i2c_notifier; + ret = bus_register_notifier(&i2c_bus_type, &i2c->bus_notifier); + if (ret) + return ret; + + ret = xe_i2c_register_adapter(i2c); + if (ret) { + bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); + return ret; + } + + ret = xe_amc_init(i2c); + if (ret) { + xe_i2c_remove(i2c); + return ret; + } + + xe_i2c_irq_postinstall(xe); + return devm_add_action_or_reset(drm_dev, xe_i2c_remove, i2c); +} diff --git a/drivers/gpu/drm/xe/xe_i2c.h b/drivers/gpu/drm/xe/xe_i2c.h new file mode 100644 index 000000000..5969054a6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_i2c.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef _XE_I2C_H_ +#define _XE_I2C_H_ + +#include +#include +#include +#include + +struct device; +struct fwnode_handle; +struct i2c_adapter; +struct i2c_client; +struct platform_device; +struct xe_amc; +struct xe_device; +struct xe_mmio; + +#define XE_I2C_EP_COOKIE_DEVICE 0xde + +/* Endpoint Capabilities */ +#define XE_I2C_EP_CAP_IRQ BIT(0) + +enum XE_I2C_CLIENT { + XE_I2C_CLIENT_AMC = 1, + XE_I2C_MAX_CLIENTS = 3, +}; + +struct xe_i2c_endpoint { + u8 cookie; + u8 capabilities; + u16 addr[XE_I2C_MAX_CLIENTS]; +}; + +struct xe_i2c { + struct platform_device *pdev; + struct i2c_adapter *adapter; + struct i2c_client *client[XE_I2C_MAX_CLIENTS]; + unsigned int ic_enable; + + struct notifier_block bus_notifier; + struct work_struct work; + + struct xe_i2c_endpoint ep; + struct device *drm_dev; + + struct xe_mmio *mmio; + struct xe_amc *amc; +}; + +#if IS_ENABLED(CONFIG_I2C) +int xe_i2c_probe(struct xe_device *xe); +bool xe_i2c_present(struct xe_device *xe); +void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl); +void xe_i2c_irq_postinstall(struct xe_device *xe); +void xe_i2c_irq_reset(struct xe_device *xe); +void xe_i2c_pm_suspend(struct xe_device *xe); +void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold); +#else +static inline int xe_i2c_probe(struct xe_device *xe) { return 0; } +static inline bool xe_i2c_present(struct xe_device *xe) { return false; } +static inline void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl) { } +static inline void xe_i2c_irq_postinstall(struct xe_device *xe) { } +static inline void xe_i2c_irq_reset(struct xe_device *xe) { } +static inline void xe_i2c_pm_suspend(struct xe_device *xe) { } +static inline void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c new file mode 100644 index 000000000..9e49e2241 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -0,0 +1,1053 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_irq.h" + +#include + +#include + +#include "display/xe_display.h" +#include "regs/xe_irq_regs.h" +#include "xe_device.h" +#include "xe_drv.h" +#include "xe_gsc_proxy.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_hw_engine.h" +#include "xe_hw_error.h" +#include "xe_i2c.h" +#include "xe_memirq.h" +#include "xe_mert.h" +#include "xe_mmio.h" +#include "xe_pxp.h" +#include "xe_sriov.h" +#include "xe_sysctrl.h" +#include "xe_tile.h" + +/* + * Interrupt registers for a unit are always consecutive and ordered + * ISR, IMR, IIR, IER. + */ +#define IMR(offset) XE_REG(offset + 0x4) +#define IIR(offset) XE_REG(offset + 0x8) +#define IER(offset) XE_REG(offset + 0xc) + +static int xe_irq_msix_init(struct xe_device *xe); +static void xe_irq_msix_free(struct xe_device *xe); +static int xe_irq_msix_request_irqs(struct xe_device *xe); +static void xe_irq_msix_synchronize_irq(struct xe_device *xe); + +static void assert_iir_is_zero(struct xe_mmio *mmio, struct xe_reg reg) +{ + u32 val = xe_mmio_read32(mmio, reg); + + if (val == 0) + return; + + drm_WARN(&mmio->tile->xe->drm, 1, + "Interrupt register 0x%x is not zero: 0x%08x\n", + reg.addr, val); + xe_mmio_write32(mmio, reg, 0xffffffff); + xe_mmio_read32(mmio, reg); + xe_mmio_write32(mmio, reg, 0xffffffff); + xe_mmio_read32(mmio, reg); +} + +/* + * Unmask and enable the specified interrupts. Does not check current state, + * so any bits not specified here will become masked and disabled. + */ +static void unmask_and_enable(struct xe_tile *tile, u32 irqregs, u32 bits) +{ + struct xe_mmio *mmio = &tile->mmio; + + /* + * If we're just enabling an interrupt now, it shouldn't already + * be raised in the IIR. + */ + assert_iir_is_zero(mmio, IIR(irqregs)); + + xe_mmio_write32(mmio, IER(irqregs), bits); + xe_mmio_write32(mmio, IMR(irqregs), ~bits); + + /* Posting read */ + xe_mmio_read32(mmio, IMR(irqregs)); +} + +/* Mask and disable all interrupts. */ +static void mask_and_disable(struct xe_tile *tile, u32 irqregs) +{ + struct xe_mmio *mmio = &tile->mmio; + + xe_mmio_write32(mmio, IMR(irqregs), ~0); + /* Posting read */ + xe_mmio_read32(mmio, IMR(irqregs)); + + xe_mmio_write32(mmio, IER(irqregs), 0); + + /* IIR can theoretically queue up two events. Be paranoid. */ + xe_mmio_write32(mmio, IIR(irqregs), ~0); + xe_mmio_read32(mmio, IIR(irqregs)); + xe_mmio_write32(mmio, IIR(irqregs), ~0); + xe_mmio_read32(mmio, IIR(irqregs)); +} + +static u32 xelp_intr_disable(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0); + + /* + * Now with master disabled, get a sample of level indications + * for this interrupt. Indications will be cleared on related acks. + * New indications can and will light up during processing, + * and will generate new interrupt after enabling master. + */ + return xe_mmio_read32(mmio, GFX_MSTR_IRQ); +} + +static u32 +gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + u32 iir; + + if (!(master_ctl & GU_MISC_IRQ)) + return 0; + + iir = xe_mmio_read32(mmio, IIR(GU_MISC_IRQ_OFFSET)); + if (likely(iir)) + xe_mmio_write32(mmio, IIR(GU_MISC_IRQ_OFFSET), iir); + + return iir; +} + +static inline void xelp_intr_enable(struct xe_device *xe, bool stall) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + xe_mmio_write32(mmio, GFX_MSTR_IRQ, MASTER_IRQ); + if (stall) + xe_mmio_read32(mmio, GFX_MSTR_IRQ); +} + +/* Enable/unmask the HWE interrupts for a specific GT's engines. */ +void xe_irq_enable_hwe(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_mmio *mmio = >->mmio; + u32 common_mask, val, gsc_mask = 0, heci_mask = 0, + rcs_mask = 0, bcs_mask = 0, vcs_mask = 0, vecs_mask = 0, + ccs_mask = 0; + + if (xe_device_uses_memirq(xe)) + return; + + if (xe_device_uc_enabled(xe)) { + common_mask = GT_MI_USER_INTERRUPT | + GT_FLUSH_COMPLETE_INTERRUPT; + + /* Enable Compute Walker Interrupt for non-MSIX platforms */ + if (GRAPHICS_VERx100(xe) >= 3511 && !xe_device_has_msix(xe)) { + rcs_mask |= GT_COMPUTE_WALKER_INTERRUPT; + ccs_mask |= GT_COMPUTE_WALKER_INTERRUPT; + } + } else { + common_mask = GT_MI_USER_INTERRUPT | + GT_CS_MASTER_ERROR_INTERRUPT | + GT_CONTEXT_SWITCH_INTERRUPT | + GT_WAIT_SEMAPHORE_INTERRUPT; + } + + rcs_mask |= common_mask; + bcs_mask |= common_mask; + vcs_mask |= common_mask; + vecs_mask |= common_mask; + ccs_mask |= common_mask; + + if (xe_gt_is_main_type(gt)) { + /* + * For enabling the interrupts, the information about fused off + * engines doesn't matter much, but this also allows to check if + * the engine is available architecturally in the platform + */ + u32 ccs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE); + u32 bcs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY); + + /* Enable interrupts for each engine class */ + xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, + REG_FIELD_PREP(ENGINE1_MASK, rcs_mask) | + REG_FIELD_PREP(ENGINE0_MASK, bcs_mask)); + if (ccs_fuse_mask) + xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, + REG_FIELD_PREP(ENGINE1_MASK, ccs_mask)); + + /* Unmask interrupts for each engine instance */ + val = ~REG_FIELD_PREP(ENGINE1_MASK, rcs_mask); + xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, val); + val = ~REG_FIELD_PREP(ENGINE1_MASK, bcs_mask); + xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, val); + + val = ~(REG_FIELD_PREP(ENGINE1_MASK, bcs_mask) | + REG_FIELD_PREP(ENGINE0_MASK, bcs_mask)); + if (bcs_fuse_mask & (BIT(1)|BIT(2))) + xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, val); + if (bcs_fuse_mask & (BIT(3)|BIT(4))) + xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, val); + if (bcs_fuse_mask & (BIT(5)|BIT(6))) + xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, val); + if (bcs_fuse_mask & (BIT(7)|BIT(8))) + xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, val); + + val = ~(REG_FIELD_PREP(ENGINE1_MASK, ccs_mask) | + REG_FIELD_PREP(ENGINE0_MASK, ccs_mask)); + if (ccs_fuse_mask & (BIT(0)|BIT(1))) + xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, val); + if (ccs_fuse_mask & (BIT(2)|BIT(3))) + xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, val); + } + + if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) { + u32 vcs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); + u32 vecs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); + u32 other_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER); + + /* Enable interrupts for each engine class */ + xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, + REG_FIELD_PREP(ENGINE1_MASK, vcs_mask) | + REG_FIELD_PREP(ENGINE0_MASK, vecs_mask)); + + /* Unmask interrupts for each engine instance */ + val = ~(REG_FIELD_PREP(ENGINE1_MASK, vcs_mask) | + REG_FIELD_PREP(ENGINE0_MASK, vcs_mask)); + if (vcs_fuse_mask & (BIT(0) | BIT(1))) + xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, val); + if (vcs_fuse_mask & (BIT(2) | BIT(3))) + xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, val); + if (vcs_fuse_mask & (BIT(4) | BIT(5))) + xe_mmio_write32(mmio, VCS4_VCS5_INTR_MASK, val); + if (vcs_fuse_mask & (BIT(6) | BIT(7))) + xe_mmio_write32(mmio, VCS6_VCS7_INTR_MASK, val); + + val = ~(REG_FIELD_PREP(ENGINE1_MASK, vecs_mask) | + REG_FIELD_PREP(ENGINE0_MASK, vecs_mask)); + if (vecs_fuse_mask & (BIT(0) | BIT(1))) + xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, val); + if (vecs_fuse_mask & (BIT(2) | BIT(3))) + xe_mmio_write32(mmio, VECS2_VECS3_INTR_MASK, val); + + /* + * the heci2 interrupt is enabled via the same register as the + * GSCCS interrupts, but it has its own mask register. + */ + if (other_fuse_mask) { + gsc_mask = common_mask | GSC_ER_COMPLETE; + heci_mask = GSC_IRQ_INTF(1); + } else if (xe->info.has_heci_gscfi) { + gsc_mask = GSC_IRQ_INTF(1); + } + + if (gsc_mask) { + xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, gsc_mask | heci_mask); + xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~gsc_mask); + } + if (heci_mask) + xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~(heci_mask << 16)); + + if (xe_pxp_is_supported(xe)) { + u32 kcr_mask = KCR_PXP_STATE_TERMINATED_INTERRUPT | + KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT | + KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT; + + xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, kcr_mask << 16); + xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~(kcr_mask << 16)); + } + } +} + +static u32 +gt_engine_identity(struct xe_device *xe, + struct xe_mmio *mmio, + const unsigned int bank, + const unsigned int bit) +{ + u32 timeout_ts; + u32 ident; + + lockdep_assert_held(&xe->irq.lock); + + xe_mmio_write32(mmio, IIR_REG_SELECTOR(bank), BIT(bit)); + + /* + * NB: Specs do not specify how long to spin wait, + * so we do ~100us as an educated guess. + */ + timeout_ts = (local_clock() >> 10) + 100; + do { + ident = xe_mmio_read32(mmio, INTR_IDENTITY_REG(bank)); + } while (!(ident & INTR_DATA_VALID) && + !time_after32(local_clock() >> 10, timeout_ts)); + + if (unlikely(!(ident & INTR_DATA_VALID))) { + drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", + bank, bit, ident); + return 0; + } + + xe_mmio_write32(mmio, INTR_IDENTITY_REG(bank), ident); + + return ident; +} + +#define OTHER_MEDIA_GUC_INSTANCE 16 + +static void +gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir) +{ + if (instance == OTHER_GUC_INSTANCE && xe_gt_is_main_type(gt)) + return xe_guc_irq_handler(>->uc.guc, iir); + if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt)) + return xe_guc_irq_handler(>->uc.guc, iir); + if (instance == OTHER_GSC_HECI2_INSTANCE && xe_gt_is_media_type(gt)) + return xe_gsc_proxy_irq_handler(>->uc.gsc, iir); + + if (instance != OTHER_GUC_INSTANCE && + instance != OTHER_MEDIA_GUC_INSTANCE) { + WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", + instance, iir); + } +} + +static struct xe_gt *pick_engine_gt(struct xe_tile *tile, + enum xe_engine_class class, + unsigned int instance) +{ + struct xe_device *xe = tile_to_xe(tile); + + if (MEDIA_VER(xe) < 13) + return tile->primary_gt; + + switch (class) { + case XE_ENGINE_CLASS_VIDEO_DECODE: + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + return tile->media_gt; + case XE_ENGINE_CLASS_OTHER: + switch (instance) { + case OTHER_MEDIA_GUC_INSTANCE: + case OTHER_GSC_INSTANCE: + case OTHER_GSC_HECI2_INSTANCE: + return tile->media_gt; + default: + break; + } + fallthrough; + default: + return tile->primary_gt; + } +} + +static void gt_irq_handler(struct xe_tile *tile, + u32 master_ctl, unsigned long *intr_dw, + u32 *identity) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_mmio *mmio = &tile->mmio; + unsigned int bank, bit; + u16 instance, intr_vec; + enum xe_engine_class class; + struct xe_hw_engine *hwe; + + spin_lock(&xe->irq.lock); + + for (bank = 0; bank < 2; bank++) { + if (!(master_ctl & GT_DW_IRQ(bank))) + continue; + + intr_dw[bank] = xe_mmio_read32(mmio, GT_INTR_DW(bank)); + for_each_set_bit(bit, intr_dw + bank, 32) + identity[bit] = gt_engine_identity(xe, mmio, bank, bit); + xe_mmio_write32(mmio, GT_INTR_DW(bank), intr_dw[bank]); + + for_each_set_bit(bit, intr_dw + bank, 32) { + struct xe_gt *engine_gt; + + class = INTR_ENGINE_CLASS(identity[bit]); + instance = INTR_ENGINE_INSTANCE(identity[bit]); + intr_vec = INTR_ENGINE_INTR(identity[bit]); + + engine_gt = pick_engine_gt(tile, class, instance); + + hwe = xe_gt_hw_engine(engine_gt, class, instance, false); + if (hwe) { + xe_hw_engine_handle_irq(hwe, intr_vec); + continue; + } + + if (class == XE_ENGINE_CLASS_OTHER) { + /* + * HECI GSCFI interrupts come from outside of GT. + * KCR irqs come from inside GT but are handled + * by the global PXP subsystem. + */ + if (xe->info.has_heci_gscfi && instance == OTHER_GSC_INSTANCE) + xe_heci_gsc_irq_handler(xe, intr_vec); + else if (instance == OTHER_KCR_INSTANCE) + xe_pxp_irq_handler(xe, intr_vec); + else + gt_other_irq_handler(engine_gt, instance, intr_vec); + } + } + } + + spin_unlock(&xe->irq.lock); +} + +/* + * Top-level interrupt handler for Xe_LP platforms (which did not have + * a "master tile" interrupt register. + */ +static irqreturn_t xelp_irq_handler(int irq, void *arg) +{ + struct xe_device *xe = arg; + struct xe_tile *tile = xe_device_get_root_tile(xe); + u32 master_ctl, gu_misc_iir; + unsigned long intr_dw[2]; + u32 identity[32]; + + if (!atomic_read(&xe->irq.enabled)) + return IRQ_NONE; + + master_ctl = xelp_intr_disable(xe); + if (!master_ctl) { + xelp_intr_enable(xe, false); + return IRQ_NONE; + } + + gt_irq_handler(tile, master_ctl, intr_dw, identity); + + xe_display_irq_handler(xe, master_ctl); + + gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); + + xelp_intr_enable(xe, false); + + xe_display_irq_enable(xe, gu_misc_iir); + + return IRQ_HANDLED; +} + +static u32 dg1_intr_disable(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + u32 val; + + /* First disable interrupts */ + xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0); + + /* Get the indication levels and ack the master unit */ + val = xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR); + if (unlikely(!val)) + return 0; + + xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, val); + + return val; +} + +static void dg1_intr_enable(struct xe_device *xe, bool stall) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ); + if (stall) + xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR); +} + +/* + * Top-level interrupt handler for Xe_LP+ and beyond. These platforms have + * a "master tile" interrupt register which must be consulted before the + * "graphics master" interrupt register. + */ +static irqreturn_t dg1_irq_handler(int irq, void *arg) +{ + struct xe_device *xe = arg; + struct xe_tile *tile; + u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0; + unsigned long intr_dw[2]; + u32 identity[32]; + u8 id; + + /* TODO: This really shouldn't be copied+pasted */ + + if (!atomic_read(&xe->irq.enabled)) + return IRQ_NONE; + + master_tile_ctl = dg1_intr_disable(xe); + if (!master_tile_ctl) { + dg1_intr_enable(xe, false); + return IRQ_NONE; + } + + for_each_tile(tile, xe, id) { + struct xe_mmio *mmio = &tile->mmio; + + if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0) + continue; + + master_ctl = xe_mmio_read32(mmio, GFX_MSTR_IRQ); + + /* + * We might be in irq handler just when PCIe DPC is initiated + * and all MMIO reads will be returned with all 1's. Ignore this + * irq as device is inaccessible. + */ + if (master_ctl == REG_GENMASK(31, 0)) { + drm_dbg(&tile_to_xe(tile)->drm, + "Ignore this IRQ as device might be in DPC containment.\n"); + return IRQ_HANDLED; + } + + xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl); + + gt_irq_handler(tile, master_ctl, intr_dw, identity); + xe_hw_error_irq_handler(tile, master_ctl); + + /* + * Display interrupts (including display backlight operations + * that get reported as Gunit GSE) would only be hooked up to + * the primary tile. + */ + if (id == 0) { + if (xe->info.has_heci_cscfi) + xe_heci_csc_irq_handler(xe, master_ctl); + xe_display_irq_handler(xe, master_ctl); + xe_i2c_irq_handler(xe, master_ctl); + xe_sysctrl_irq_handler(xe, master_ctl); + xe_mert_irq_handler(xe, master_ctl); + gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); + } + } + + dg1_intr_enable(xe, false); + xe_display_irq_enable(xe, gu_misc_iir); + + return IRQ_HANDLED; +} + +static void gt_irq_reset(struct xe_tile *tile) +{ + struct xe_mmio *mmio = &tile->mmio; + u32 ccs_mask = ~0; + u32 bcs_mask = ~0; + + if (tile->primary_gt) { + ccs_mask = xe_hw_engine_mask_per_class(tile->primary_gt, + XE_ENGINE_CLASS_COMPUTE); + bcs_mask = xe_hw_engine_mask_per_class(tile->primary_gt, + XE_ENGINE_CLASS_COPY); + } + + /* Disable RCS, BCS, VCS and VECS class engines. */ + xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0); + xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0); + if (ccs_mask) + xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0); + + /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ + xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~0); + xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~0); + if (bcs_mask & (BIT(1)|BIT(2))) + xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0); + if (bcs_mask & (BIT(3)|BIT(4))) + xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0); + if (bcs_mask & (BIT(5)|BIT(6))) + xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0); + if (bcs_mask & (BIT(7)|BIT(8))) + xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0); + xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~0); + xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~0); + xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~0); + if (ccs_mask & (BIT(0)|BIT(1))) + xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0); + if (ccs_mask & (BIT(2)|BIT(3))) + xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0); + + if ((tile->media_gt && + xe_hw_engine_mask_per_class(tile->media_gt, XE_ENGINE_CLASS_OTHER)) || + tile_to_xe(tile)->info.has_heci_gscfi) { + xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0); + xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0); + xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0); + xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, 0); + xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~0); + } + + xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0); + xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0); + xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0); + xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0); +} + +static void xelp_irq_reset(struct xe_tile *tile) +{ + xelp_intr_disable(tile_to_xe(tile)); + + gt_irq_reset(tile); + + if (IS_SRIOV_VF(tile_to_xe(tile))) + return; + + mask_and_disable(tile, PCU_IRQ_OFFSET); +} + +static void dg1_irq_reset(struct xe_tile *tile) +{ + if (xe_tile_is_root(tile)) + dg1_intr_disable(tile_to_xe(tile)); + + gt_irq_reset(tile); + + if (IS_SRIOV_VF(tile_to_xe(tile))) + return; + + mask_and_disable(tile, PCU_IRQ_OFFSET); +} + +static void dg1_irq_reset_mstr(struct xe_tile *tile) +{ + struct xe_mmio *mmio = &tile->mmio; + + xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0); +} + +static void vf_irq_reset(struct xe_device *xe) +{ + struct xe_tile *tile; + unsigned int id; + + xe_assert(xe, IS_SRIOV_VF(xe)); + + if (GRAPHICS_VERx100(xe) < 1210) + xelp_intr_disable(xe); + else + xe_assert(xe, xe_device_has_memirq(xe)); + + for_each_tile(tile, xe, id) { + if (xe_device_has_memirq(xe)) + xe_memirq_reset(&tile->memirq); + else + gt_irq_reset(tile); + } +} + +static void xe_irq_reset(struct xe_device *xe) +{ + struct xe_tile *tile; + u8 id; + + if (IS_SRIOV_VF(xe)) + return vf_irq_reset(xe); + + if (xe_device_uses_memirq(xe)) { + for_each_tile(tile, xe, id) + xe_memirq_reset(&tile->memirq); + } + + for_each_tile(tile, xe, id) { + if (GRAPHICS_VERx100(xe) >= 1210) + dg1_irq_reset(tile); + else + xelp_irq_reset(tile); + } + + tile = xe_device_get_root_tile(xe); + mask_and_disable(tile, GU_MISC_IRQ_OFFSET); + xe_display_irq_reset(xe); + xe_i2c_irq_reset(xe); + + /* + * The tile's top-level status register should be the last one + * to be reset to avoid possible bit re-latching from lower + * level interrupts. + */ + if (GRAPHICS_VERx100(xe) >= 1210) { + for_each_tile(tile, xe, id) + dg1_irq_reset_mstr(tile); + } +} + +static void vf_irq_postinstall(struct xe_device *xe) +{ + struct xe_tile *tile; + unsigned int id; + + for_each_tile(tile, xe, id) + if (xe_device_has_memirq(xe)) + xe_memirq_postinstall(&tile->memirq); + + if (GRAPHICS_VERx100(xe) < 1210) + xelp_intr_enable(xe, true); + else + xe_assert(xe, xe_device_has_memirq(xe)); +} + +static void xe_irq_postinstall(struct xe_device *xe) +{ + if (IS_SRIOV_VF(xe)) + return vf_irq_postinstall(xe); + + if (xe_device_uses_memirq(xe)) { + struct xe_tile *tile; + unsigned int id; + + for_each_tile(tile, xe, id) + xe_memirq_postinstall(&tile->memirq); + } + + xe_display_irq_postinstall(xe); + xe_i2c_irq_postinstall(xe); + + /* + * ASLE backlight operations are reported via GUnit GSE interrupts + * on the root tile. + */ + unmask_and_enable(xe_device_get_root_tile(xe), + GU_MISC_IRQ_OFFSET, GU_MISC_GSE); + + /* Enable top-level interrupts */ + if (GRAPHICS_VERx100(xe) >= 1210) + dg1_intr_enable(xe, true); + else + xelp_intr_enable(xe, true); +} + +static irqreturn_t vf_mem_irq_handler(int irq, void *arg) +{ + struct xe_device *xe = arg; + struct xe_tile *tile; + unsigned int id; + + if (!atomic_read(&xe->irq.enabled)) + return IRQ_NONE; + + for_each_tile(tile, xe, id) + xe_memirq_handler(&tile->memirq); + + return IRQ_HANDLED; +} + +static irq_handler_t xe_irq_handler(struct xe_device *xe) +{ + if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) + return vf_mem_irq_handler; + + if (GRAPHICS_VERx100(xe) >= 1210) + return dg1_irq_handler; + else + return xelp_irq_handler; +} + +static int xe_irq_msi_request_irqs(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + irq_handler_t irq_handler; + int irq, err; + + irq_handler = xe_irq_handler(xe); + if (!irq_handler) { + drm_err(&xe->drm, "No supported interrupt handler"); + return -EINVAL; + } + + irq = pci_irq_vector(pdev, 0); + err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe); + if (err < 0) { + drm_err(&xe->drm, "Failed to request MSI IRQ %d\n", err); + return err; + } + + return 0; +} + +static void xe_irq_msi_free(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int irq; + + irq = pci_irq_vector(pdev, 0); + free_irq(irq, xe); +} + +static void irq_uninstall(void *arg) +{ + struct xe_device *xe = arg; + + if (!atomic_xchg(&xe->irq.enabled, 0)) + return; + + xe_irq_reset(xe); + + if (xe_device_has_msix(xe)) + xe_irq_msix_free(xe); + else + xe_irq_msi_free(xe); +} + +int xe_irq_init(struct xe_device *xe) +{ + spin_lock_init(&xe->irq.lock); + + return xe_irq_msix_init(xe); +} + +int xe_irq_install(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + unsigned int irq_flags = PCI_IRQ_MSI; + int nvec = 1; + int err; + + xe_hw_error_init(xe); + + xe_irq_reset(xe); + + if (xe_device_has_msix(xe)) { + nvec = xe->irq.msix.nvec; + irq_flags = PCI_IRQ_MSIX; + } + + err = pci_alloc_irq_vectors(pdev, nvec, nvec, irq_flags); + if (err < 0) { + drm_err(&xe->drm, "Failed to allocate IRQ vectors: %d\n", err); + return err; + } + + err = xe_device_has_msix(xe) ? xe_irq_msix_request_irqs(xe) : + xe_irq_msi_request_irqs(xe); + if (err) + return err; + + atomic_set(&xe->irq.enabled, 1); + + xe_irq_postinstall(xe); + + return devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe); +} + +static void xe_irq_msi_synchronize_irq(struct xe_device *xe) +{ + synchronize_irq(to_pci_dev(xe->drm.dev)->irq); +} + +void xe_irq_suspend(struct xe_device *xe) +{ + atomic_set(&xe->irq.enabled, 0); /* no new irqs */ + + /* flush irqs */ + if (xe_device_has_msix(xe)) + xe_irq_msix_synchronize_irq(xe); + else + xe_irq_msi_synchronize_irq(xe); + xe_irq_reset(xe); /* turn irqs off */ +} + +void xe_irq_resume(struct xe_device *xe) +{ + struct xe_gt *gt; + int id; + + /* + * lock not needed: + * 1. no irq will arrive before the postinstall + * 2. display is not yet resumed + */ + atomic_set(&xe->irq.enabled, 1); + xe_irq_reset(xe); + xe_irq_postinstall(xe); /* turn irqs on */ + + for_each_gt(gt, xe, id) + xe_irq_enable_hwe(gt); +} + +/* MSI-X related definitions and functions below. */ + +enum xe_irq_msix_static { + GUC2HOST_MSIX = 0, + DEFAULT_MSIX = XE_IRQ_DEFAULT_MSIX, + /* Must be last */ + NUM_OF_STATIC_MSIX, +}; + +static int xe_irq_msix_init(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int nvec = pci_msix_vec_count(pdev); + + if (nvec == -EINVAL) + return 0; /* MSI */ + + if (nvec < 0) { + drm_err(&xe->drm, "Failed getting MSI-X vectors count: %d\n", nvec); + return nvec; + } + + xe->irq.msix.nvec = nvec; + xa_init_flags(&xe->irq.msix.indexes, XA_FLAGS_ALLOC); + return 0; +} + +static irqreturn_t xe_irq_msix_default_hwe_handler(int irq, void *arg) +{ + unsigned int tile_id, gt_id; + struct xe_device *xe = arg; + struct xe_memirq *memirq; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_tile *tile; + struct xe_gt *gt; + + if (!atomic_read(&xe->irq.enabled)) + return IRQ_NONE; + + for_each_tile(tile, xe, tile_id) { + memirq = &tile->memirq; + if (!memirq->bo) + continue; + + for_each_gt(gt, xe, gt_id) { + if (gt->tile != tile) + continue; + + for_each_hw_engine(hwe, gt, id) + xe_memirq_hwe_handler(memirq, hwe); + } + } + + return IRQ_HANDLED; +} + +static int xe_irq_msix_alloc_vector(struct xe_device *xe, void *irq_buf, + bool dynamic_msix, u16 *msix) +{ + struct xa_limit limit; + int ret; + u32 id; + + limit = (dynamic_msix) ? XA_LIMIT(NUM_OF_STATIC_MSIX, xe->irq.msix.nvec - 1) : + XA_LIMIT(*msix, *msix); + ret = xa_alloc(&xe->irq.msix.indexes, &id, irq_buf, limit, GFP_KERNEL); + if (ret) + return ret; + + if (dynamic_msix) + *msix = id; + + return 0; +} + +static void xe_irq_msix_release_vector(struct xe_device *xe, u16 msix) +{ + xa_erase(&xe->irq.msix.indexes, msix); +} + +static int xe_irq_msix_request_irq_internal(struct xe_device *xe, irq_handler_t handler, + void *irq_buf, const char *name, u16 msix) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int ret, irq; + + irq = pci_irq_vector(pdev, msix); + if (irq < 0) + return irq; + + ret = request_irq(irq, handler, IRQF_SHARED, name, irq_buf); + if (ret < 0) + return ret; + + return 0; +} + +int xe_irq_msix_request_irq(struct xe_device *xe, irq_handler_t handler, void *irq_buf, + const char *name, bool dynamic_msix, u16 *msix) +{ + int ret; + + ret = xe_irq_msix_alloc_vector(xe, irq_buf, dynamic_msix, msix); + if (ret) + return ret; + + ret = xe_irq_msix_request_irq_internal(xe, handler, irq_buf, name, *msix); + if (ret) { + drm_err(&xe->drm, "Failed to request IRQ for MSI-X %u\n", *msix); + xe_irq_msix_release_vector(xe, *msix); + return ret; + } + + return 0; +} + +void xe_irq_msix_free_irq(struct xe_device *xe, u16 msix) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int irq; + void *irq_buf; + + irq_buf = xa_load(&xe->irq.msix.indexes, msix); + if (!irq_buf) + return; + + irq = pci_irq_vector(pdev, msix); + if (irq < 0) { + drm_err(&xe->drm, "MSI-X %u can't be released, there is no matching IRQ\n", msix); + return; + } + + free_irq(irq, irq_buf); + xe_irq_msix_release_vector(xe, msix); +} + +int xe_irq_msix_request_irqs(struct xe_device *xe) +{ + int err; + u16 msix; + + msix = GUC2HOST_MSIX; + err = xe_irq_msix_request_irq(xe, xe_irq_handler(xe), xe, + DRIVER_NAME "-guc2host", false, &msix); + if (err) + return err; + + msix = DEFAULT_MSIX; + err = xe_irq_msix_request_irq(xe, xe_irq_msix_default_hwe_handler, xe, + DRIVER_NAME "-default-msix", false, &msix); + if (err) { + xe_irq_msix_free_irq(xe, GUC2HOST_MSIX); + return err; + } + + return 0; +} + +void xe_irq_msix_free(struct xe_device *xe) +{ + unsigned long msix; + u32 *dummy; + + xa_for_each(&xe->irq.msix.indexes, msix, dummy) + xe_irq_msix_free_irq(xe, msix); + xa_destroy(&xe->irq.msix.indexes); +} + +void xe_irq_msix_synchronize_irq(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + unsigned long msix; + u32 *dummy; + + xa_for_each(&xe->irq.msix.indexes, msix, dummy) + synchronize_irq(pci_irq_vector(pdev, msix)); +} diff --git a/drivers/gpu/drm/xe/xe_irq.h b/drivers/gpu/drm/xe/xe_irq.h new file mode 100644 index 000000000..a28bd577b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_irq.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_IRQ_H_ +#define _XE_IRQ_H_ + +#include + +#define XE_IRQ_DEFAULT_MSIX 1 + +struct xe_device; +struct xe_tile; +struct xe_gt; + +int xe_irq_init(struct xe_device *xe); +int xe_irq_install(struct xe_device *xe); +void xe_irq_suspend(struct xe_device *xe); +void xe_irq_resume(struct xe_device *xe); +void xe_irq_enable_hwe(struct xe_gt *gt); +int xe_irq_msix_request_irq(struct xe_device *xe, irq_handler_t handler, void *irq_buf, + const char *name, bool dynamic_msix, u16 *msix); +void xe_irq_msix_free_irq(struct xe_device *xe, u16 msix); + +#endif diff --git a/drivers/gpu/drm/xe/xe_late_bind_fw.c b/drivers/gpu/drm/xe/xe_late_bind_fw.c new file mode 100644 index 000000000..768442ca7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_late_bind_fw.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_late_bind_fw.h" +#include "xe_pcode.h" +#include "xe_pcode_api.h" +#include "xe_pm.h" + +/* + * The component should load quite quickly in most cases, but it could take + * a bit. Using a very big timeout just to cover the worst case scenario + */ +#define LB_INIT_TIMEOUT_MS 20000 + +/* + * Retry interval set to 6 seconds, in steps of 200 ms, to allow time for + * other OS components to release the MEI CL handle + */ +#define LB_FW_LOAD_RETRY_MAXCOUNT 30 +#define LB_FW_LOAD_RETRY_PAUSE_MS 200 + +static const u32 fw_id_to_type[] = { + [XE_LB_FW_FAN_CONTROL] = INTEL_LB_TYPE_FAN_CONTROL, + }; + +static const char * const fw_id_to_name[] = { + [XE_LB_FW_FAN_CONTROL] = "fan_control", + }; + +static struct xe_device * +late_bind_to_xe(struct xe_late_bind *late_bind) +{ + return container_of(late_bind, struct xe_device, late_bind); +} + +static struct xe_device * +late_bind_fw_to_xe(struct xe_late_bind_fw *lb_fw) +{ + return container_of(lb_fw, struct xe_device, late_bind.late_bind_fw[lb_fw->id]); +} + +/* Refer to the "Late Bind based Firmware Layout" documentation entry for details */ +static int parse_cpd_header(struct xe_late_bind_fw *lb_fw, + const void *data, size_t size, const char *manifest_entry) +{ + struct xe_device *xe = late_bind_fw_to_xe(lb_fw); + const struct gsc_cpd_header_v2 *header = data; + const struct gsc_manifest_header *manifest; + const struct gsc_cpd_entry *entry; + size_t min_size = sizeof(*header); + u32 offset = 0; + int i; + + /* manifest_entry is mandatory */ + xe_assert(xe, manifest_entry); + + if (size < min_size || header->header_marker != GSC_CPD_HEADER_MARKER) + return -ENOENT; + + if (header->header_length < sizeof(struct gsc_cpd_header_v2)) { + drm_err(&xe->drm, "%s late binding fw: Invalid CPD header length %u!\n", + fw_id_to_name[lb_fw->id], header->header_length); + return -EINVAL; + } + + min_size = header->header_length + sizeof(struct gsc_cpd_entry) * header->num_of_entries; + if (size < min_size) { + drm_err(&xe->drm, "%s late binding fw: too small! %zu < %zu\n", + fw_id_to_name[lb_fw->id], size, min_size); + return -ENODATA; + } + + /* Look for the manifest first */ + entry = (void *)header + header->header_length; + for (i = 0; i < header->num_of_entries; i++, entry++) + if (strcmp(entry->name, manifest_entry) == 0) + offset = entry->offset & GSC_CPD_ENTRY_OFFSET_MASK; + + if (!offset) { + drm_err(&xe->drm, "%s late binding fw: Failed to find manifest_entry\n", + fw_id_to_name[lb_fw->id]); + return -ENODATA; + } + + min_size = offset + sizeof(struct gsc_manifest_header); + if (size < min_size) { + drm_err(&xe->drm, "%s late binding fw: too small! %zu < %zu\n", + fw_id_to_name[lb_fw->id], size, min_size); + return -ENODATA; + } + + manifest = data + offset; + + lb_fw->version = manifest->fw_version; + + return 0; +} + +/* Refer to the "Late Bind based Firmware Layout" documentation entry for details */ +static int parse_lb_layout(struct xe_late_bind_fw *lb_fw, + const void *data, size_t size, const char *fpt_entry) +{ + struct xe_device *xe = late_bind_fw_to_xe(lb_fw); + const struct csc_fpt_header *header = data; + const struct csc_fpt_entry *entry; + size_t min_size = sizeof(*header); + u32 offset = 0; + int i; + + /* fpt_entry is mandatory */ + xe_assert(xe, fpt_entry); + + if (size < min_size || header->header_marker != CSC_FPT_HEADER_MARKER) + return -ENOENT; + + if (header->header_length < sizeof(struct csc_fpt_header)) { + drm_err(&xe->drm, "%s late binding fw: Invalid FPT header length %u!\n", + fw_id_to_name[lb_fw->id], header->header_length); + return -EINVAL; + } + + min_size = header->header_length + sizeof(struct csc_fpt_entry) * header->num_of_entries; + if (size < min_size) { + drm_err(&xe->drm, "%s late binding fw: too small! %zu < %zu\n", + fw_id_to_name[lb_fw->id], size, min_size); + return -ENODATA; + } + + /* Look for the cpd header first */ + entry = (void *)header + header->header_length; + for (i = 0; i < header->num_of_entries; i++, entry++) + if (strcmp(entry->name, fpt_entry) == 0) + offset = entry->offset; + + if (!offset) { + drm_err(&xe->drm, "%s late binding fw: Failed to find fpt_entry\n", + fw_id_to_name[lb_fw->id]); + return -ENODATA; + } + + min_size = offset + sizeof(struct gsc_cpd_header_v2); + if (size < min_size) { + drm_err(&xe->drm, "%s late binding fw: too small! %zu < %zu\n", + fw_id_to_name[lb_fw->id], size, min_size); + return -ENODATA; + } + + return parse_cpd_header(lb_fw, data + offset, size - offset, "LTES.man"); +} + +static const char *xe_late_bind_parse_status(uint32_t status) +{ + switch (status) { + case INTEL_LB_STATUS_SUCCESS: + return "success"; + case INTEL_LB_STATUS_4ID_MISMATCH: + return "4Id Mismatch"; + case INTEL_LB_STATUS_ARB_FAILURE: + return "ARB Failure"; + case INTEL_LB_STATUS_GENERAL_ERROR: + return "General Error"; + case INTEL_LB_STATUS_INVALID_PARAMS: + return "Invalid Params"; + case INTEL_LB_STATUS_INVALID_SIGNATURE: + return "Invalid Signature"; + case INTEL_LB_STATUS_INVALID_PAYLOAD: + return "Invalid Payload"; + case INTEL_LB_STATUS_TIMEOUT: + return "Timeout"; + default: + return "Unknown error"; + } +} + +static int xe_late_bind_fw_num_fans(struct xe_late_bind *late_bind, u32 *num_fans) +{ + struct xe_device *xe = late_bind_to_xe(late_bind); + struct xe_tile *root_tile = xe_device_get_root_tile(xe); + + return xe_pcode_read(root_tile, + PCODE_MBOX(FAN_SPEED_CONTROL, FSC_READ_NUM_FANS, 0), num_fans, NULL); +} + +void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind) +{ + struct xe_device *xe = late_bind_to_xe(late_bind); + struct xe_late_bind_fw *lbfw; + int fw_id; + + for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) { + lbfw = &late_bind->late_bind_fw[fw_id]; + if (lbfw->payload && late_bind->wq) { + drm_dbg(&xe->drm, "Flush work: load %s firmware\n", + fw_id_to_name[lbfw->id]); + flush_work(&lbfw->work); + } + } +} + +static void xe_late_bind_work(struct work_struct *work) +{ + struct xe_late_bind_fw *lbfw = container_of(work, struct xe_late_bind_fw, work); + struct xe_late_bind *late_bind = container_of(lbfw, struct xe_late_bind, + late_bind_fw[lbfw->id]); + struct xe_device *xe = late_bind_to_xe(late_bind); + int retry = LB_FW_LOAD_RETRY_MAXCOUNT; + int ret; + int slept; + + xe_device_assert_mem_access(xe); + + /* we can queue this before the component is bound */ + for (slept = 0; slept < LB_INIT_TIMEOUT_MS; slept += 100) { + if (late_bind->component.ops) + break; + msleep(100); + } + + if (!late_bind->component.ops) { + drm_err(&xe->drm, "Late bind component not bound\n"); + /* Do not re-attempt fw load */ + drmm_kfree(&xe->drm, (void *)lbfw->payload); + lbfw->payload = NULL; + goto out; + } + + drm_dbg(&xe->drm, "Load %s firmware\n", fw_id_to_name[lbfw->id]); + + do { + ret = late_bind->component.ops->push_payload(late_bind->component.mei_dev, + lbfw->type, + lbfw->flags, + lbfw->payload, + lbfw->payload_size); + if (!ret) + break; + msleep(LB_FW_LOAD_RETRY_PAUSE_MS); + } while (--retry && ret == -EBUSY); + + if (!ret) { + drm_dbg(&xe->drm, "Load %s firmware successful\n", + fw_id_to_name[lbfw->id]); + goto out; + } + + if (ret > 0) + drm_err(&xe->drm, "Load %s firmware failed with err %d, %s\n", + fw_id_to_name[lbfw->id], ret, xe_late_bind_parse_status(ret)); + else + drm_err(&xe->drm, "Load %s firmware failed with err %d", + fw_id_to_name[lbfw->id], ret); + /* Do not re-attempt fw load */ + drmm_kfree(&xe->drm, (void *)lbfw->payload); + lbfw->payload = NULL; + +out: + xe_pm_runtime_put(xe); +} + +int xe_late_bind_fw_load(struct xe_late_bind *late_bind) +{ + struct xe_device *xe = late_bind_to_xe(late_bind); + struct xe_late_bind_fw *lbfw; + int fw_id; + + if (!late_bind->component_added) + return -ENODEV; + + if (late_bind->disable) + return 0; + + for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) { + lbfw = &late_bind->late_bind_fw[fw_id]; + if (lbfw->payload) { + xe_pm_runtime_get_noresume(xe); + queue_work(late_bind->wq, &lbfw->work); + } + } + return 0; +} + +static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) +{ + struct xe_device *xe = late_bind_to_xe(late_bind); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct xe_late_bind_fw *lb_fw; + const struct firmware *fw; + u32 num_fans; + int ret; + + if (fw_id >= XE_LB_FW_MAX_ID) + return -EINVAL; + + lb_fw = &late_bind->late_bind_fw[fw_id]; + + lb_fw->id = fw_id; + lb_fw->type = fw_id_to_type[lb_fw->id]; + lb_fw->flags &= ~INTEL_LB_FLAG_IS_PERSISTENT; + + if (lb_fw->type == INTEL_LB_TYPE_FAN_CONTROL) { + ret = xe_late_bind_fw_num_fans(late_bind, &num_fans); + if (ret) { + drm_dbg(&xe->drm, "Failed to read number of fans: %d\n", ret); + return 0; /* Not a fatal error, continue without fan control */ + } + drm_dbg(&xe->drm, "Number of Fans: %d\n", num_fans); + if (!num_fans) + return 0; + } + + snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin", + fw_id_to_name[lb_fw->id], pdev->device, + pdev->subsystem_vendor, pdev->subsystem_device); + + drm_dbg(&xe->drm, "Request late binding firmware %s\n", lb_fw->blob_path); + ret = firmware_request_nowarn(&fw, lb_fw->blob_path, xe->drm.dev); + if (ret) { + drm_dbg(&xe->drm, "%s late binding fw not available for current device", + fw_id_to_name[lb_fw->id]); + return 0; + } + + if (fw->size > XE_LB_MAX_PAYLOAD_SIZE) { + drm_err(&xe->drm, "Firmware %s size %zu is larger than max pay load size %u\n", + lb_fw->blob_path, fw->size, XE_LB_MAX_PAYLOAD_SIZE); + release_firmware(fw); + return -ENODATA; + } + + ret = parse_lb_layout(lb_fw, fw->data, fw->size, "LTES"); + if (ret) + return ret; + + lb_fw->payload_size = fw->size; + lb_fw->payload = drmm_kzalloc(&xe->drm, lb_fw->payload_size, GFP_KERNEL); + if (!lb_fw->payload) { + release_firmware(fw); + return -ENOMEM; + } + + drm_info(&xe->drm, "Using %s firmware from %s version %u.%u.%u.%u\n", + fw_id_to_name[lb_fw->id], lb_fw->blob_path, + lb_fw->version.major, lb_fw->version.minor, + lb_fw->version.hotfix, lb_fw->version.build); + + memcpy((void *)lb_fw->payload, fw->data, lb_fw->payload_size); + release_firmware(fw); + INIT_WORK(&lb_fw->work, xe_late_bind_work); + + return 0; +} + +static int xe_late_bind_fw_init(struct xe_late_bind *late_bind) +{ + int ret; + int fw_id; + + late_bind->wq = alloc_ordered_workqueue("late-bind-ordered-wq", 0); + if (!late_bind->wq) + return -ENOMEM; + + for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) { + ret = __xe_late_bind_fw_init(late_bind, fw_id); + if (ret) + return ret; + } + + return 0; +} + +static int xe_late_bind_component_bind(struct device *xe_kdev, + struct device *mei_kdev, void *data) +{ + struct xe_device *xe = kdev_to_xe_device(xe_kdev); + struct xe_late_bind *late_bind = &xe->late_bind; + + late_bind->component.ops = data; + late_bind->component.mei_dev = mei_kdev; + + return 0; +} + +static void xe_late_bind_component_unbind(struct device *xe_kdev, + struct device *mei_kdev, void *data) +{ + struct xe_device *xe = kdev_to_xe_device(xe_kdev); + struct xe_late_bind *late_bind = &xe->late_bind; + + xe_late_bind_wait_for_worker_completion(late_bind); + + late_bind->component.ops = NULL; +} + +static const struct component_ops xe_late_bind_component_ops = { + .bind = xe_late_bind_component_bind, + .unbind = xe_late_bind_component_unbind, +}; + +static void xe_late_bind_remove(void *arg) +{ + struct xe_late_bind *late_bind = arg; + struct xe_device *xe = late_bind_to_xe(late_bind); + + xe_late_bind_wait_for_worker_completion(late_bind); + + late_bind->component_added = false; + + component_del(xe->drm.dev, &xe_late_bind_component_ops); + if (late_bind->wq) { + destroy_workqueue(late_bind->wq); + late_bind->wq = NULL; + } +} + +/** + * xe_late_bind_init() - add xe mei late binding component + * @late_bind: pointer to late bind structure. + * + * Return: 0 if the initialization was successful, a negative errno otherwise. + */ +int xe_late_bind_init(struct xe_late_bind *late_bind) +{ + struct xe_device *xe = late_bind_to_xe(late_bind); + int err; + + if (!xe->info.has_late_bind) + return 0; + + if (!IS_ENABLED(CONFIG_INTEL_MEI_LB) || !IS_ENABLED(CONFIG_INTEL_MEI_GSC)) { + drm_info(&xe->drm, "Can't init xe mei late bind missing mei component\n"); + return 0; + } + + err = component_add_typed(xe->drm.dev, &xe_late_bind_component_ops, + INTEL_COMPONENT_LB); + if (err < 0) { + drm_err(&xe->drm, "Failed to add mei late bind component (%pe)\n", ERR_PTR(err)); + return err; + } + + late_bind->component_added = true; + + err = devm_add_action_or_reset(xe->drm.dev, xe_late_bind_remove, late_bind); + if (err) + return err; + + err = xe_late_bind_fw_init(late_bind); + if (err) + return err; + + return xe_late_bind_fw_load(late_bind); +} diff --git a/drivers/gpu/drm/xe/xe_late_bind_fw.h b/drivers/gpu/drm/xe/xe_late_bind_fw.h new file mode 100644 index 000000000..07e437390 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_late_bind_fw.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_LATE_BIND_FW_H_ +#define _XE_LATE_BIND_FW_H_ + +#include + +struct xe_late_bind; + +int xe_late_bind_init(struct xe_late_bind *late_bind); +int xe_late_bind_fw_load(struct xe_late_bind *late_bind); +void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind); + +#endif diff --git a/drivers/gpu/drm/xe/xe_late_bind_fw_types.h b/drivers/gpu/drm/xe/xe_late_bind_fw_types.h new file mode 100644 index 000000000..ee5efe607 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_late_bind_fw_types.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_LATE_BIND_FW_TYPES_H_ +#define _XE_LATE_BIND_FW_TYPES_H_ + +#include +#include +#include +#include + +#include "abi/uc_fw_abi.h" + +#define XE_LB_MAX_PAYLOAD_SIZE SZ_4K + +/** + * enum xe_late_bind_fw_id - enum to determine late binding fw index + */ +enum xe_late_bind_fw_id { + /** @XE_LB_FW_FAN_CONTROL: Fan control */ + XE_LB_FW_FAN_CONTROL = 0, + /** @XE_LB_FW_MAX_ID: Number of IDs */ + XE_LB_FW_MAX_ID +}; + +/** + * struct xe_late_bind_fw + */ +struct xe_late_bind_fw { + /** @id: firmware index */ + u32 id; + /** @blob_path: firmware binary path */ + char blob_path[PATH_MAX]; + /** @type: firmware type */ + u32 type; + /** @flags: firmware flags */ + u32 flags; + /** @payload: to store the late binding blob */ + const u8 *payload; + /** @payload_size: late binding blob payload_size */ + size_t payload_size; + /** @work: worker to upload latebind blob */ + struct work_struct work; + /** @version: late binding blob manifest version */ + struct gsc_version version; +}; + +/** + * struct xe_late_bind_component - Late Binding services component + * @mei_dev: device that provide Late Binding service. + * @ops: Ops implemented by Late Binding driver, used by Xe driver. + * + * Communication between Xe and MEI drivers for Late Binding services + */ +struct xe_late_bind_component { + struct device *mei_dev; + const struct intel_lb_component_ops *ops; +}; + +/** + * struct xe_late_bind + */ +struct xe_late_bind { + /** @component: struct for communication with mei component */ + struct xe_late_bind_component component; + /** @late_bind_fw: late binding firmware array */ + struct xe_late_bind_fw late_bind_fw[XE_LB_FW_MAX_ID]; + /** @wq: workqueue to submit request to download late bind blob */ + struct workqueue_struct *wq; + /** @component_added: whether the component has been added */ + bool component_added; + /** @disable: to block late binding reload during pm resume flow*/ + bool disable; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c new file mode 100644 index 000000000..0c726eda9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lmtt.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include + +#include "regs/xe_gt_regs.h" +#include "regs/xe_mert_regs.h" + +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_tlb_inval.h" +#include "xe_lmtt.h" +#include "xe_map.h" +#include "xe_mert.h" +#include "xe_mmio.h" +#include "xe_res_cursor.h" +#include "xe_sriov.h" +#include "xe_tile.h" +#include "xe_tile_sriov_printk.h" + +/** + * DOC: Local Memory Translation Table + * + * The Local Memory Translation Table (LMTT) provides additional abstraction + * when Virtual Function (VF) is accessing device Local Memory (VRAM). + * + * The Root LMTT Page Directory contains one entry for each VF. Entries are + * indexed by the function number (1-based, index 0 is unused). + * + * See `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_. + */ + +#define lmtt_assert(lmtt, condition) xe_tile_assert(lmtt_to_tile(lmtt), condition) +#define lmtt_debug(lmtt, msg...) xe_tile_sriov_dbg_verbose(lmtt_to_tile(lmtt), "LMTT: " msg) + +static bool xe_has_multi_level_lmtt(struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) >= 1260; +} + +static struct xe_tile *lmtt_to_tile(struct xe_lmtt *lmtt) +{ + return container_of(lmtt, struct xe_tile, sriov.pf.lmtt); +} + +static struct xe_device *lmtt_to_xe(struct xe_lmtt *lmtt) +{ + return tile_to_xe(lmtt_to_tile(lmtt)); +} + +static u64 lmtt_page_size(struct xe_lmtt *lmtt) +{ + return BIT_ULL(lmtt->ops->lmtt_pte_shift(0)); +} + +/** + * xe_lmtt_page_size() - Get LMTT page size. + * @lmtt: the &xe_lmtt + * + * This function shall be called only by PF. + * + * Return: LMTT page size. + */ +u64 xe_lmtt_page_size(struct xe_lmtt *lmtt) +{ + lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt))); + lmtt_assert(lmtt, xe_device_has_lmtt(lmtt_to_xe(lmtt))); + lmtt_assert(lmtt, lmtt->ops); + + return lmtt_page_size(lmtt); +} + +static struct xe_lmtt_pt *lmtt_pt_alloc(struct xe_lmtt *lmtt, unsigned int level) +{ + unsigned int num_entries = level ? lmtt->ops->lmtt_pte_num(level) : 0; + struct xe_lmtt_pt *pt; + struct xe_bo *bo; + int err; + + pt = kzalloc_flex(*pt, entries, num_entries); + if (!pt) { + err = -ENOMEM; + goto out; + } + + bo = xe_bo_create_pin_map_novm(lmtt_to_xe(lmtt), lmtt_to_tile(lmtt), + PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) * + lmtt->ops->lmtt_pte_num(level)), + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) | + XE_BO_FLAG_NEEDS_64K, false); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + goto out_free_pt; + } + + lmtt_assert(lmtt, xe_bo_is_vram(bo)); + lmtt_debug(lmtt, "level=%u addr=%#llx\n", level, (u64)xe_bo_main_addr(bo, XE_PAGE_SIZE)); + + xe_map_memset(lmtt_to_xe(lmtt), &bo->vmap, 0, 0, xe_bo_size(bo)); + + pt->level = level; + pt->bo = bo; + return pt; + +out_free_pt: + kfree(pt); +out: + return ERR_PTR(err); +} + +static void lmtt_pt_free(struct xe_lmtt_pt *pt) +{ + lmtt_debug(&pt->bo->tile->sriov.pf.lmtt, "level=%u addr=%llx\n", + pt->level, (u64)xe_bo_main_addr(pt->bo, XE_PAGE_SIZE)); + + xe_bo_unpin_map_no_vm(pt->bo); + kfree(pt); +} + +static int lmtt_init_pd(struct xe_lmtt *lmtt) +{ + struct xe_lmtt_pt *pd; + + lmtt_assert(lmtt, !lmtt->pd); + lmtt_assert(lmtt, lmtt->ops->lmtt_root_pd_level()); + + pd = lmtt_pt_alloc(lmtt, lmtt->ops->lmtt_root_pd_level()); + if (IS_ERR(pd)) + return PTR_ERR(pd); + + lmtt->pd = pd; + return 0; +} + +static void lmtt_fini_pd(struct xe_lmtt *lmtt) +{ + struct xe_lmtt_pt *pd = lmtt->pd; + unsigned int num_entries = lmtt->ops->lmtt_pte_num(pd->level); + unsigned int n = 0; + + /* make sure we don't leak */ + for (n = 0; n < num_entries; n++) + lmtt_assert(lmtt, !pd->entries[n]); + + lmtt->pd = NULL; + lmtt_pt_free(pd); +} + +static void fini_lmtt(struct drm_device *drm, void *arg) +{ + struct xe_lmtt *lmtt = arg; + + lmtt_assert(lmtt, !(!!lmtt->ops ^ !!lmtt->pd)); + + if (!lmtt->pd) + return; + + lmtt_fini_pd(lmtt); + lmtt->ops = NULL; +} + +/** + * xe_lmtt_init - LMTT software initialization. + * @lmtt: the &xe_lmtt to initialize + * + * The LMTT initialization requires two steps. + * + * The xe_lmtt_init() checks if LMTT is required on current device and selects + * and initialize proper variant of the LMTT Root Directory. Currently supported + * variants are `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_. + * + * In next step xe_lmtt_init_hw() will register this directory on the hardware. + * + * Notes: + * The LMTT allocations are managed and will be implicitly released on driver unload. + * This function shall be called only once and only when running as a PF driver. + * Any LMTT initialization failure should block VFs enabling. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_lmtt_init(struct xe_lmtt *lmtt) +{ + struct xe_device *xe = lmtt_to_xe(lmtt); + int err; + + lmtt_assert(lmtt, IS_SRIOV_PF(xe)); + lmtt_assert(lmtt, !lmtt->ops); + + if (!xe_device_has_lmtt(xe)) + return 0; + + if (xe_has_multi_level_lmtt(xe)) + lmtt->ops = &lmtt_ml_ops; + else + lmtt->ops = &lmtt_2l_ops; + + err = lmtt_init_pd(lmtt); + if (unlikely(err)) + goto fail; + + return drmm_add_action_or_reset(&xe->drm, fini_lmtt, lmtt); + +fail: + lmtt->ops = NULL; + return err; +} + +static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt) +{ + struct xe_tile *tile = lmtt_to_tile(lmtt); + struct xe_device *xe = tile_to_xe(tile); + dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE); + struct xe_gt *gt; + u32 config; + u8 id; + + lmtt_debug(lmtt, "DIR offset %pad\n", &offset); + lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo)); + lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K)); + + config = LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K); + + for_each_gt_on_tile(gt, tile, id) + xe_mmio_write32(>->mmio, + GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG, + config); + + if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) + xe_mmio_write32(&tile->mmio, MERT_LMEM_CFG, config); +} + +/** + * xe_lmtt_init_hw - Perform LMTT hardware initialization. + * @lmtt: the &xe_lmtt to initialize + * + * This function is a second step of the LMTT initialization. + * This function registers LMTT Root Directory prepared in xe_lmtt_init(). + * + * This function shall be called after every hardware reset. + * This function shall be called only when running as a PF driver. + */ +void xe_lmtt_init_hw(struct xe_lmtt *lmtt) +{ + if (!lmtt->pd) + return; + + lmtt_setup_dir_ptr(lmtt); +} + +static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) +{ + struct xe_tlb_inval_fence fences[XE_MAX_GT_PER_TILE]; + struct xe_tlb_inval_fence *fence = fences; + struct xe_tile *tile = lmtt_to_tile(lmtt); + struct xe_gt *gt; + int result = 0; + int err; + u8 id; + + for_each_gt_on_tile(gt, tile, id) { + xe_tlb_inval_fence_init(>->tlb_inval, fence, true); + err = xe_tlb_inval_all(>->tlb_inval, fence); + result = result ?: err; + fence++; + } + + lmtt_debug(lmtt, "num_fences=%d err=%d\n", (int)(fence - fences), result); + + /* + * It is fine to wait for all fences, even for those which covers the + * invalidation request that failed, as such fence should be already + * marked as signaled. + */ + fence = fences; + for_each_gt_on_tile(gt, tile, id) + xe_tlb_inval_fence_wait(fence++); + + return result; +} + +/** + * xe_lmtt_invalidate_hw - Invalidate LMTT hardware. + * @lmtt: the &xe_lmtt to invalidate + * + * Send requests to all GuCs on this tile to invalidate all TLBs. + * If the platform has a standalone MERT, also invalidate MERT's TLB. + * + * This function should be called only when running as a PF driver. + */ +void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt) +{ + struct xe_tile *tile = lmtt_to_tile(lmtt); + struct xe_device *xe = lmtt_to_xe(lmtt); + int err; + + lmtt_assert(lmtt, IS_SRIOV_PF(xe)); + + err = lmtt_invalidate_hw(lmtt); + if (err) + xe_tile_sriov_err(tile, "LMTT invalidation failed (%pe)", + ERR_PTR(err)); + + if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) { + err = xe_mert_invalidate_lmtt(xe); + if (err) + xe_tile_sriov_err(tile, "MERT LMTT invalidation failed (%pe)", + ERR_PTR(err)); + } +} + +static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt, + u64 pte, unsigned int idx) +{ + unsigned int level = pt->level; + + lmtt_assert(lmtt, idx <= lmtt->ops->lmtt_pte_num(level)); + lmtt_debug(lmtt, "WRITE level=%u index=%u pte=%#llx\n", level, idx, pte); + + switch (lmtt->ops->lmtt_pte_size(level)) { + case sizeof(u32): + lmtt_assert(lmtt, !overflows_type(pte, u32)); + lmtt_assert(lmtt, !pte || !iosys_map_rd(&pt->bo->vmap, idx * sizeof(u32), u32)); + + xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u32), u32, pte); + break; + case sizeof(u64): + lmtt_assert(lmtt, !pte || !iosys_map_rd(&pt->bo->vmap, idx * sizeof(u64), u64)); + + xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u64), u64, pte); + break; + default: + lmtt_assert(lmtt, !!!"invalid pte size"); + } +} + +static void lmtt_destroy_pt(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd) +{ + unsigned int num_entries = pd->level ? lmtt->ops->lmtt_pte_num(pd->level) : 0; + struct xe_lmtt_pt *pt; + unsigned int i; + + for (i = 0; i < num_entries; i++) { + pt = pd->entries[i]; + pd->entries[i] = NULL; + if (!pt) + continue; + + lmtt_destroy_pt(lmtt, pt); + } + + lmtt_pt_free(pd); +} + +static void lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid) +{ + struct xe_lmtt_pt *pd = lmtt->pd; + struct xe_lmtt_pt *pt; + + pt = pd->entries[vfid]; + pd->entries[vfid] = NULL; + if (!pt) + return; + + lmtt_write_pte(lmtt, pd, LMTT_PTE_INVALID, vfid); + lmtt_invalidate_hw(lmtt); + + lmtt_assert(lmtt, pd->level > 0); + lmtt_assert(lmtt, pt->level == pd->level - 1); + lmtt_destroy_pt(lmtt, pt); +} + +static int __lmtt_alloc_range(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd, + u64 start, u64 end) +{ + u64 pte_addr_shift = BIT_ULL(lmtt->ops->lmtt_pte_shift(pd->level)); + u64 offset; + int err; + + lmtt_assert(lmtt, pd->level > 0); + + offset = start; + while (offset < end) { + struct xe_lmtt_pt *pt; + u64 next, pde, pt_addr; + unsigned int idx; + + pt = lmtt_pt_alloc(lmtt, pd->level - 1); + if (IS_ERR(pt)) + return PTR_ERR(pt); + + pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE); + + idx = lmtt->ops->lmtt_pte_index(offset, pd->level); + pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level); + + lmtt_write_pte(lmtt, pd, pde, idx); + + pd->entries[idx] = pt; + + next = min(end, round_up(offset + 1, pte_addr_shift)); + + if (pt->level != 0) { + err = __lmtt_alloc_range(lmtt, pt, offset, next); + if (err) + return err; + } + + offset = next; + } + + return 0; +} + +static int lmtt_alloc_range(struct xe_lmtt *lmtt, unsigned int vfid, u64 start, u64 end) +{ + struct xe_lmtt_pt *pd = lmtt->pd; + struct xe_lmtt_pt *pt; + u64 pt_addr; + u64 pde; + int err; + + lmtt_assert(lmtt, pd->level > 0); + lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level)); + lmtt_assert(lmtt, IS_ALIGNED(start, lmtt_page_size(lmtt))); + lmtt_assert(lmtt, IS_ALIGNED(end, lmtt_page_size(lmtt))); + + if (pd->entries[vfid]) + return -ENOTEMPTY; + + pt = lmtt_pt_alloc(lmtt, pd->level - 1); + if (IS_ERR(pt)) + return PTR_ERR(pt); + + pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE); + + pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level); + + lmtt_write_pte(lmtt, pd, pde, vfid); + + pd->entries[vfid] = pt; + + if (pt->level != 0) { + err = __lmtt_alloc_range(lmtt, pt, start, end); + if (err) + goto out_free_pt; + } + + return 0; + +out_free_pt: + lmtt_pt_free(pt); + return err; +} + +static struct xe_lmtt_pt *lmtt_leaf_pt(struct xe_lmtt *lmtt, unsigned int vfid, u64 addr) +{ + struct xe_lmtt_pt *pd = lmtt->pd; + struct xe_lmtt_pt *pt; + + lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level)); + pt = pd->entries[vfid]; + + while (pt->level) { + lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <= + lmtt->ops->lmtt_pte_num(pt->level)); + + pt = pt->entries[lmtt->ops->lmtt_pte_index(addr, pt->level)]; + + addr >>= lmtt->ops->lmtt_pte_shift(pt->level); + } + + lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <= + lmtt->ops->lmtt_pte_num(pt->level)); + lmtt_assert(lmtt, pt->level != pd->level); + lmtt_assert(lmtt, pt->level == 0); + return pt; +} + +static void lmtt_insert_bo(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 start) +{ + u64 page_size = lmtt_page_size(lmtt); + struct xe_res_cursor cur; + struct xe_lmtt_pt *pt; + u64 addr, vram_offset; + + lmtt_assert(lmtt, IS_ALIGNED(start, page_size)); + lmtt_assert(lmtt, IS_ALIGNED(xe_bo_size(bo), page_size)); + lmtt_assert(lmtt, xe_bo_is_vram(bo)); + + vram_offset = vram_region_gpu_offset(bo->ttm.resource); + xe_res_first(bo->ttm.resource, 0, xe_bo_size(bo), &cur); + while (cur.remaining) { + addr = xe_res_dma(&cur); + addr += vram_offset; /* XXX */ + + pt = lmtt_leaf_pt(lmtt, vfid, start); + + lmtt_write_pte(lmtt, pt, lmtt->ops->lmtt_pte_encode(addr, 0), + lmtt->ops->lmtt_pte_index(start, 0)); + + xe_res_next(&cur, page_size); + start += page_size; + } +} + +/** + * xe_lmtt_prepare_pages - Create VF's LMTT Page Tables. + * @lmtt: the &xe_lmtt to update + * @vfid: the VF identifier (1-based) + * @range: top range of LMEM offset to be supported + * + * This function creates empty LMTT page tables for given VF to support + * up to maximum #range LMEM offset. The LMTT page tables created by this + * function must be released using xe_lmtt_drop_pages() function. + * + * Notes: + * This function shall be called only after successful LMTT initialization. + * See xe_lmtt_init(). + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_lmtt_prepare_pages(struct xe_lmtt *lmtt, unsigned int vfid, u64 range) +{ + lmtt_assert(lmtt, lmtt->pd); + lmtt_assert(lmtt, vfid); + + return lmtt_alloc_range(lmtt, vfid, 0, range); +} + +/** + * xe_lmtt_populate_pages - Update VF's LMTT Page Table Entries. + * @lmtt: the &xe_lmtt to update + * @vfid: the VF identifier (1-based) + * @bo: the buffer object with LMEM allocation to be mapped + * @offset: the offset at which #bo should be mapped + * + * This function updates VF's LMTT entries to use given buffer object as a backstore. + * + * Notes: + * This function shall be called only after successful preparation of the + * VF's LMTT Page Tables. See xe_lmtt_prepare(). + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_lmtt_populate_pages(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 offset) +{ + lmtt_assert(lmtt, lmtt->pd); + lmtt_assert(lmtt, vfid); + + lmtt_insert_bo(lmtt, vfid, bo, offset); + return 0; +} + +/** + * xe_lmtt_drop_pages - Remove VF's LMTT Pages. + * @lmtt: the &xe_lmtt to update + * @vfid: the VF identifier (1-based) + * + * This function removes all LMTT Page Tables prepared by xe_lmtt_prepare_pages(). + * + * This function shall be called only after successful LMTT initialization. + * See xe_lmtt_init(). + */ +void xe_lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid) +{ + lmtt_assert(lmtt, lmtt->pd); + lmtt_assert(lmtt, vfid); + + lmtt_drop_pages(lmtt, vfid); +} + +/** + * xe_lmtt_estimate_pt_size - Estimate size of LMTT PT allocations. + * @lmtt: the &xe_lmtt + * @size: the size of the LMEM to be mapped over LMTT (including any offset) + * + * This function shall be called only by PF. + * + * Return: size of the PT allocation(s) needed to support given LMEM size. + */ +u64 xe_lmtt_estimate_pt_size(struct xe_lmtt *lmtt, u64 size) +{ + unsigned int level = 0; + u64 pt_size; + + lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt))); + lmtt_assert(lmtt, xe_device_has_lmtt(lmtt_to_xe(lmtt))); + lmtt_assert(lmtt, lmtt->ops); + + pt_size = PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) * + lmtt->ops->lmtt_pte_num(level)); + + while (++level < lmtt->ops->lmtt_root_pd_level()) { + pt_size *= lmtt->ops->lmtt_pte_index(size, level) + 1; + pt_size += PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) * + lmtt->ops->lmtt_pte_num(level)); + } + + return pt_size; +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_lmtt_test.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_lmtt.h b/drivers/gpu/drm/xe/xe_lmtt.h new file mode 100644 index 000000000..8fa387b38 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lmtt.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_LMTT_H_ +#define _XE_LMTT_H_ + +#include + +struct xe_bo; +struct xe_lmtt; +struct xe_lmtt_ops; + +#ifdef CONFIG_PCI_IOV +int xe_lmtt_init(struct xe_lmtt *lmtt); +void xe_lmtt_init_hw(struct xe_lmtt *lmtt); +void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt); +int xe_lmtt_prepare_pages(struct xe_lmtt *lmtt, unsigned int vfid, u64 range); +int xe_lmtt_populate_pages(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 offset); +void xe_lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid); +u64 xe_lmtt_estimate_pt_size(struct xe_lmtt *lmtt, u64 size); +u64 xe_lmtt_page_size(struct xe_lmtt *lmtt); +#else +static inline int xe_lmtt_init(struct xe_lmtt *lmtt) { return 0; } +static inline void xe_lmtt_init_hw(struct xe_lmtt *lmtt) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_lmtt_2l.c b/drivers/gpu/drm/xe/xe_lmtt_2l.c new file mode 100644 index 000000000..84bc5c421 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lmtt_2l.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include +#include + +#include "xe_lmtt_types.h" +#include "xe_macros.h" + +/** + * DOC: Two-Level LMTT Structure + * + * LMHAW (Local Memory Host Address Width) is 37 bit (128GB) + * + * LMGAW (Local Memory Guest Address Width) is 37 bit (128GB) + * + * The following figure illustrates the structure and function of the 2L LMTT:: + * + * LMTT Directory + * (1 Entry per VF) + * +-----------+ LMTT (per VF) + * | | +-----------+ + * | | | | + * | | index: | | + * | | LMEM VF +===========+ + * | | offset --> | PTE | ==> LMEM PF offset + * | | +===========+ + * index: +===========+ | | + * VFID --> | PDE | -----------------> +-----------+ + * +===========+ / \. + * | | / \. + * | | / \. + * | | / \. + * +-----------+ <== [LMTT Directory Ptr] \. + * / \ / \. + * / \ +-----------+-----------------+------+---+ + * / \ | 31:HAW-16 | HAW-17:5 | 4:1 | 0 | + * / \ +===========+=================+======+===+ + * / \ | Reserved | LMEM Page (2MB) | Rsvd | V | + * / \ +-----------+-----------------+------+---+ + * / \. + * +-----------+-----------------+------+---+ + * | 31:HAW-12 | HAW-13:4 | 3:1 | 0 | + * +===========+=================+======+===+ + * | Reserved | LMTT Ptr (64KB) | Rsvd | V | + * +-----------+-----------------+------+---+ + * + */ + +typedef u32 lmtt_2l_pde_t; +typedef u32 lmtt_2l_pte_t; + +#if IS_ENABLED(CONFIG_DRM_XE_LMTT_2L_128GB) +#define LMTT_2L_HAW 37 /* 128 GiB */ +#else +#define LMTT_2L_HAW 35 /* 32 GiB */ +#endif + +#define LMTT_2L_PDE_MAX_NUM 64 /* SRIOV with PF and 63 VFs, index 0 (PF) is unused */ +#define LMTT_2L_PDE_LMTT_PTR GENMASK(LMTT_2L_HAW - 13, 4) +#define LMTT_2L_PDE_VALID BIT(0) + +#define LMTT_2L_PTE_MAX_NUM BIT(LMTT_2L_HAW - ilog2(SZ_2M)) +#define LMTT_2L_PTE_LMEM_PAGE GENMASK(LMTT_2L_HAW - 17, 5) +#define LMTT_2L_PTE_VALID BIT(0) + +static unsigned int lmtt_2l_root_pd_level(void) +{ + return 1; /* implementation is 0-based */ +} + +static unsigned int lmtt_2l_pte_num(unsigned int level) +{ + switch (level) { + case 1: + return LMTT_2L_PDE_MAX_NUM; + case 0: + BUILD_BUG_ON(LMTT_2L_HAW == 37 && LMTT_2L_PTE_MAX_NUM != SZ_64K); + BUILD_BUG_ON(LMTT_2L_HAW == 35 && LMTT_2L_PTE_MAX_NUM != SZ_16K); + return LMTT_2L_PTE_MAX_NUM; + default: + return 0; + } +} + +static unsigned int lmtt_2l_pte_size(unsigned int level) +{ + switch (level) { + case 1: + return sizeof(lmtt_2l_pde_t); + case 0: + return sizeof(lmtt_2l_pte_t); + default: + return 0; + } +} + +static unsigned int lmtt_2l_pte_shift(unsigned int level) +{ + switch (level) { + case 0: + return ilog2(SZ_2M); + default: + return 0; + } +} + +static unsigned int lmtt_2l_pte_index(u64 addr, unsigned int level) +{ + addr >>= lmtt_2l_pte_shift(level); + + switch (level) { + case 0: + /* SZ_2M increments */ + BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_2L_PTE_MAX_NUM); + return addr & (LMTT_2L_PTE_MAX_NUM - 1); + default: + return 0; + } +} + +static u64 lmtt_2l_pte_encode(unsigned long offset, unsigned int level) +{ + switch (level) { + case 0: + XE_WARN_ON(!IS_ALIGNED(offset, SZ_2M)); + XE_WARN_ON(!FIELD_FIT(LMTT_2L_PTE_LMEM_PAGE, offset / SZ_2M)); + return FIELD_PREP(LMTT_2L_PTE_LMEM_PAGE, offset / SZ_2M) | LMTT_2L_PTE_VALID; + case 1: + XE_WARN_ON(!IS_ALIGNED(offset, SZ_64K)); + XE_WARN_ON(!FIELD_FIT(LMTT_2L_PDE_LMTT_PTR, offset / SZ_64K)); + return FIELD_PREP(LMTT_2L_PDE_LMTT_PTR, offset / SZ_64K) | LMTT_2L_PDE_VALID; + default: + XE_WARN_ON(true); + return 0; + } +} + +const struct xe_lmtt_ops lmtt_2l_ops = { + .lmtt_root_pd_level = lmtt_2l_root_pd_level, + .lmtt_pte_num = lmtt_2l_pte_num, + .lmtt_pte_size = lmtt_2l_pte_size, + .lmtt_pte_shift = lmtt_2l_pte_shift, + .lmtt_pte_index = lmtt_2l_pte_index, + .lmtt_pte_encode = lmtt_2l_pte_encode, +}; diff --git a/drivers/gpu/drm/xe/xe_lmtt_ml.c b/drivers/gpu/drm/xe/xe_lmtt_ml.c new file mode 100644 index 000000000..b21215a2e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lmtt_ml.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include +#include + +#include "xe_lmtt_types.h" +#include "xe_macros.h" + +/** + * DOC: Multi-Level LMTT Structure + * + * LMHAW (Local Memory Host Address Width) is 48 bit (256TB) + * + * LMGAW (Local Memory Guest Address Width) is 48 bit (256TB) + * + * The following figure illustrates the structure and function of the ML LMTT:: + * + * LMTT L3 Directory + * (1 Entry per VF) LMTT L1 Leaf + * +-----------+ +-----------+ + * | | LMTT L2 (per VF) | | + * | | +-----------+ | | + * | | | | index: +===========+ + * | | | | GDPA --> | PTE | => LMEM PF offset + * | | | | 34:21 +===========+ + * | | index: | | | | + * | | LMEM VF +===========+ | | + * | | offset -> | PTE | ----------> +-----------+ + * | | GAW-1:35 +===========+ / \. + * index: +===========+ | | / \. + * VFID --> | PDE | ---------> +-----------+ / \. + * +===========+ / / / \. + * | | / / / \. + * +-----------+ <== [LMTT Directory Ptr] / \. + * / \ / / / \. + * / \ / / +-----------+-----------------+------+---+ + * / /\ / | 31:HAW-16 | HAW-17:5 | 4:1 | 0 | + * / / \ / +===========+=================+======+===+ + * / / \ / | Reserved | LMEM Page (2MB) | Rsvd | V | + * / / +-----------+-----------------+------+---+ + * / / + * +-----------+-----------------+------+---+ + * | 63:HAW-12 | HAW-13:4 | 3:1 | 0 | + * +===========+=================+======+===+ + * | Reserved | LMTT Ptr (64KB) | Rsvd | V | + * +-----------+-----------------+------+---+ + * + */ + +typedef u64 lmtt_ml_pde_t; +typedef u32 lmtt_ml_pte_t; + +#define LMTT_ML_HAW 48 /* 256 TiB */ + +#define LMTT_ML_PDE_MAX_NUM 64 /* SRIOV with PF and 63 VFs, index 0 (PF) is unused */ +#define LMTT_ML_PDE_LMTT_PTR GENMASK_ULL(LMTT_ML_HAW - 13, 4) +#define LMTT_ML_PDE_VALID BIT(0) + +#define LMTT_ML_PDE_L2_SHIFT 35 +#define LMTT_ML_PDE_L2_MAX_NUM BIT_ULL(LMTT_ML_HAW - 35) + +#define LMTT_ML_PTE_MAX_NUM BIT(35 - ilog2(SZ_2M)) +#define LMTT_ML_PTE_LMEM_PAGE GENMASK(LMTT_ML_HAW - 17, 5) +#define LMTT_ML_PTE_VALID BIT(0) + +static unsigned int lmtt_ml_root_pd_level(void) +{ + return 2; /* implementation is 0-based */ +} + +static unsigned int lmtt_ml_pte_num(unsigned int level) +{ + switch (level) { + case 2: + return LMTT_ML_PDE_MAX_NUM; + case 1: + BUILD_BUG_ON(LMTT_ML_HAW == 48 && LMTT_ML_PDE_L2_MAX_NUM != SZ_8K); + return LMTT_ML_PDE_L2_MAX_NUM; + case 0: + BUILD_BUG_ON(LMTT_ML_PTE_MAX_NUM != SZ_16K); + return LMTT_ML_PTE_MAX_NUM; + default: + return 0; + } +} + +static unsigned int lmtt_ml_pte_size(unsigned int level) +{ + switch (level) { + case 2: + case 1: + return sizeof(lmtt_ml_pde_t); + case 0: + return sizeof(lmtt_ml_pte_t); + default: + return 0; + } +} + +static unsigned int lmtt_ml_pte_shift(unsigned int level) +{ + switch (level) { + case 1: + BUILD_BUG_ON(BIT_ULL(LMTT_ML_PDE_L2_SHIFT) != SZ_32G); + return ilog2(SZ_32G); + case 0: + return ilog2(SZ_2M); + default: + return 0; + } +} + +static unsigned int lmtt_ml_pte_index(u64 addr, unsigned int level) +{ + addr >>= lmtt_ml_pte_shift(level); + + switch (level) { + case 1: + /* SZ_32G increments */ + BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_ML_PDE_L2_MAX_NUM); + return addr & (LMTT_ML_PDE_L2_MAX_NUM - 1); + case 0: + /* SZ_2M increments */ + BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_ML_PTE_MAX_NUM); + return addr & (LMTT_ML_PTE_MAX_NUM - 1); + default: + return 0; + } +} + +static u64 lmtt_ml_pte_encode(unsigned long offset, unsigned int level) +{ + switch (level) { + case 0: + XE_WARN_ON(!IS_ALIGNED(offset, SZ_2M)); + XE_WARN_ON(!FIELD_FIT(LMTT_ML_PTE_LMEM_PAGE, offset / SZ_2M)); + return FIELD_PREP(LMTT_ML_PTE_LMEM_PAGE, offset / SZ_2M) | LMTT_ML_PTE_VALID; + case 1: + case 2: + XE_WARN_ON(!IS_ALIGNED(offset, SZ_64K)); + XE_WARN_ON(!FIELD_FIT(LMTT_ML_PDE_LMTT_PTR, offset / SZ_64K)); + return FIELD_PREP(LMTT_ML_PDE_LMTT_PTR, offset / SZ_64K) | LMTT_ML_PDE_VALID; + default: + XE_WARN_ON(true); + return 0; + } +} + +const struct xe_lmtt_ops lmtt_ml_ops = { + .lmtt_root_pd_level = lmtt_ml_root_pd_level, + .lmtt_pte_num = lmtt_ml_pte_num, + .lmtt_pte_size = lmtt_ml_pte_size, + .lmtt_pte_shift = lmtt_ml_pte_shift, + .lmtt_pte_index = lmtt_ml_pte_index, + .lmtt_pte_encode = lmtt_ml_pte_encode, +}; diff --git a/drivers/gpu/drm/xe/xe_lmtt_types.h b/drivers/gpu/drm/xe/xe_lmtt_types.h new file mode 100644 index 000000000..b37abad23 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lmtt_types.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_LMTT_TYPES_H_ +#define _XE_LMTT_TYPES_H_ + +#include + +struct xe_bo; +struct xe_lmtt; +struct xe_lmtt_pt; +struct xe_lmtt_ops; + +#define LMTT_PTE_INVALID ULL(0) + +/** + * struct xe_lmtt - Local Memory Translation Table Manager + */ +struct xe_lmtt { + /** @pd: root LMTT Directory */ + struct xe_lmtt_pt *pd; + + /** @ops: LMTT functions */ + const struct xe_lmtt_ops *ops; +}; + +/** + * struct xe_lmtt_pt - Local Memory Translation Table Page Table + * + * Represents single level of the LMTT. + */ +struct xe_lmtt_pt { + /** @level: page table level, 0 is leaf */ + unsigned int level; + + /** @bo: buffer object with actual LMTT PTE values */ + struct xe_bo *bo; + + /** @entries: leaf page tables, exist only for root/non-leaf */ + struct xe_lmtt_pt *entries[]; +}; + +/** + * struct xe_lmtt_ops - Local Memory Translation Table Operations + * + * Provides abstraction of the LMTT variants. + */ +struct xe_lmtt_ops { + /* private: */ + unsigned int (*lmtt_root_pd_level)(void); + unsigned int (*lmtt_pte_num)(unsigned int level); + unsigned int (*lmtt_pte_size)(unsigned int level); + unsigned int (*lmtt_pte_shift)(unsigned int level); + unsigned int (*lmtt_pte_index)(u64 addr, unsigned int level); + u64 (*lmtt_pte_encode)(unsigned long offset, unsigned int level); +}; + +extern const struct xe_lmtt_ops lmtt_2l_ops; +extern const struct xe_lmtt_ops lmtt_ml_ops; + +#endif diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c new file mode 100644 index 000000000..35b4e8289 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -0,0 +1,2818 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_lrc.h" + +#include + +#include +#include + +#include "instructions/xe_mi_commands.h" +#include "instructions/xe_gfxpipe_commands.h" +#include "instructions/xe_gfx_state_commands.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_lrc_layout.h" +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_drm_client.h" +#include "xe_exec_queue.h" +#include "xe_exec_queue_types.h" +#include "xe_gt.h" +#include "xe_gt_clock.h" +#include "xe_gt_printk.h" +#include "xe_hw_fence.h" +#include "xe_map.h" +#include "xe_memirq.h" +#include "xe_mmio.h" +#include "xe_ring_ops.h" +#include "xe_sriov.h" +#include "xe_trace_lrc.h" +#include "xe_vm.h" +#include "xe_wa.h" + +#define LRC_VALID BIT_ULL(0) +#define LRC_PRIVILEGE BIT_ULL(8) +#define LRC_ADDRESSING_MODE GENMASK_ULL(4, 3) +#define LRC_LEGACY_64B_CONTEXT 3 + +#define LRC_ENGINE_CLASS GENMASK_ULL(63, 61) +#define LRC_ENGINE_INSTANCE GENMASK_ULL(53, 48) + +#define LRC_PPHWSP_SIZE SZ_4K +#define LRC_INDIRECT_CTX_BO_SIZE SZ_4K +#define LRC_INDIRECT_RING_STATE_SIZE SZ_4K + +#define LRC_PRIORITY GENMASK_ULL(10, 9) +#define LRC_PRIORITY_LOW 0 +#define LRC_PRIORITY_NORMAL 1 +#define LRC_PRIORITY_HIGH 2 + +/* + * Layout of the LRC and associated data allocated as + * lrc->bo: + * + * Region Size + * +============================+=================================+ <- __xe_lrc_ring_offset() + * | Ring | ring_size, see | + * | | xe_lrc_init() | + * +============================+=================================+ <- __xe_lrc_pphwsp_offset() + * | PPHWSP (includes SW state) | 4K | + * +----------------------------+---------------------------------+ <- __xe_lrc_regs_offset() + * | Engine Context Image | n * 4K, see | + * | | xe_gt_lrc_size() | + * +----------------------------+---------------------------------+ <- __xe_lrc_indirect_ring_offset() + * | Indirect Ring State Page | 0 or 4k, see | + * | | XE_LRC_FLAG_INDIRECT_RING_STATE | + * +============================+=================================+ <- __xe_lrc_indirect_ctx_offset() + * | Indirect Context Page | 0 or 4k, see | + * | | XE_LRC_FLAG_INDIRECT_CTX | + * +============================+=================================+ <- __xe_lrc_wa_bb_offset() + * | WA BB Per Ctx | 4k | + * +============================+=================================+ <- xe_bo_size(lrc->bo) + */ + +static struct xe_device * +lrc_to_xe(struct xe_lrc *lrc) +{ + return gt_to_xe(lrc->fence_ctx.gt); +} + +static bool +gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (XE_GT_WA(gt, 16010904313) && + (class == XE_ENGINE_CLASS_RENDER || + class == XE_ENGINE_CLASS_COMPUTE)) + return true; + + if (xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev), + class, NULL)) + return true; + + if (gt->ring_ops[class]->emit_aux_table_inv) + return true; + + return false; +} + +/** + * xe_gt_lrc_hang_replay_size() - Hang replay size + * @gt: The GT + * @class: Hardware engine class + * + * Determine size of GPU hang replay state for a GT and hardware engine class. + * + * Return: Size of GPU hang replay size + */ +size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class) +{ + struct xe_device *xe = gt_to_xe(gt); + size_t size = 0; + + /* Engine context image */ + switch (class) { + case XE_ENGINE_CLASS_RENDER: + if (GRAPHICS_VERx100(xe) >= 3510) + size += 7 * SZ_4K; + else if (GRAPHICS_VER(xe) >= 20) + size += 3 * SZ_4K; + else + size += 13 * SZ_4K; + break; + case XE_ENGINE_CLASS_COMPUTE: + if (GRAPHICS_VERx100(xe) >= 3510) + size += 5 * SZ_4K; + else if (GRAPHICS_VER(xe) >= 20) + size += 2 * SZ_4K; + else + size += 13 * SZ_4K; + break; + default: + WARN(1, "Unknown engine class: %d", class); + fallthrough; + case XE_ENGINE_CLASS_COPY: + case XE_ENGINE_CLASS_VIDEO_DECODE: + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + case XE_ENGINE_CLASS_OTHER: + size += 1 * SZ_4K; + } + + return size; +} + +size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) +{ + size_t size = xe_gt_lrc_hang_replay_size(gt, class); + + /* Add indirect ring state page */ + if (xe_gt_has_indirect_ring_state(gt)) + size += LRC_INDIRECT_RING_STATE_SIZE; + + return size + LRC_PPHWSP_SIZE; +} + +/* + * The per-platform tables are u8-encoded in @data. Decode @data and set the + * addresses' offset and commands in @regs. The following encoding is used + * for each byte. There are 2 steps: decoding commands and decoding addresses. + * + * Commands: + * [7]: create NOPs - number of NOPs are set in lower bits + * [6]: When creating MI_LOAD_REGISTER_IMM command, allow to set + * MI_LRI_FORCE_POSTED + * [5:0]: Number of NOPs or registers to set values to in case of + * MI_LOAD_REGISTER_IMM + * + * Addresses: these are decoded after a MI_LOAD_REGISTER_IMM command by "count" + * number of registers. They are set by using the REG/REG16 macros: the former + * is used for offsets smaller than 0x200 while the latter is for values bigger + * than that. Those macros already set all the bits documented below correctly: + * + * [7]: When a register offset needs more than 6 bits, use additional bytes, to + * follow, for the lower bits + * [6:0]: Register offset, without considering the engine base. + * + * This function only tweaks the commands and register offsets. Values are not + * filled out. + */ +static void set_offsets(u32 *regs, + const u8 *data, + const struct xe_hw_engine *hwe) +#define NOP(x) (BIT(7) | (x)) +#define LRI(count, flags) ((flags) << 6 | (count) | \ + BUILD_BUG_ON_ZERO(count >= BIT(6))) +#define POSTED BIT(0) +#define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) +#define REG16(x) \ + (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ + (((x) >> 2) & 0x7f) +{ + const u32 base = hwe->mmio_base; + + while (*data) { + u8 count, flags; + + if (*data & BIT(7)) { /* skip */ + count = *data++ & ~BIT(7); + regs += count; + continue; + } + + count = *data & 0x3f; + flags = *data >> 6; + data++; + + *regs = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count); + if (flags & POSTED) + *regs |= MI_LRI_FORCE_POSTED; + *regs |= MI_LRI_LRM_CS_MMIO; + regs++; + + xe_gt_assert(hwe->gt, count); + do { + u32 offset = 0; + u8 v; + + do { + v = *data++; + offset <<= 7; + offset |= v & ~BIT(7); + } while (v & BIT(7)); + + regs[0] = base + (offset << 2); + regs += 2; + } while (--count); + } + + *regs = MI_BATCH_BUFFER_END | BIT(0); +} + +static const u8 gen12_xcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + 0 +}; + +static const u8 dg2_xcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + 0 +}; + +static const u8 gen12_rcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + NOP(3 + 9 + 1), + + LRI(51, POSTED), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG(0x028), + REG(0x09c), + REG(0x0c0), + REG(0x178), + REG(0x17c), + REG16(0x358), + REG(0x170), + REG(0x150), + REG(0x154), + REG(0x158), + REG16(0x41c), + REG16(0x600), + REG16(0x604), + REG16(0x608), + REG16(0x60c), + REG16(0x610), + REG16(0x614), + REG16(0x618), + REG16(0x61c), + REG16(0x620), + REG16(0x624), + REG16(0x628), + REG16(0x62c), + REG16(0x630), + REG16(0x634), + REG16(0x638), + REG16(0x63c), + REG16(0x640), + REG16(0x644), + REG16(0x648), + REG16(0x64c), + REG16(0x650), + REG16(0x654), + REG16(0x658), + REG16(0x65c), + REG16(0x660), + REG16(0x664), + REG16(0x668), + REG16(0x66c), + REG16(0x670), + REG16(0x674), + REG16(0x678), + REG16(0x67c), + REG(0x068), + REG(0x084), + NOP(1), + + 0 +}; + +static const u8 xehp_rcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + 0 +}; + +static const u8 dg2_rcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + 0 +}; + +static const u8 mtl_rcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + NOP(2), + LRI(2, POSTED), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + 0 +}; + +#define XE2_CTX_COMMON \ + NOP(1), /* [0x00] */ \ + LRI(15, POSTED), /* [0x01] */ \ + REG16(0x244), /* [0x02] CTXT_SR_CTL */ \ + REG(0x034), /* [0x04] RING_BUFFER_HEAD */ \ + REG(0x030), /* [0x06] RING_BUFFER_TAIL */ \ + REG(0x038), /* [0x08] RING_BUFFER_START */ \ + REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ \ + REG(0x168), /* [0x0c] BB_ADDR_UDW */ \ + REG(0x140), /* [0x0e] BB_ADDR */ \ + REG(0x110), /* [0x10] BB_STATE */ \ + REG(0x1c0), /* [0x12] BB_PER_CTX_PTR */ \ + REG(0x1c4), /* [0x14] RCS_INDIRECT_CTX */ \ + REG(0x1c8), /* [0x16] RCS_INDIRECT_CTX_OFFSET */ \ + REG(0x180), /* [0x18] CCID */ \ + REG16(0x2b4), /* [0x1a] SEMAPHORE_TOKEN */ \ + REG(0x120), /* [0x1c] PRT_BB_STATE */ \ + REG(0x124), /* [0x1e] PRT_BB_STATE_UDW */ \ + \ + NOP(1), /* [0x20] */ \ + LRI(9, POSTED), /* [0x21] */ \ + REG16(0x3a8), /* [0x22] CTX_TIMESTAMP */ \ + REG16(0x3ac), /* [0x24] CTX_TIMESTAMP_UDW */ \ + REG(0x108), /* [0x26] INDIRECT_RING_STATE */ \ + REG16(0x284), /* [0x28] dummy reg */ \ + REG16(0x280), /* [0x2a] CS_ACC_CTR_THOLD */ \ + REG16(0x27c), /* [0x2c] CS_CTX_SYS_PASID */ \ + REG16(0x278), /* [0x2e] CS_CTX_ASID */ \ + REG16(0x274), /* [0x30] PTBP_UDW */ \ + REG16(0x270) /* [0x32] PTBP_LDW */ + +static const u8 xe2_rcs_offsets[] = { + XE2_CTX_COMMON, + + NOP(2), /* [0x34] */ + LRI(2, POSTED), /* [0x36] */ + REG16(0x5a8), /* [0x37] CONTEXT_SCHEDULING_ATTRIBUTES */ + REG16(0x5ac), /* [0x39] PREEMPTION_STATUS */ + + NOP(6), /* [0x41] */ + LRI(1, 0), /* [0x47] */ + REG(0x0c8), /* [0x48] R_PWR_CLK_STATE */ + + 0 +}; + +static const u8 xe2_bcs_offsets[] = { + XE2_CTX_COMMON, + + NOP(4 + 8 + 1), /* [0x34] */ + LRI(2, POSTED), /* [0x41] */ + REG16(0x200), /* [0x42] BCS_SWCTRL */ + REG16(0x204), /* [0x44] BLIT_CCTL */ + + 0 +}; + +static const u8 xe2_xcs_offsets[] = { + XE2_CTX_COMMON, + + 0 +}; + +static const u8 xe2_indirect_ring_state_offsets[] = { + NOP(1), /* [0x00] */ + LRI(5, POSTED), /* [0x01] */ + REG(0x034), /* [0x02] RING_BUFFER_HEAD */ + REG(0x030), /* [0x04] RING_BUFFER_TAIL */ + REG(0x038), /* [0x06] RING_BUFFER_START */ + REG(0x048), /* [0x08] RING_BUFFER_START_UDW */ + REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ + + NOP(5), /* [0x0c] */ + LRI(9, POSTED), /* [0x11] */ + REG(0x168), /* [0x12] BB_ADDR_UDW */ + REG(0x140), /* [0x14] BB_ADDR */ + REG(0x110), /* [0x16] BB_STATE */ + REG16(0x588), /* [0x18] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x20] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x22] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x24] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x26] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x28] BB_STACK_WRITE_PORT */ + + NOP(12), /* [0x00] */ + + 0 +}; + +#undef REG16 +#undef REG +#undef LRI +#undef NOP + +static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class) +{ + if (class == XE_ENGINE_CLASS_RENDER) { + if (GRAPHICS_VER(xe) >= 20) + return xe2_rcs_offsets; + else if (GRAPHICS_VERx100(xe) >= 1270) + return mtl_rcs_offsets; + else if (GRAPHICS_VERx100(xe) >= 1255) + return dg2_rcs_offsets; + else if (GRAPHICS_VERx100(xe) >= 1250) + return xehp_rcs_offsets; + else + return gen12_rcs_offsets; + } else if (class == XE_ENGINE_CLASS_COPY) { + if (GRAPHICS_VER(xe) >= 20) + return xe2_bcs_offsets; + else + return gen12_xcs_offsets; + } else { + if (GRAPHICS_VER(xe) >= 20) + return xe2_xcs_offsets; + else if (GRAPHICS_VERx100(xe) >= 1255) + return dg2_xcs_offsets; + else + return gen12_xcs_offsets; + } +} + +static void set_context_control(u32 *regs, struct xe_hw_engine *hwe) +{ + regs[CTX_CONTEXT_CONTROL] = REG_MASKED_FIELD_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | + CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); + + if (xe_gt_has_indirect_ring_state(hwe->gt)) + regs[CTX_CONTEXT_CONTROL] |= + REG_MASKED_FIELD_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE); +} + +static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) +{ + struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq; + struct xe_device *xe = gt_to_xe(hwe->gt); + u8 num_regs; + + if (!xe_device_uses_memirq(xe)) + return; + + regs[CTX_LRM_INT_MASK_ENABLE] = MI_LOAD_REGISTER_MEM | + MI_LRI_LRM_CS_MMIO | MI_LRM_USE_GGTT; + regs[CTX_INT_MASK_ENABLE_REG] = RING_IMR(0).addr; + regs[CTX_INT_MASK_ENABLE_PTR] = xe_memirq_enable_ptr(memirq); + + num_regs = xe_device_has_msix(xe) ? 3 : 2; + regs[CTX_LRI_INT_REPORT_PTR] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(num_regs) | + MI_LRI_LRM_CS_MMIO | MI_LRI_FORCE_POSTED; + regs[CTX_INT_STATUS_REPORT_REG] = RING_INT_STATUS_RPT_PTR(0).addr; + regs[CTX_INT_STATUS_REPORT_PTR] = xe_memirq_status_ptr(memirq, hwe); + regs[CTX_INT_SRC_REPORT_REG] = RING_INT_SRC_RPT_PTR(0).addr; + regs[CTX_INT_SRC_REPORT_PTR] = xe_memirq_source_ptr(memirq, hwe); + + if (xe_device_has_msix(xe)) { + regs[CTX_CS_INT_VEC_REG] = CS_INT_VEC(0).addr; + /* CTX_CS_INT_VEC_DATA will be set in xe_lrc_init */ + } +} + +static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc) +{ + return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; +} + +static inline u32 __xe_lrc_ring_offset(struct xe_lrc *lrc) +{ + return 0; +} + +u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc) +{ + return lrc->ring.size; +} + +/* Make the magic macros work */ +#define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset +#define __xe_lrc_regs_offset xe_lrc_regs_offset + +#define LRC_CTX_JOB_TIMESTAMP_OFFSET 512 +#define LRC_ENGINE_ID_PPHWSP_OFFSET 1024 +#define LRC_PARALLEL_PPHWSP_OFFSET 2048 + +#define LRC_SEQNO_OFFSET 0 +#define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8) + +u32 xe_lrc_regs_offset(struct xe_lrc *lrc) +{ + return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; +} + +/** + * xe_lrc_reg_size() - Get size of the LRC registers area within queues + * @xe: the &xe_device struct instance + * + * Returns: Size of the LRC registers area for current platform + */ +size_t xe_lrc_reg_size(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1250) + return 96 * sizeof(u32); + else + return 80 * sizeof(u32); +} + +/** + * xe_lrc_engine_state_size() - Get size of the engine state within LRC + * @gt: the &xe_gt struct instance + * @class: Hardware engine class + * + * Returns: Size of the engine state + */ +size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class) +{ + return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt)); +} + +static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) +{ + return LRC_SEQNO_OFFSET; +} + +static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc) +{ + return LRC_START_SEQNO_OFFSET; +} + +static u32 __xe_lrc_ctx_job_timestamp_offset(struct xe_lrc *lrc) +{ + /* This is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_CTX_JOB_TIMESTAMP_OFFSET; +} + +static inline u32 __xe_lrc_parallel_offset(struct xe_lrc *lrc) +{ + /* The parallel is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_PARALLEL_PPHWSP_OFFSET; +} + +static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc) +{ + return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET; +} + +static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32); +} + +static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); +} + +static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32); +} + +static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32); +} + +static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) +{ + u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - + LRC_INDIRECT_RING_STATE_SIZE; + + if (lrc->flags & XE_LRC_FLAG_INDIRECT_CTX) + offset -= LRC_INDIRECT_CTX_BO_SIZE; + + return offset; +} + +static inline u32 __xe_lrc_indirect_ctx_offset(struct xe_lrc *lrc) +{ + return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - LRC_INDIRECT_CTX_BO_SIZE; +} + +static inline u32 __xe_lrc_wa_bb_offset(struct xe_lrc *lrc) +{ + return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE; +} + +#define DECL_MAP_ADDR_HELPERS(elem, bo_expr) \ +static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \ +{ \ + struct xe_bo *bo = (bo_expr); \ + struct iosys_map map = bo->vmap; \ +\ + xe_assert(lrc_to_xe(lrc), !iosys_map_is_null(&map)); \ + iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \ + return map; \ +} \ +static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \ +{ \ + struct xe_bo *bo = (bo_expr); \ +\ + return xe_bo_ggtt_addr(bo) + __xe_lrc_##elem##_offset(lrc); \ +} \ + +DECL_MAP_ADDR_HELPERS(ring, lrc->bo) +DECL_MAP_ADDR_HELPERS(pphwsp, lrc->bo) +DECL_MAP_ADDR_HELPERS(seqno, lrc->seqno_bo) +DECL_MAP_ADDR_HELPERS(regs, lrc->bo) +DECL_MAP_ADDR_HELPERS(start_seqno, lrc->seqno_bo) +DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo) +DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo) +DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) +DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) +DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) +DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo) + +#undef DECL_MAP_ADDR_HELPERS + +/** + * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp GGTT address + */ +u32 xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_ctx_timestamp_ggtt_addr(lrc); +} + +/** + * xe_lrc_ctx_timestamp_udw_ggtt_addr() - Get ctx timestamp udw GGTT address + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp udw GGTT address + */ +u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); +} + +/** + * xe_lrc_ctx_timestamp() - Read ctx timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp value + */ +static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + u32 ldw, udw = 0; + + map = __xe_lrc_ctx_timestamp_map(lrc); + ldw = xe_map_read32(xe, &map); + + if (xe->info.has_64bit_timestamp) { + map = __xe_lrc_ctx_timestamp_udw_map(lrc); + udw = xe_map_read32(xe, &map); + } + + return (u64)udw << 32 | ldw; +} + +/** + * xe_lrc_queue_timestamp() - Read queue timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: queue timestamp value + */ +static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + u32 ldw, udw = 0; + + xe_assert(xe, xe_lrc_is_multi_queue(lrc)); + + map = __xe_lrc_queue_timestamp_map(lrc); + ldw = xe_map_read32(xe, &map); + + map = __xe_lrc_queue_timestamp_udw_map(lrc); + udw = xe_map_read32(xe, &map); + + return (u64)udw << 32 | ldw; +} + +/** + * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp job GGTT address + */ +u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); +} + +/** + * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp job value + */ +u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_ctx_job_timestamp_map(lrc); + return xe_map_read32(xe, &map); +} + +u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_pphwsp_ggtt_addr(lrc); +} + +u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc) +{ + if (!xe_lrc_has_indirect_ring_state(lrc)) + return 0; + + return __xe_lrc_indirect_ring_ggtt_addr(lrc); +} + +static u32 xe_lrc_read_indirect_ctx_reg(struct xe_lrc *lrc, int reg_nr) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_indirect_ring_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + return xe_map_read32(xe, &map); +} + +static void xe_lrc_write_indirect_ctx_reg(struct xe_lrc *lrc, + int reg_nr, u32 val) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_indirect_ring_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + xe_map_write32(xe, &map, val); +} + +u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_regs_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + return xe_map_read32(xe, &map); +} + +void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_regs_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + xe_map_write32(xe, &map, val); +} + +static void *empty_lrc_data(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + void *data; + u32 *regs; + + data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL); + if (!data) + return NULL; + + /* 1st page: Per-Process of HW status Page */ + regs = data + LRC_PPHWSP_SIZE; + set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); + set_context_control(regs, hwe); + set_memory_based_intr(regs, hwe); + if (xe_gt_has_indirect_ring_state(gt)) { + regs = data + xe_gt_lrc_size(gt, hwe->class) - + LRC_INDIRECT_RING_STATE_SIZE; + set_offsets(regs, xe2_indirect_ring_state_offsets, hwe); + } + + return data; +} + +/** + * xe_default_lrc_update_memirq_regs_with_address - Re-compute GGTT references in default LRC + * of given engine. + * @hwe: the &xe_hw_engine struct instance + */ +void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + u32 *regs; + + if (!gt->default_lrc[hwe->class]) + return; + + regs = gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE; + set_memory_based_intr(regs, hwe); +} + +/** + * xe_lrc_update_memirq_regs_with_address - Re-compute GGTT references in mem interrupt data + * for given LRC. + * @lrc: the &xe_lrc struct instance + * @hwe: the &xe_hw_engine struct instance + * @regs: scratch buffer to be used as temporary storage + */ +void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *regs) +{ + struct xe_gt *gt = hwe->gt; + struct iosys_map map; + size_t regs_len; + + if (!xe_device_uses_memirq(gt_to_xe(gt))) + return; + + map = __xe_lrc_regs_map(lrc); + regs_len = xe_lrc_reg_size(gt_to_xe(gt)); + xe_map_memcpy_from(gt_to_xe(gt), regs, &map, 0, regs_len); + set_memory_based_intr(regs, hwe); + xe_map_memcpy_to(gt_to_xe(gt), &map, 0, regs, regs_len); +} + +static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) +{ + u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt)); + + xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc)); + xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc)); +} + +static void xe_lrc_finish(struct xe_lrc *lrc) +{ + xe_hw_fence_ctx_finish(&lrc->fence_ctx); + xe_bo_unpin_map_no_vm(lrc->bo); + xe_bo_unpin_map_no_vm(lrc->seqno_bo); +} + +/* + * wa_bb_setup_utilization() - Write commands to wa bb to assist + * in calculating active context run ticks. + * + * Context Timestamp (CTX_TIMESTAMP) in the LRC accumulates the run ticks of the + * context, but only gets updated when the context switches out. In order to + * check how long a context has been active before it switches out, two things + * are required: + * + * (1) Determine if the context is running: + * To do so, we program the WA BB to set an initial value for CTX_TIMESTAMP in + * the LRC. The value chosen is 1 since 0 is the initial value when the LRC is + * initialized. During a query, we just check for this value to determine if the + * context is active. If the context switched out, it would overwrite this + * location with the actual CTX_TIMESTAMP MMIO value. Note that WA BB runs as + * the last part of context restore, so reusing this LRC location will not + * clobber anything. + * + * (2) Calculate the time that the context has been active for: + * The CTX_TIMESTAMP ticks only when the context is active. If a context is + * active, we just use the CTX_TIMESTAMP MMIO as the new value of utilization. + * While doing so, we need to read the CTX_TIMESTAMP MMIO for the specific + * engine instance. Since we do not know which instance the context is running + * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and + * store it in the PPHSWP. + */ +#define CONTEXT_ACTIVE XE_LRC_CTX_TIMESTAMP_ACTIVE +static ssize_t setup_utilization_wa(struct xe_lrc *lrc, + struct xe_hw_engine *hwe, + u32 *batch, + size_t max_len) +{ + u32 *cmd = batch; + + if (IS_SRIOV_VF(gt_to_xe(lrc->gt))) + return 0; + + if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) + return -ENOSPC; + + *cmd++ = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; + *cmd++ = ENGINE_ID(0).addr; + *cmd++ = __xe_lrc_engine_id_ggtt_addr(lrc); + *cmd++ = 0; + + *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); + *cmd++ = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); + *cmd++ = 0; + *cmd++ = lower_32_bits(CONTEXT_ACTIVE); + + if (lrc_to_xe(lrc)->info.has_64bit_timestamp) { + *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); + *cmd++ = __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); + *cmd++ = 0; + *cmd++ = upper_32_bits(CONTEXT_ACTIVE); + } + + return cmd - batch; +} + +static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); + u32 *cmd = batch; + + if (!XE_GT_WA(lrc->gt, 16010904313) || + !(hwe->class == XE_ENGINE_CLASS_RENDER || + hwe->class == XE_ENGINE_CLASS_COMPUTE || + hwe->class == XE_ENGINE_CLASS_COPY || + hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE || + hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE)) + return 0; + + if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) + return -ENOSPC; + + *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | + MI_LRM_ASYNC; + *cmd++ = RING_CTX_TIMESTAMP(0).addr; + *cmd++ = ts_addr; + *cmd++ = 0; + + *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | + MI_LRM_ASYNC; + *cmd++ = RING_CTX_TIMESTAMP(0).addr; + *cmd++ = ts_addr; + *cmd++ = 0; + + *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO; + *cmd++ = RING_CTX_TIMESTAMP(0).addr; + *cmd++ = ts_addr; + *cmd++ = 0; + + return cmd - batch; +} + +static ssize_t setup_configfs_post_ctx_restore_bb(struct xe_lrc *lrc, + struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + const u32 *user_batch; + u32 *cmd = batch; + u32 count; + + count = xe_configfs_get_ctx_restore_post_bb(to_pci_dev(xe->drm.dev), + hwe->class, &user_batch); + if (!count) + return 0; + + if (count > max_len) + return -ENOSPC; + + /* + * This should be used only for tests and validation. Taint the kernel + * as anything could be submitted directly in context switches + */ + add_taint(TAINT_TEST, LOCKDEP_STILL_OK); + + memcpy(cmd, user_batch, count * sizeof(u32)); + cmd += count; + + return cmd - batch; +} + +static ssize_t setup_configfs_mid_ctx_restore_bb(struct xe_lrc *lrc, + struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + const u32 *user_batch; + u32 *cmd = batch; + u32 count; + + count = xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev), + hwe->class, &user_batch); + if (!count) + return 0; + + if (count > max_len) + return -ENOSPC; + + /* + * This should be used only for tests and validation. Taint the kernel + * as anything could be submitted directly in context switches + */ + add_taint(TAINT_TEST, LOCKDEP_STILL_OK); + + memcpy(cmd, user_batch, count * sizeof(u32)); + cmd += count; + + return cmd - batch; +} + +static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, + struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + u32 *cmd = batch; + + if (!XE_GT_WA(lrc->gt, 18022495364) || + hwe->class != XE_ENGINE_CLASS_RENDER) + return 0; + + if (xe_gt_WARN_ON(lrc->gt, max_len < 3)) + return -ENOSPC; + + *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1); + *cmd++ = CS_DEBUG_MODE2(0).addr; + *cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); + + return cmd - batch; +} + +static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc, + struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + struct xe_gt *gt = lrc->gt; + u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = + gt->ring_ops[hwe->class]->emit_aux_table_inv; + + if (!emit) + return 0; + + if (xe_gt_WARN_ON(gt, max_len < 8)) + return -ENOSPC; + + return emit(gt, batch) - batch; +} + +struct bo_setup { + ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *batch, size_t max_size); +}; + +struct bo_setup_state { + /* Input: */ + struct xe_lrc *lrc; + struct xe_hw_engine *hwe; + size_t max_size; + size_t reserve_dw; + unsigned int offset; + const struct bo_setup *funcs; + unsigned int num_funcs; + + /* State: */ + u32 *buffer; + u32 *ptr; + unsigned int written; +}; + +static int setup_bo(struct bo_setup_state *state) +{ + ssize_t remain; + + if (state->lrc->bo->vmap.is_iomem) { + xe_gt_assert(state->hwe->gt, state->buffer); + state->ptr = state->buffer; + } else { + state->ptr = state->lrc->bo->vmap.vaddr + state->offset; + } + + remain = state->max_size / sizeof(u32); + + for (size_t i = 0; i < state->num_funcs; i++) { + ssize_t len = state->funcs[i].setup(state->lrc, state->hwe, + state->ptr, remain); + + remain -= len; + + /* + * Caller has asked for at least reserve_dw to remain unused. + */ + if (len < 0 || + xe_gt_WARN_ON(state->lrc->gt, remain < state->reserve_dw)) + goto fail; + + state->ptr += len; + state->written += len; + } + + return 0; + +fail: + return -ENOSPC; +} + +static void finish_bo(struct bo_setup_state *state) +{ + if (!state->lrc->bo->vmap.is_iomem) + return; + + xe_map_memcpy_to(gt_to_xe(state->lrc->gt), &state->lrc->bo->vmap, + state->offset, state->buffer, + state->written * sizeof(u32)); +} + +/** + * xe_lrc_setup_wa_bb_with_scratch - Execute all wa bb setup callbacks. + * @lrc: the &xe_lrc struct instance + * @hwe: the &xe_hw_engine struct instance + * @scratch: preallocated scratch buffer for temporary storage + * Return: 0 on success, negative error code on failure + */ +int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *scratch) +{ + static const struct bo_setup funcs[] = { + { .setup = setup_timestamp_wa }, + { .setup = setup_invalidate_state_cache_wa }, + { .setup = setup_utilization_wa }, + { .setup = setup_configfs_post_ctx_restore_bb }, + }; + struct bo_setup_state state = { + .lrc = lrc, + .hwe = hwe, + .max_size = LRC_WA_BB_SIZE, + .buffer = scratch, + .reserve_dw = 1, + .offset = __xe_lrc_wa_bb_offset(lrc), + .funcs = funcs, + .num_funcs = ARRAY_SIZE(funcs), + }; + int ret; + + ret = setup_bo(&state); + if (ret) + return ret; + + *state.ptr++ = MI_BATCH_BUFFER_END; + state.written++; + + finish_bo(&state); + + xe_lrc_write_ctx_reg(lrc, CTX_BB_PER_CTX_PTR, + xe_bo_ggtt_addr(lrc->bo) + state.offset + 1); + + return 0; +} + +static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) +{ + u32 *buf = NULL; + int ret; + + if (lrc->bo->vmap.is_iomem) { + buf = kmalloc(LRC_WA_BB_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + } + + ret = xe_lrc_setup_wa_bb_with_scratch(lrc, hwe, buf); + + kfree(buf); + + return ret; +} + +static int +setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) +{ + static const struct bo_setup rcs_funcs[] = { + { .setup = setup_timestamp_wa }, + { .setup = setup_invalidate_auxccs_wa }, + { .setup = setup_configfs_mid_ctx_restore_bb }, + }; + static const struct bo_setup xcs_funcs[] = { + { .setup = setup_invalidate_auxccs_wa }, + { .setup = setup_configfs_mid_ctx_restore_bb }, + }; + struct bo_setup_state state = { + .lrc = lrc, + .hwe = hwe, + .max_size = (63 * 64) /* max 63 cachelines */, + .buffer = NULL, + .offset = __xe_lrc_indirect_ctx_offset(lrc), + }; + int ret; + + if (!(lrc->flags & XE_LRC_FLAG_INDIRECT_CTX)) + return 0; + + if (hwe->class == XE_ENGINE_CLASS_RENDER || + hwe->class == XE_ENGINE_CLASS_COMPUTE) { + state.funcs = rcs_funcs; + state.num_funcs = ARRAY_SIZE(rcs_funcs); + } else { + state.funcs = xcs_funcs; + state.num_funcs = ARRAY_SIZE(xcs_funcs); + } + + if (xe_gt_WARN_ON(lrc->gt, !state.funcs)) + return 0; + + if (lrc->bo->vmap.is_iomem) { + state.buffer = kmalloc(state.max_size, GFP_KERNEL); + if (!state.buffer) + return -ENOMEM; + } + + ret = setup_bo(&state); + if (ret) { + kfree(state.buffer); + return ret; + } + + /* + * Align to 64B cacheline so there's no garbage at the end for CS to + * execute: size for indirect ctx must be a multiple of 64. + */ + while (state.written & 0xf) { + *state.ptr++ = MI_NOOP; + state.written++; + } + + finish_bo(&state); + kfree(state.buffer); + + /* + * Enable INDIRECT_CTX leaving INDIRECT_CTX_OFFSET at its default: it + * varies per engine class, but the default is good enough + */ + xe_lrc_write_ctx_reg(lrc, + CTX_CS_INDIRECT_CTX, + (xe_bo_ggtt_addr(lrc->bo) + state.offset) | + /* Size in CLs. */ + (state.written * sizeof(u32) / 64)); + + return 0; +} + +static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + + xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW && + priority <= XE_MULTI_QUEUE_PRIORITY_HIGH)); + + /* xe_multi_queue_priority is directly mapped to LRC priority values */ + return priority; +} + +/** + * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC + * @lrc: Logical Ring Context + * @priority: Multi queue priority of the exec queue + * + * Convert @priority to LRC multi queue priority and update the @lrc descriptor + */ +void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) +{ + lrc->desc &= ~LRC_PRIORITY; + lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority)); +} + +static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, + void *replay_state, u16 msix_vec, u32 init_flags) +{ + struct xe_gt *gt = hwe->gt; + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct iosys_map map; + u32 arb_enable; + u32 state_cache_perf_fix[3]; + int err; + + /* + * Init Per-Process of HW status Page, LRC / context state to known + * values. If there's already a primed default_lrc, just copy it, otherwise + * it's the early submission to record the lrc: build a new empty one from + * scratch. + */ + map = __xe_lrc_pphwsp_map(lrc); + if (gt->default_lrc[hwe->class] || replay_state) { + xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ + xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, + gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, + lrc->size - LRC_PPHWSP_SIZE); + if (replay_state) + xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, + replay_state, lrc->replay_size); + } else { + void *init_data = empty_lrc_data(hwe); + + if (!init_data) { + return -ENOMEM; + } + + xe_map_memcpy_to(xe, &map, 0, init_data, lrc->size); + kfree(init_data); + } + + if (vm) + xe_lrc_set_ppgtt(lrc, vm); + + if (xe_device_has_msix(xe)) { + xe_lrc_write_ctx_reg(lrc, CTX_INT_STATUS_REPORT_PTR, + xe_memirq_status_ptr(&tile->memirq, hwe)); + xe_lrc_write_ctx_reg(lrc, CTX_INT_SRC_REPORT_PTR, + xe_memirq_source_ptr(&tile->memirq, hwe)); + xe_lrc_write_ctx_reg(lrc, CTX_CS_INT_VEC_DATA, msix_vec << 16 | msix_vec); + } + + if (xe_gt_has_indirect_ring_state(gt)) { + xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, + __xe_lrc_indirect_ring_ggtt_addr(lrc)); + + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, + __xe_lrc_ring_ggtt_addr(lrc)); + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START_UDW, 0); + + /* Match head and tail pointers */ + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, lrc->ring.tail); + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail); + + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_CTL, + RING_CTL_SIZE(lrc->ring.size) | RING_VALID); + } else { + xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); + + /* Match head and tail pointers */ + xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, lrc->ring.tail); + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); + + xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL, + RING_CTL_SIZE(lrc->ring.size) | RING_VALID); + } + + if (init_flags & XE_LRC_CREATE_RUNALONE) + xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, + xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | + REG_MASKED_FIELD_ENABLE(CTX_CTRL_RUN_ALONE)); + + if (init_flags & XE_LRC_CREATE_PXP) + xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, + xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | + REG_MASKED_FIELD_ENABLE(CTX_CTRL_PXP_ENABLE)); + + lrc->ctx_timestamp = 0; + xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0); + if (lrc_to_xe(lrc)->info.has_64bit_timestamp) + xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); + + /* + * Note: It's possible that this LRC may belong to an exec_queue that is + * not part of a multi-queue group. That said, it doesn't hurt to set + * this field anyways since any class that supports multi-queue will + * have these LRC fields defined. + */ + if (xe_gt_supports_multi_queue(gt, hwe->class)) { + lrc->queue_timestamp = 0; + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0); + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0); + } + + if (xe->info.has_asid && vm) + xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid); + + lrc->desc = LRC_VALID; + lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT); + /* TODO: Priority */ + + /* While this appears to have something about privileged batches or + * some such, it really just means PPGTT mode. + */ + if (vm) + lrc->desc |= LRC_PRIVILEGE; + + if (GRAPHICS_VERx100(xe) < 1250) { + lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance); + lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class); + } + + arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE; + xe_lrc_write_ring(lrc, &arb_enable, sizeof(arb_enable)); + + if (init_flags & XE_LRC_DISABLE_STATE_CACHE_PERF_FIX) { + state_cache_perf_fix[0] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + state_cache_perf_fix[1] = COMMON_SLICE_CHICKEN3.addr; + state_cache_perf_fix[2] = REG_MASKED_FIELD_ENABLE(DISABLE_STATE_CACHE_PERF_FIX); + xe_lrc_write_ring(lrc, state_cache_perf_fix, sizeof(state_cache_perf_fix)); + } + + map = __xe_lrc_seqno_map(lrc); + xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); + + map = __xe_lrc_start_seqno_map(lrc); + xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); + + err = setup_wa_bb(lrc, hwe); + if (err) + return err; + + err = setup_indirect_ctx(lrc, hwe); + + return err; +} + +static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, + void *replay_state, u32 ring_size, u16 msix_vec, u32 init_flags) +{ + struct xe_gt *gt = hwe->gt; + const u32 lrc_size = xe_gt_lrc_size(gt, hwe->class); + u32 bo_size = ring_size + lrc_size + LRC_WA_BB_SIZE; + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *bo; + u32 bo_flags; + int err; + + kref_init(&lrc->refcount); + lrc->gt = gt; + lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class); + lrc->size = lrc_size; + lrc->flags = 0; + lrc->ring.size = ring_size; + lrc->ring.tail = 0; + + if (gt_engine_needs_indirect_ctx(gt, hwe->class)) { + lrc->flags |= XE_LRC_FLAG_INDIRECT_CTX; + bo_size += LRC_INDIRECT_CTX_BO_SIZE; + } + + if (xe_gt_has_indirect_ring_state(gt)) + lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; + + bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE; + + if ((vm && vm->xef) || init_flags & XE_LRC_CREATE_USER_CTX) /* userspace */ + bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM; + + bo = xe_bo_create_pin_map_novm(xe, tile, bo_size, + ttm_bo_type_kernel, + bo_flags, false); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + lrc->bo = bo; + + bo = xe_bo_create_pin_map_novm(xe, tile, PAGE_SIZE, + ttm_bo_type_kernel, + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_SYSTEM, false); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + goto err_lrc_finish; + } + lrc->seqno_bo = bo; + + xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt, + hwe->fence_irq, hwe->name); + + err = xe_lrc_ctx_init(lrc, hwe, vm, replay_state, msix_vec, init_flags); + if (err) + goto err_lrc_finish; + + if (vm && vm->xef) + xe_drm_client_add_bo(vm->xef->client, lrc->bo); + + return 0; + +err_lrc_finish: + xe_lrc_finish(lrc); + return err; +} + +/** + * xe_lrc_create - Create a LRC + * @hwe: Hardware Engine + * @vm: The VM (address space) + * @replay_state: GPU hang replay state + * @ring_size: LRC ring size + * @msix_vec: MSI-X interrupt vector (for platforms that support it) + * @flags: LRC initialization flags + * + * Allocate and initialize the Logical Ring Context (LRC). + * + * Return pointer to created LRC upon success and an error pointer + * upon failure. + */ +struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, + void *replay_state, u32 ring_size, u16 msix_vec, u32 flags) +{ + struct xe_lrc *lrc; + int err; + + lrc = kzalloc_obj(*lrc); + if (!lrc) + return ERR_PTR(-ENOMEM); + + err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags); + if (err) { + kfree(lrc); + return ERR_PTR(err); + } + + return lrc; +} + +/** + * xe_lrc_destroy - Destroy the LRC + * @ref: reference to LRC + * + * Called when ref == 0, release resources held by the Logical Ring Context + * (LRC) and free the LRC memory. + */ +void xe_lrc_destroy(struct kref *ref) +{ + struct xe_lrc *lrc = container_of(ref, struct xe_lrc, refcount); + + xe_lrc_finish(lrc); + kfree(lrc); +} + +/** + * xe_lrc_update_hwctx_regs_with_address - Re-compute GGTT references within given LRC. + * @lrc: the &xe_lrc struct instance + */ +void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) { + xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, + __xe_lrc_indirect_ring_ggtt_addr(lrc)); + + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, + __xe_lrc_ring_ggtt_addr(lrc)); + } else { + xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); + } +} + +void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, tail); + else + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail); +} + +u32 xe_lrc_ring_tail(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL) & TAIL_ADDR; + else + return xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL) & TAIL_ADDR; +} + +static u32 xe_lrc_ring_start(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START); + else + return xe_lrc_read_ctx_reg(lrc, CTX_RING_START); +} + +void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, head); + else + xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head); +} + +u32 xe_lrc_ring_head(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD) & HEAD_ADDR; + else + return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR; +} + +u32 xe_lrc_ring_space(struct xe_lrc *lrc) +{ + const u32 head = xe_lrc_ring_head(lrc); + const u32 tail = lrc->ring.tail; + const u32 size = lrc->ring.size; + + return ((head - tail - 1) & (size - 1)) + 1; +} + +static void __xe_lrc_write_ring(struct xe_lrc *lrc, struct iosys_map ring, + const void *data, size_t size) +{ + struct xe_device *xe = lrc_to_xe(lrc); + + iosys_map_incr(&ring, lrc->ring.tail); + xe_map_memcpy_to(xe, &ring, 0, data, size); + lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1); +} + +void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map ring; + u32 rhs; + size_t aligned_size; + + xe_assert(xe, IS_ALIGNED(size, 4)); + aligned_size = ALIGN(size, 8); + + ring = __xe_lrc_ring_map(lrc); + + xe_assert(xe, lrc->ring.tail < lrc->ring.size); + rhs = lrc->ring.size - lrc->ring.tail; + if (size > rhs) { + __xe_lrc_write_ring(lrc, ring, data, rhs); + __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs); + } else { + __xe_lrc_write_ring(lrc, ring, data, size); + } + + if (aligned_size > size) { + u32 noop = MI_NOOP; + + __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop)); + } + + /* + * The ring and the LRC context image are both WC, so the ring tail + * update which publishes these writes can become visible to the device + * first. Ensure the ring contents are visible before returning. + */ + xe_device_wmb(xe); +} + +u64 xe_lrc_descriptor(struct xe_lrc *lrc) +{ + return lrc->desc | xe_lrc_ggtt_addr(lrc); +} + +u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_seqno_ggtt_addr(lrc); +} + +/** + * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence. + * + * Allocate but don't initialize an lrc seqno fence. + * + * Return: Pointer to the allocated fence or + * negative error pointer on error. + */ +struct dma_fence *xe_lrc_alloc_seqno_fence(void) +{ + return xe_hw_fence_alloc(); +} + +/** + * xe_lrc_free_seqno_fence() - Free an lrc seqno fence. + * @fence: Pointer to the fence to free. + * + * Frees an lrc seqno fence that hasn't yet been + * initialized. + */ +void xe_lrc_free_seqno_fence(struct dma_fence *fence) +{ + xe_hw_fence_free(fence); +} + +/** + * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence. + * @lrc: Pointer to the lrc. + * @fence: Pointer to the fence to initialize. + * + * Initializes a pre-allocated lrc seqno fence. + * After initialization, the fence is subject to normal + * dma-fence refcounting. + */ +void xe_lrc_init_seqno_fence(struct xe_lrc *lrc, struct dma_fence *fence) +{ + xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc)); +} + +s32 xe_lrc_seqno(struct xe_lrc *lrc) +{ + struct iosys_map map = __xe_lrc_seqno_map(lrc); + + return xe_map_read32(lrc_to_xe(lrc), &map); +} + +s32 xe_lrc_start_seqno(struct xe_lrc *lrc) +{ + struct iosys_map map = __xe_lrc_start_seqno_map(lrc); + + return xe_map_read32(lrc_to_xe(lrc), &map); +} + +u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_start_seqno_ggtt_addr(lrc); +} + +u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_parallel_ggtt_addr(lrc); +} + +struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc) +{ + return __xe_lrc_parallel_map(lrc); +} + +/** + * xe_lrc_engine_id() - Read engine id value + * @lrc: Pointer to the lrc. + * + * Returns: context id value + */ +static u32 xe_lrc_engine_id(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_engine_id_map(lrc); + return xe_map_read32(xe, &map); +} + +static int instr_dw(u32 cmd_header) +{ + /* GFXPIPE "SINGLE_DW" opcodes are a single dword */ + if ((cmd_header & (XE_INSTR_CMD_TYPE | GFXPIPE_PIPELINE)) == + GFXPIPE_SINGLE_DW_CMD(0, 0)) + return 1; + + /* 3DSTATE_SO_DECL_LIST has a 9-bit dword length rather than 8 */ + if ((cmd_header & GFXPIPE_MATCH_MASK) == CMD_3DSTATE_SO_DECL_LIST) + return REG_FIELD_GET(CMD_3DSTATE_SO_DECL_LIST_DW_LEN, cmd_header) + 2; + + /* Most instructions have the # of dwords (minus 2) in 7:0 */ + return REG_FIELD_GET(XE_INSTR_LEN_MASK, cmd_header) + 2; +} + +static int dump_mi_command(struct drm_printer *p, + struct xe_gt *gt, + u32 *start, + u32 *dw, + int remaining_dw) +{ + u32 inst_header = *dw; + u32 numdw = instr_dw(inst_header); + u32 opcode = REG_FIELD_GET(MI_OPCODE, inst_header); + int num_noop; + + /* First check for commands that don't have/use a '# DW' field */ + switch (inst_header & MI_OPCODE) { + case MI_NOOP: + num_noop = 1; + while (num_noop < remaining_dw && + (*(++dw) & REG_GENMASK(31, 23)) == MI_NOOP) + num_noop++; + drm_printf(p, "LRC[%#5tx] = [%#010x] MI_NOOP (%d dwords)\n", + dw - num_noop - start, inst_header, num_noop); + return num_noop; + + case MI_TOPOLOGY_FILTER: + drm_printf(p, "LRC[%#5tx] = [%#010x] MI_TOPOLOGY_FILTER\n", + dw - start, inst_header); + return 1; + + case MI_BATCH_BUFFER_END: + drm_printf(p, "LRC[%#5tx] = [%#010x] MI_BATCH_BUFFER_END\n", + dw - start, inst_header); + /* Return 'remaining_dw' to consume the rest of the LRC */ + return remaining_dw; + } + + /* + * Any remaining commands include a # of dwords. We should make sure + * it doesn't exceed the remaining size of the LRC. + */ + if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) + numdw = remaining_dw; + + switch (inst_header & MI_OPCODE) { + case MI_LOAD_REGISTER_IMM: + drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_IMM: %d regs\n", + dw - start, inst_header, (numdw - 1) / 2); + for (int i = 1; i < numdw; i += 2) + drm_printf(p, "LRC[%#5tx] = - %#6x = %#010x\n", + &dw[i] - start, dw[i], dw[i + 1]); + return numdw; + + case MI_LOAD_REGISTER_MEM & MI_OPCODE: + drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_MEM: %s%s\n", + dw - start, inst_header, + dw[0] & MI_LRI_LRM_CS_MMIO ? "CS_MMIO " : "", + dw[0] & MI_LRM_USE_GGTT ? "USE_GGTT " : ""); + if (numdw == 4) + drm_printf(p, "LRC[%#5tx] = - %#6x = %#010llx\n", + dw - start, + dw[1], ((u64)(dw[3]) << 32 | (u64)(dw[2]))); + else + drm_printf(p, "LRC[%#5tx] = - %*ph (%s)\n", + dw - start, (int)sizeof(u32) * (numdw - 1), + dw + 1, numdw < 4 ? "truncated" : "malformed"); + return numdw; + + case MI_FORCE_WAKEUP: + drm_printf(p, "LRC[%#5tx] = [%#010x] MI_FORCE_WAKEUP\n", + dw - start, inst_header); + return numdw; + + default: + drm_printf(p, "LRC[%#5tx] = [%#010x] unknown MI opcode %#x, likely %d dwords\n", + dw - start, inst_header, opcode, numdw); + return numdw; + } +} + +static int dump_gfxpipe_command(struct drm_printer *p, + struct xe_gt *gt, + u32 *start, + u32 *dw, + int remaining_dw) +{ + u32 numdw = instr_dw(*dw); + u32 pipeline = REG_FIELD_GET(GFXPIPE_PIPELINE, *dw); + u32 opcode = REG_FIELD_GET(GFXPIPE_OPCODE, *dw); + u32 subopcode = REG_FIELD_GET(GFXPIPE_SUBOPCODE, *dw); + + /* + * Make sure we haven't mis-parsed a number of dwords that exceeds the + * remaining size of the LRC. + */ + if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) + numdw = remaining_dw; + + switch (*dw & GFXPIPE_MATCH_MASK) { +#define MATCH(cmd) \ + case cmd: \ + drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \ + dw - start, *dw, numdw); \ + return numdw +#define MATCH3D(cmd) \ + case CMD_##cmd: \ + drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \ + dw - start, *dw, numdw); \ + return numdw + + MATCH(STATE_BASE_ADDRESS); + MATCH(STATE_SIP); + MATCH(GPGPU_CSR_BASE_ADDRESS); + MATCH(STATE_COMPUTE_MODE); + MATCH3D(3DSTATE_BTD); + MATCH(STATE_SYSTEM_MEM_FENCE_ADDRESS); + MATCH(STATE_CONTEXT_DATA_BASE_ADDRESS); + + MATCH3D(3DSTATE_VF_STATISTICS); + + MATCH(PIPELINE_SELECT); + + MATCH3D(3DSTATE_DRAWING_RECTANGLE_FAST); + MATCH3D(3DSTATE_CUSTOM_SAMPLE_PATTERN); + MATCH3D(3DSTATE_CLEAR_PARAMS); + MATCH3D(3DSTATE_DEPTH_BUFFER); + MATCH3D(3DSTATE_STENCIL_BUFFER); + MATCH3D(3DSTATE_HIER_DEPTH_BUFFER); + MATCH3D(3DSTATE_VERTEX_BUFFERS); + MATCH3D(3DSTATE_VERTEX_ELEMENTS); + MATCH3D(3DSTATE_INDEX_BUFFER); + MATCH3D(3DSTATE_VF); + MATCH3D(3DSTATE_MULTISAMPLE); + MATCH3D(3DSTATE_CC_STATE_POINTERS); + MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS); + MATCH3D(3DSTATE_VS); + MATCH3D(3DSTATE_GS); + MATCH3D(3DSTATE_CLIP); + MATCH3D(3DSTATE_SF); + MATCH3D(3DSTATE_WM); + MATCH3D(3DSTATE_CONSTANT_VS); + MATCH3D(3DSTATE_CONSTANT_GS); + MATCH3D(3DSTATE_CONSTANT_PS); + MATCH3D(3DSTATE_SAMPLE_MASK); + MATCH3D(3DSTATE_CONSTANT_HS); + MATCH3D(3DSTATE_CONSTANT_DS); + MATCH3D(3DSTATE_HS); + MATCH3D(3DSTATE_TE); + MATCH3D(3DSTATE_DS); + MATCH3D(3DSTATE_STREAMOUT); + MATCH3D(3DSTATE_SBE); + MATCH3D(3DSTATE_PS); + MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP); + MATCH3D(3DSTATE_CPS_POINTERS); + MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC); + MATCH3D(3DSTATE_BLEND_STATE_POINTERS); + MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_VS); + MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_HS); + MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_DS); + MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_GS); + MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_PS); + MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_VS); + MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_HS); + MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_DS); + MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_GS); + MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_PS); + MATCH3D(3DSTATE_VF_INSTANCING); + MATCH3D(3DSTATE_VF_SGVS); + MATCH3D(3DSTATE_VF_TOPOLOGY); + MATCH3D(3DSTATE_WM_CHROMAKEY); + MATCH3D(3DSTATE_PS_BLEND); + MATCH3D(3DSTATE_WM_DEPTH_STENCIL); + MATCH3D(3DSTATE_PS_EXTRA); + MATCH3D(3DSTATE_RASTER); + MATCH3D(3DSTATE_SBE_SWIZ); + MATCH3D(3DSTATE_WM_HZ_OP); + MATCH3D(3DSTATE_VF_COMPONENT_PACKING); + MATCH3D(3DSTATE_VF_SGVS_2); + MATCH3D(3DSTATE_VFG); + MATCH3D(3DSTATE_URB_ALLOC_VS); + MATCH3D(3DSTATE_URB_ALLOC_HS); + MATCH3D(3DSTATE_URB_ALLOC_DS); + MATCH3D(3DSTATE_URB_ALLOC_GS); + MATCH3D(3DSTATE_SO_BUFFER_INDEX_0); + MATCH3D(3DSTATE_SO_BUFFER_INDEX_1); + MATCH3D(3DSTATE_SO_BUFFER_INDEX_2); + MATCH3D(3DSTATE_SO_BUFFER_INDEX_3); + MATCH3D(3DSTATE_PRIMITIVE_REPLICATION); + MATCH3D(3DSTATE_TBIMR_TILE_PASS_INFO); + MATCH3D(3DSTATE_AMFS); + MATCH3D(3DSTATE_DEPTH_BOUNDS); + MATCH3D(3DSTATE_AMFS_TEXTURE_POINTERS); + MATCH3D(3DSTATE_CONSTANT_TS_POINTER); + MATCH3D(3DSTATE_MESH_CONTROL); + MATCH3D(3DSTATE_MESH_DISTRIB); + MATCH3D(3DSTATE_TASK_REDISTRIB); + MATCH3D(3DSTATE_MESH_SHADER); + MATCH3D(3DSTATE_MESH_SHADER_DATA); + MATCH3D(3DSTATE_TASK_CONTROL); + MATCH3D(3DSTATE_TASK_SHADER); + MATCH3D(3DSTATE_TASK_SHADER_DATA); + MATCH3D(3DSTATE_URB_ALLOC_MESH); + MATCH3D(3DSTATE_URB_ALLOC_TASK); + MATCH3D(3DSTATE_CLIP_MESH); + MATCH3D(3DSTATE_SBE_MESH); + MATCH3D(3DSTATE_CPSIZE_CONTROL_BUFFER); + MATCH3D(3DSTATE_COARSE_PIXEL); + MATCH3D(3DSTATE_MESH_SHADER_DATA_EXT); + MATCH3D(3DSTATE_TASK_SHADER_DATA_EXT); + MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC_2); + MATCH3D(3DSTATE_CC_STATE_POINTERS_2); + MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS_2); + MATCH3D(3DSTATE_BLEND_STATE_POINTERS_2); + MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2); + + MATCH3D(3DSTATE_DRAWING_RECTANGLE); + MATCH3D(3DSTATE_URB_MEMORY); + MATCH3D(3DSTATE_CHROMA_KEY); + MATCH3D(3DSTATE_POLY_STIPPLE_OFFSET); + MATCH3D(3DSTATE_POLY_STIPPLE_PATTERN); + MATCH3D(3DSTATE_LINE_STIPPLE); + MATCH3D(3DSTATE_AA_LINE_PARAMETERS); + MATCH3D(3DSTATE_MONOFILTER_SIZE); + MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_VS); + MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_HS); + MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_DS); + MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_GS); + MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_PS); + MATCH3D(3DSTATE_SO_DECL_LIST); + MATCH3D(3DSTATE_SO_BUFFER); + MATCH3D(3DSTATE_BINDING_TABLE_POOL_ALLOC); + MATCH3D(3DSTATE_SAMPLE_PATTERN); + MATCH3D(3DSTATE_3D_MODE); + MATCH3D(3DSTATE_SUBSLICE_HASH_TABLE); + MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTERS); + MATCH3D(3DSTATE_PTBR_TILE_PASS_INFO); + MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTER_2); + + default: + drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n", + dw - start, *dw, pipeline, opcode, subopcode, numdw); + return numdw; + } +} + +static int dump_gfx_state_command(struct drm_printer *p, + struct xe_gt *gt, + u32 *start, + u32 *dw, + int remaining_dw) +{ + u32 numdw = instr_dw(*dw); + u32 opcode = REG_FIELD_GET(GFX_STATE_OPCODE, *dw); + + /* + * Make sure we haven't mis-parsed a number of dwords that exceeds the + * remaining size of the LRC. + */ + if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) + numdw = remaining_dw; + + switch (*dw & (XE_INSTR_GFX_STATE | GFX_STATE_OPCODE)) { + MATCH(STATE_WRITE_INLINE); + + default: + drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n", + dw - start, *dw, opcode, numdw); + return numdw; + } +} + +void xe_lrc_dump_default(struct drm_printer *p, + struct xe_gt *gt, + enum xe_engine_class hwe_class) +{ + u32 *dw, *start; + int remaining_dw, num_dw; + + if (!gt->default_lrc[hwe_class]) { + drm_printf(p, "No default LRC for class %d\n", hwe_class); + return; + } + + /* + * Skip the beginning of the LRC since it contains the per-process + * hardware status page. + */ + dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; + start = dw; + remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; + + while (remaining_dw > 0) { + if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) { + num_dw = dump_mi_command(p, gt, start, dw, remaining_dw); + } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) { + num_dw = dump_gfxpipe_command(p, gt, start, dw, remaining_dw); + } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) { + num_dw = dump_gfx_state_command(p, gt, start, dw, remaining_dw); + } else { + num_dw = min(instr_dw(*dw), remaining_dw); + drm_printf(p, "LRC[%#5tx] = [%#10x] Unknown instruction of type %#x, likely %d dwords\n", + dw - start, + *dw, REG_FIELD_GET(XE_INSTR_CMD_TYPE, *dw), + num_dw); + } + + dw += num_dw; + remaining_dw -= num_dw; + } +} + +/* + * Lookup the value of a register within the offset/value pairs of an + * MI_LOAD_REGISTER_IMM instruction. + * + * Return -ENOENT if the register is not present in the MI_LRI instruction. + */ +static int lookup_reg_in_mi_lri(u32 offset, u32 *value, + const u32 *dword_pair, int num_regs) +{ + for (int i = 0; i < num_regs; i++) { + if (dword_pair[2 * i] == offset) { + *value = dword_pair[2 * i + 1]; + return 0; + } + } + + return -ENOENT; +} + +/* + * Lookup the value of a register in a specific engine type's default LRC. + * + * Return -EINVAL if the default LRC doesn't exist, or ENOENT if the register + * cannot be found in the default LRC. + */ +int xe_lrc_lookup_default_reg_value(struct xe_gt *gt, + enum xe_engine_class hwe_class, + u32 offset, + u32 *value) +{ + u32 *dw; + int remaining_dw, ret; + + if (!gt->default_lrc[hwe_class]) + return -EINVAL; + + /* + * Skip the beginning of the LRC since it contains the per-process + * hardware status page. + */ + dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; + remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; + + while (remaining_dw > 0) { + u32 num_dw = instr_dw(*dw); + + if (num_dw > remaining_dw) + num_dw = remaining_dw; + + switch (*dw & XE_INSTR_CMD_TYPE) { + case XE_INSTR_MI: + switch (*dw & MI_OPCODE) { + case MI_BATCH_BUFFER_END: + /* End of LRC; register not found */ + return -ENOENT; + + case MI_NOOP: + case MI_TOPOLOGY_FILTER: + /* + * MI_NOOP and MI_TOPOLOGY_FILTER don't have + * a length field and are always 1-dword + * instructions. + */ + remaining_dw--; + dw++; + break; + + case MI_LOAD_REGISTER_IMM: + ret = lookup_reg_in_mi_lri(offset, value, + dw + 1, (num_dw - 1) / 2); + if (ret == 0) + return 0; + + fallthrough; + + default: + /* + * Jump to next instruction based on length + * field. + */ + remaining_dw -= num_dw; + dw += num_dw; + break; + } + break; + + default: + /* Jump to next instruction based on length field. */ + remaining_dw -= num_dw; + dw += num_dw; + } + } + + return -ENOENT; +} + +struct instr_state { + u32 instr; + u16 num_dw; +}; + +static const struct instr_state xe_hpg_svg_state[] = { + { .instr = CMD_3DSTATE_CONSTANT_VS, .num_dw = 11 }, + { .instr = CMD_3DSTATE_CONSTANT_HS, .num_dw = 11 }, + { .instr = CMD_3DSTATE_CONSTANT_DS, .num_dw = 11 }, + { .instr = CMD_3DSTATE_CONSTANT_GS, .num_dw = 11 }, + { .instr = CMD_3DSTATE_VERTEX_ELEMENTS, .num_dw = 69 }, + { .instr = CMD_3DSTATE_VF_COMPONENT_PACKING, .num_dw = 5 }, + { .instr = CMD_3DSTATE_VF_SGVS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_VF_SGVS_2, .num_dw = 3 }, + { .instr = CMD_3DSTATE_VS, .num_dw = 9 }, + { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_VS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_VS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_URB_ALLOC_VS, .num_dw = 3 }, + { .instr = CMD_3DSTATE_STREAMOUT, .num_dw = 5 }, + { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_0, .num_dw = 8 }, + { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_1, .num_dw = 8 }, + { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_2, .num_dw = 8 }, + { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_3, .num_dw = 8 }, + { .instr = CMD_3DSTATE_CLIP, .num_dw = 4 }, + { .instr = CMD_3DSTATE_PRIMITIVE_REPLICATION, .num_dw = 6 }, + { .instr = CMD_3DSTATE_CLIP_MESH, .num_dw = 2 }, + { .instr = CMD_3DSTATE_SF, .num_dw = 4 }, + { .instr = CMD_3DSTATE_SCISSOR_STATE_POINTERS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, .num_dw = 2 }, + { .instr = CMD_3DSTATE_RASTER, .num_dw = 5 }, + { .instr = CMD_3DSTATE_TBIMR_TILE_PASS_INFO, .num_dw = 4 }, + { .instr = CMD_3DSTATE_WM_HZ_OP, .num_dw = 6 }, + { .instr = CMD_3DSTATE_MULTISAMPLE, .num_dw = 2 }, + { .instr = CMD_3DSTATE_HS, .num_dw = 9 }, + { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_HS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_HS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_URB_ALLOC_HS, .num_dw = 3 }, + { .instr = CMD_3DSTATE_TASK_CONTROL, .num_dw = 3 }, + { .instr = CMD_3DSTATE_TASK_SHADER, .num_dw = 7 }, + { .instr = CMD_3DSTATE_TASK_SHADER_DATA, .num_dw = 10 }, + { .instr = CMD_3DSTATE_URB_ALLOC_TASK, .num_dw = 3 }, + { .instr = CMD_3DSTATE_TE, .num_dw = 5 }, + { .instr = CMD_3DSTATE_TASK_REDISTRIB, .num_dw = 2 }, + { .instr = CMD_3DSTATE_DS, .num_dw = 11 }, + { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_DS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_DS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_URB_ALLOC_DS, .num_dw = 3 }, + { .instr = CMD_3DSTATE_GS, .num_dw = 10 }, + { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_GS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_GS, .num_dw = 2 }, + { .instr = CMD_3DSTATE_URB_ALLOC_GS, .num_dw = 3 }, + { .instr = CMD_3DSTATE_MESH_CONTROL, .num_dw = 3 }, + { .instr = CMD_3DSTATE_MESH_SHADER_DATA, .num_dw = 10 }, + { .instr = CMD_3DSTATE_URB_ALLOC_MESH, .num_dw = 3 }, + { .instr = CMD_3DSTATE_MESH_SHADER, .num_dw = 8 }, + { .instr = CMD_3DSTATE_DRAWING_RECTANGLE, .num_dw = 4 }, +}; + +u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs) +{ + struct xe_gt *gt = q->hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + const struct instr_state *state_table = NULL; + int state_table_size = 0; + + /* + * Wa_14019789679 + * + * If the driver doesn't explicitly emit the SVG instructions while + * setting up the default LRC, the context switch will write 0's + * (noops) into the LRC memory rather than the expected instruction + * headers. Application contexts start out as a copy of the default + * LRC, and if they also do not emit specific settings for some SVG + * state, then on context restore they'll unintentionally inherit + * whatever state setting the previous context had programmed into the + * hardware (i.e., the lack of a 3DSTATE_* instruction in the LRC will + * prevent the hardware from resetting that state back to any specific + * value). + * + * The official workaround only requires emitting 3DSTATE_MESH_CONTROL + * since that's a specific state setting that can easily cause GPU + * hangs if unintentionally inherited. However to be safe we'll + * continue to emit all of the SVG state since it's best not to leak + * any of the state between contexts, even if that leakage is harmless. + */ + if (XE_GT_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { + state_table = xe_hpg_svg_state; + state_table_size = ARRAY_SIZE(xe_hpg_svg_state); + } + + if (!state_table) { + xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n", + GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); + return cs; + } + + for (int i = 0; i < state_table_size; i++) { + u32 instr = state_table[i].instr; + u16 num_dw = state_table[i].num_dw; + bool is_single_dw = ((instr & GFXPIPE_PIPELINE) == PIPELINE_SINGLE_DW); + + xe_gt_assert(gt, (instr & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE); + xe_gt_assert(gt, num_dw != 0); + xe_gt_assert(gt, is_single_dw ^ (num_dw > 1)); + + /* + * Xe2's SVG context is the same as the one on DG2 / MTL + * except that 3DSTATE_DRAWING_RECTANGLE (non-pipelined) has + * been replaced by 3DSTATE_DRAWING_RECTANGLE_FAST (pipelined). + * Just make the replacement here rather than defining a + * whole separate table for the single trivial change. + */ + if (GRAPHICS_VER(xe) >= 20 && + instr == CMD_3DSTATE_DRAWING_RECTANGLE) + instr = CMD_3DSTATE_DRAWING_RECTANGLE_FAST; + + *cs = instr; + if (!is_single_dw) + *cs |= (num_dw - 2); + + cs += num_dw; + } + + return cs; +} + +struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) +{ + struct xe_lrc_snapshot *snapshot = kmalloc_obj(*snapshot, GFP_NOWAIT); + + if (!snapshot) + return NULL; + + snapshot->context_desc = xe_lrc_ggtt_addr(lrc); + snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc); + snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc); + snapshot->head = xe_lrc_ring_head(lrc); + snapshot->tail.internal = lrc->ring.tail; + snapshot->tail.memory = xe_lrc_ring_tail(lrc); + snapshot->start = xe_lrc_ring_start(lrc); + snapshot->start_seqno = xe_lrc_start_seqno(lrc); + snapshot->seqno = xe_lrc_seqno(lrc); + snapshot->lrc_bo = xe_bo_get(lrc->bo); + snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); + snapshot->lrc_size = lrc->size; + snapshot->replay_offset = 0; + snapshot->replay_size = lrc->replay_size; + snapshot->lrc_snapshot = NULL; + snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); + snapshot->ctx_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); + if (xe_lrc_is_multi_queue(lrc)) { + snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc); + snapshot->queue_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp); + } else { + snapshot->queue_timestamp = 0; + snapshot->queue_timestamp_ms = 0; + } + snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); + return snapshot; +} + +void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot) +{ + struct xe_bo *bo; + struct iosys_map src; + + if (!snapshot) + return; + + bo = snapshot->lrc_bo; + snapshot->lrc_bo = NULL; + + snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL); + if (!snapshot->lrc_snapshot) + goto put_bo; + + xe_bo_lock(bo, false); + if (!ttm_bo_vmap(&bo->ttm, &src)) { + xe_map_memcpy_from(xe_bo_device(bo), + snapshot->lrc_snapshot, &src, snapshot->lrc_offset, + snapshot->lrc_size); + ttm_bo_vunmap(&bo->ttm, &src); + } else { + kvfree(snapshot->lrc_snapshot); + snapshot->lrc_snapshot = NULL; + } + xe_bo_unlock(bo); +put_bo: + xe_bo_put(bo); +} + +void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p) +{ + unsigned long i; + + if (!snapshot) + return; + + drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc); + drm_printf(p, "\tHW Ring address: 0x%08x\n", + snapshot->ring_addr); + drm_printf(p, "\tHW Indirect Ring State: 0x%08x\n", + snapshot->indirect_context_desc); + drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head); + drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n", + snapshot->tail.internal, snapshot->tail.memory); + drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); + drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); + drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); + drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); + drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp); + drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms); + drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); + + if (!snapshot->lrc_snapshot) + return; + + drm_printf(p, "\t[HWSP].length: 0x%x\n", LRC_PPHWSP_SIZE); + drm_puts(p, "\t[HWSP].data: "); + for (i = 0; i < LRC_PPHWSP_SIZE; i += sizeof(u32)) { + u32 *val = snapshot->lrc_snapshot + i; + char dumped[ASCII85_BUFSZ]; + + drm_puts(p, ascii85_encode(*val, dumped)); + } + + drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); + drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset); + drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size); + + drm_puts(p, "\t[HWCTX].data: "); + for (; i < snapshot->lrc_size; i += sizeof(u32)) { + u32 *val = snapshot->lrc_snapshot + i; + char dumped[ASCII85_BUFSZ]; + + drm_puts(p, ascii85_encode(*val, dumped)); + } + drm_puts(p, "\n"); +} + +void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) +{ + if (!snapshot) + return; + + kvfree(snapshot->lrc_snapshot); + if (snapshot->lrc_bo) + xe_bo_put(snapshot->lrc_bo); + + kfree(snapshot); +} + +static bool engine_valid_for_utilization(struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + /* The USM-reserved copy engine runs kernel migrate contexts queried here */ + return hwe && (!xe_hw_engine_is_reserved(hwe) || xe_gt_is_usm_hwe(gt, hwe)); +} + +static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id) +{ + u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); + u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); + struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false); + + if (xe_gt_WARN_ONCE(gt, !engine_valid_for_utilization(gt, hwe), + "Unexpected engine class:instance %d:%d for utilization\n", + class, instance)) + return NULL; + + return hwe; +} + +static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +{ + struct xe_hw_engine *hwe; + u64 val; + + hwe = engine_id_to_hwe(lrc->gt, engine_id); + if (!hwe) + return -1; + + if (lrc_to_xe(lrc)->info.has_64bit_timestamp) + val = xe_mmio_read64_2x32(&hwe->gt->mmio, + RING_CTX_TIMESTAMP(hwe->mmio_base)); + else + val = xe_mmio_read32(&hwe->gt->mmio, + RING_CTX_TIMESTAMP(hwe->mmio_base)); + + *reg_ctx_ts = val; + + return 0; +} + +static u64 get_queue_timestamp(struct xe_hw_engine *hwe) +{ + return xe_mmio_read64_2x32(&hwe->gt->mmio, + RING_QUEUE_TIMESTAMP(hwe->mmio_base)); +} + +static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) +{ + u32 val = xe_mmio_read32(&hwe->gt->mmio, + RING_CSMQDEBUG(hwe->mmio_base)); + + return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val); +} + +static bool context_active(struct xe_lrc *lrc) +{ + return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; +} + +static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc; + struct xe_hw_engine *hwe; + u64 reg_queue_ts = lrc->queue_timestamp; + + if (IS_SRIOV_VF(xe)) + return xe_lrc_queue_timestamp(lrc); + + xe_assert(xe, primary_lrc); + + /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */ + if (!context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + /* WA BB populates engine id in PPHWSP of primary context only */ + hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc)); + if (!hwe) + return xe_lrc_queue_timestamp(lrc); + + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) + return xe_lrc_queue_timestamp(lrc); + + /* queue is active, so store the queue timestamp register */ + reg_queue_ts = get_queue_timestamp(hwe); + + /* double check queue and primary queue are both still active */ + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || + !context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + return reg_queue_ts; +} + +static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->queue_timestamp; + lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); + + trace_xe_lrc_update_queue_timestamp(lrc, *old_ts); + + return lrc->queue_timestamp; +} + +static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) +{ + u64 reg_ts, new_ts = lrc->ctx_timestamp; + u64 stored; + + /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ + if (IS_SRIOV_VF(lrc_to_xe(lrc))) + return xe_lrc_ctx_timestamp(lrc); + + /* Safely read CTX_TIMESTAMP: Avoid TOCTOU on LRC-stored CONTEXT_ACTIVE sentinel */ + stored = xe_lrc_ctx_timestamp(lrc); + if (stored != CONTEXT_ACTIVE) + return stored; + + /* Context is active: read the live timestamp from the engine's MMIO register */ + if (!get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) + new_ts = reg_ts; + + /* If the context switched out prefer using the value + * from context-save over the stale MMIO read. + */ + stored = xe_lrc_ctx_timestamp(lrc); + if (stored != CONTEXT_ACTIVE) + return stored; + + return new_ts; +} + +static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->ctx_timestamp; + lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc); + + trace_xe_lrc_update_timestamp(lrc, *old_ts); + + return lrc->ctx_timestamp; +} + +/** + * xe_lrc_timestamp() - Current lrc timestamp + * @lrc: Pointer to the lrc. + * + * Return latest lrc timestamp. With support for active contexts/queues, the + * calculation may be slightly racy, so follow a read-again logic to ensure that + * the context/queue is still active before returning the right timestamp. + * + * Returns: New lrc timestamp value + */ +u64 xe_lrc_timestamp(struct xe_lrc *lrc) +{ + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_multi_queue_timestamp(lrc); + else + return xe_lrc_context_timestamp(lrc); +} + +/** + * xe_lrc_update_timestamp() - Update lrc timestamp + * @lrc: Pointer to the lrc. + * @old_ts: Old timestamp value + * + * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and + * update saved value. + * + * Returns: New lrc timestamp value + */ +u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_update_multi_queue_timestamp(lrc, old_ts); + else + return xe_lrc_update_context_timestamp(lrc, old_ts); +} + +/** + * xe_lrc_ring_is_idle() - LRC is idle + * @lrc: Pointer to the lrc. + * + * Compare LRC ring head and tail to determine if idle. + * + * Return: True is ring is idle, False otherwise + */ +bool xe_lrc_ring_is_idle(struct xe_lrc *lrc) +{ + return xe_lrc_ring_head(lrc) == xe_lrc_ring_tail(lrc); +} diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h new file mode 100644 index 000000000..7be5e3da8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ +#ifndef _XE_LRC_H_ +#define _XE_LRC_H_ + +#include + +#include "xe_lrc_types.h" + +/* + * Sentinel value stored in lrc->ctx_timestamp while a context is starting. + * The hardware hasn't yet written the real CTX_TIMESTAMP, so this is not a + * valid elapsed-time sample and must not be used as one. + */ +#define XE_LRC_CTX_TIMESTAMP_ACTIVE 1ULL + +struct drm_printer; +struct xe_bb; +struct xe_device; +struct xe_exec_queue; +enum xe_multi_queue_priority; +enum xe_engine_class; +struct xe_gt; +struct xe_hw_engine; +struct xe_lrc; +struct xe_vm; + +struct xe_lrc_snapshot { + struct xe_bo *lrc_bo; + void *lrc_snapshot; + unsigned long lrc_size, lrc_offset; + unsigned long replay_size, replay_offset; + + u32 context_desc; + u32 ring_addr; + u32 indirect_context_desc; + u32 head; + u32 start; + struct { + u32 internal; + u32 memory; + } tail; + u32 start_seqno; + u32 seqno; + u64 ctx_timestamp; + u64 ctx_timestamp_ms; + u64 queue_timestamp; + u64 queue_timestamp_ms; + u32 ctx_job_timestamp; +}; + +#define LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR (0x34 * 4) +#define LRC_PPHWSP_PXP_INVAL_SCRATCH_ADDR (0x40 * 4) + +#define LRC_WA_BB_SIZE SZ_4K + +#define XE_LRC_CREATE_RUNALONE BIT(0) +#define XE_LRC_CREATE_PXP BIT(1) +#define XE_LRC_CREATE_USER_CTX BIT(2) +#define XE_LRC_DISABLE_STATE_CACHE_PERF_FIX BIT(3) + +struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, + void *replay_state, u32 ring_size, u16 msix_vec, u32 flags); +void xe_lrc_destroy(struct kref *ref); + +/** + * xe_lrc_get - Get reference to the LRC + * @lrc: Logical Ring Context + * + * Increment reference count of @lrc + */ +static inline struct xe_lrc *xe_lrc_get(struct xe_lrc *lrc) +{ + kref_get(&lrc->refcount); + return lrc; +} + +/** + * xe_lrc_put - Put reference of the LRC + * @lrc: Logical Ring Context + * + * Decrement reference count of @lrc, call xe_lrc_destroy when + * reference count reaches 0. + */ +static inline void xe_lrc_put(struct xe_lrc *lrc) +{ + if (lrc) + kref_put(&lrc->refcount, xe_lrc_destroy); +} + +/** + * xe_lrc_ring_size() - Xe LRC ring size + * + * Return: Size of LRC ring buffer + */ +static inline size_t xe_lrc_ring_size(void) +{ + return SZ_16K; +} + +static inline bool xe_lrc_is_multi_queue(struct xe_lrc *lrc) +{ + return lrc->multi_queue.primary_lrc; +} + +size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class); +size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class); +u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc); +u32 xe_lrc_regs_offset(struct xe_lrc *lrc); + +void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail); +u32 xe_lrc_ring_tail(struct xe_lrc *lrc); +void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head); +u32 xe_lrc_ring_head(struct xe_lrc *lrc); +u32 xe_lrc_ring_space(struct xe_lrc *lrc); +void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size); + +bool xe_lrc_ring_is_idle(struct xe_lrc *lrc); + +u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc); +u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc); +u32 *xe_lrc_regs(struct xe_lrc *lrc); +void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc); +void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe); +void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *regs); + +u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr); +void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val); + +u64 xe_lrc_descriptor(struct xe_lrc *lrc); + +u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc); +struct dma_fence *xe_lrc_alloc_seqno_fence(void); +void xe_lrc_free_seqno_fence(struct dma_fence *fence); +void xe_lrc_init_seqno_fence(struct xe_lrc *lrc, struct dma_fence *fence); +s32 xe_lrc_seqno(struct xe_lrc *lrc); + +u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc); +s32 xe_lrc_start_seqno(struct xe_lrc *lrc); + +u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc); +struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc); + +size_t xe_lrc_reg_size(struct xe_device *xe); +size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class); + +void xe_lrc_dump_default(struct drm_printer *p, + struct xe_gt *gt, + enum xe_engine_class); +int xe_lrc_lookup_default_reg_value(struct xe_gt *gt, + enum xe_engine_class hwe_class, + u32 offset, + u32 *value); + +u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs); + +void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority); + +struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc); +void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot); +void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p); +void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot); + +u32 xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc *lrc); +u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc); +u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc); +u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc); +int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *scratch); + +/** + * xe_lrc_update_timestamp - readout LRC timestamp and update cached value + * @lrc: logical ring context for this exec queue + * @old_ts: pointer where to save the previous timestamp + * + * Read the current timestamp for this LRC and update the cached value. The + * previous cached value is also returned in @old_ts so the caller can calculate + * the delta between 2 updates. Note that this is not intended to be called from + * any place, but just by the paths updating the drm client utilization. + * + * Returns the current LRC timestamp + */ +u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts); + +u64 xe_lrc_timestamp(struct xe_lrc *lrc); + +#endif diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h new file mode 100644 index 000000000..53ef48fee --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_LRC_TYPES_H_ +#define _XE_LRC_TYPES_H_ + +#include + +#include "xe_hw_fence_types.h" + +struct xe_bo; + +/** + * struct xe_lrc - Logical ring context (LRC) and submission ring object + */ +struct xe_lrc { + /** + * @bo: buffer object (memory) for logical ring context, per process HW + * status page, and submission ring. + */ + struct xe_bo *bo; + + /** + * @seqno_bo: Buffer object (memory) for seqno numbers. Always in system + * memory as this a CPU read, GPU write path object. + */ + struct xe_bo *seqno_bo; + + /** @size: size of the lrc and optional indirect ring state */ + u32 size; + + /** @replay_size: Size LRC needed for replaying a hang */ + u32 replay_size; + + /** @gt: gt which this LRC belongs to */ + struct xe_gt *gt; + + /** @flags: LRC flags */ +#define XE_LRC_FLAG_INDIRECT_CTX 0x1 +#define XE_LRC_FLAG_INDIRECT_RING_STATE 0x2 + u32 flags; + + /** @refcount: ref count of this lrc */ + struct kref refcount; + + /** @ring: submission ring state */ + struct { + /** @ring.size: size of submission ring */ + u32 size; + /** @ring.tail: tail of submission ring */ + u32 tail; + /** @ring.old_tail: shadow of tail */ + u32 old_tail; + } ring; + + /** @desc: LRC descriptor */ + u64 desc; + + /** @fence_ctx: context for hw fence */ + struct xe_hw_fence_ctx fence_ctx; + + /** @ctx_timestamp: readout value of CTX_TIMESTAMP on last update */ + u64 ctx_timestamp; + + /** @queue_timestamp: value of QUEUE_TIMESTAMP on last update */ + u64 queue_timestamp; + + /** @multi_queue: Multi queue LRC related information */ + struct { + /** @multi_queue.primary_lrc: Primary lrc of this multi-queue group*/ + struct xe_lrc *primary_lrc; + /** @multi_queue.pos: Position of LRC within the multi-queue group */ + u8 pos; + } multi_queue; +}; + +struct xe_lrc_snapshot; + +#endif diff --git a/drivers/gpu/drm/xe/xe_macros.h b/drivers/gpu/drm/xe/xe_macros.h new file mode 100644 index 000000000..8a77c2423 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_macros.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_MACROS_H_ +#define _XE_MACROS_H_ + +#include + +#define XE_WARN_ON WARN_ON + +#define XE_IOCTL_DBG(xe, cond) ({ \ + int cond__ = !!(cond); \ + if (cond__) \ + drm_dbg(&(xe)->drm, \ + "Ioctl argument check failed at %s:%d: %s", \ + __FILE__, __LINE__, #cond); \ + cond__; \ +}) + +#endif diff --git a/drivers/gpu/drm/xe/xe_map.h b/drivers/gpu/drm/xe/xe_map.h new file mode 100644 index 000000000..c44777125 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_map.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_MAP_H_ +#define _XE_MAP_H_ + +#include + +#include + +/** + * DOC: Map layer + * + * All access to any memory shared with a device (both sysmem and vram) in the + * Xe driver should go through this layer (xe_map). This layer is built on top + * of :ref:`driver-api/device-io:Generalizing Access to System and I/O Memory` + * and with extra hooks into the Xe driver that allows adding asserts to memory + * accesses (e.g. for blocking runtime_pm D3Cold on Discrete Graphics). + */ + +static inline void xe_map_memcpy_to(struct xe_device *xe, struct iosys_map *dst, + size_t dst_offset, const void *src, + size_t len) +{ + xe_device_assert_mem_access(xe); + iosys_map_memcpy_to(dst, dst_offset, src, len); +} + +static inline void xe_map_memcpy_from(struct xe_device *xe, void *dst, + const struct iosys_map *src, + size_t src_offset, size_t len) +{ + xe_device_assert_mem_access(xe); + iosys_map_memcpy_from(dst, src, src_offset, len); +} + +static inline void xe_map_memset(struct xe_device *xe, + struct iosys_map *dst, size_t offset, + int value, size_t len) +{ + xe_device_assert_mem_access(xe); + iosys_map_memset(dst, offset, value, len); +} + +/* FIXME: We likely should kill these two functions sooner or later */ +static inline u32 xe_map_read32(struct xe_device *xe, struct iosys_map *map) +{ + xe_device_assert_mem_access(xe); + + if (map->is_iomem) + return readl(map->vaddr_iomem); + else + return READ_ONCE(*(u32 *)map->vaddr); +} + +static inline void xe_map_write32(struct xe_device *xe, struct iosys_map *map, + u32 val) +{ + xe_device_assert_mem_access(xe); + + if (map->is_iomem) + writel(val, map->vaddr_iomem); + else + *(u32 *)map->vaddr = val; +} + +#define xe_map_rd(xe__, map__, offset__, type__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_rd(map__, offset__, type__); \ +}) + +#define xe_map_wr(xe__, map__, offset__, type__, val__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_wr(map__, offset__, type__, val__); \ +}) + +#define xe_map_rd_field(xe__, map__, struct_offset__, struct_type__, field__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_rd_field(map__, struct_offset__, struct_type__, field__); \ +}) + +#define xe_map_wr_field(xe__, map__, struct_offset__, struct_type__, field__, val__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_wr_field(map__, struct_offset__, struct_type__, field__, val__); \ +}) + +#endif diff --git a/drivers/gpu/drm/xe/xe_mem_pool.c b/drivers/gpu/drm/xe/xe_mem_pool.c new file mode 100644 index 000000000..d5e24d6aa --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mem_pool.c @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include + +#include + +#include "instructions/xe_mi_commands.h" +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_map.h" +#include "xe_mem_pool.h" +#include "xe_mem_pool_types.h" +#include "xe_tile_printk.h" + +/** + * struct xe_mem_pool - DRM MM pool for sub-allocating memory from a BO on an + * XE tile. + * + * The XE memory pool is a DRM MM manager that provides sub-allocation of memory + * from a backing buffer object (BO) on a specific XE tile. It is designed to + * manage memory for GPU workloads, allowing for efficient allocation and + * deallocation of memory regions within the BO. + * + * The memory pool maintains a primary BO that is pinned in the GGTT and mapped + * into the CPU address space for direct access. Optionally, it can also maintain + * a shadow BO that can be used for atomic updates to the primary BO's contents. + * + * The API provided by the memory pool allows clients to allocate and free memory + * regions, retrieve GPU and CPU addresses, and synchronize data between the + * primary and shadow BOs as needed. + */ +struct xe_mem_pool { + /** @base: Range allocator over [0, @size) in bytes */ + struct drm_mm base; + /** @bo: Active pool BO (GGTT-pinned, CPU-mapped). */ + struct xe_bo *bo; + /** @shadow: Shadow BO for atomic command updates. */ + struct xe_bo *shadow; + /** @swap_guard: Timeline guard updating @bo and @shadow */ + struct mutex swap_guard; + /** @cpu_addr: CPU virtual address of the active BO. */ + void *cpu_addr; + /** @is_iomem: Indicates if the BO mapping is I/O memory. */ + bool is_iomem; +}; + +static struct xe_mem_pool *node_to_pool(struct xe_mem_pool_node *node) +{ + return container_of(node->sa_node.mm, struct xe_mem_pool, base); +} + +static struct xe_tile *pool_to_tile(struct xe_mem_pool *pool) +{ + return pool->bo->tile; +} + +static void fini_pool_action(struct drm_device *drm, void *arg) +{ + struct xe_mem_pool *pool = arg; + + if (pool->is_iomem) + kvfree(pool->cpu_addr); + + drm_mm_takedown(&pool->base); +} + +static int pool_shadow_init(struct xe_mem_pool *pool) +{ + struct xe_tile *tile = pool->bo->tile; + struct xe_device *xe = tile_to_xe(tile); + struct xe_bo *shadow; + int ret; + + xe_assert(xe, !pool->shadow); + + ret = drmm_mutex_init(&xe->drm, &pool->swap_guard); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&pool->swap_guard); + fs_reclaim_release(GFP_KERNEL); + } + shadow = xe_managed_bo_create_pin_map(xe, tile, + xe_bo_size(pool->bo), + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(shadow)) + return PTR_ERR(shadow); + + pool->shadow = shadow; + + return 0; +} + +/** + * xe_mem_pool_init() - Initialize memory pool. + * @tile: the &xe_tile where allocate. + * @size: number of bytes to allocate. + * @guard: the size of the guard region at the end of the BO that is not + * sub-allocated, in bytes. + * @flags: flags to use to create shadow pool. + * + * Initializes a memory pool for sub-allocating memory from a backing BO on the + * specified XE tile. The backing BO is pinned in the GGTT and mapped into + * the CPU address space for direct access. Optionally, a shadow BO can also be + * initialized for atomic updates to the primary BO's contents. + * + * Returns: a pointer to the &xe_mem_pool, or an error pointer on failure. + */ +struct xe_mem_pool *xe_mem_pool_init(struct xe_tile *tile, u32 size, + u32 guard, int flags) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_mem_pool *pool; + struct xe_bo *bo; + u32 managed_size; + int ret; + + xe_tile_assert(tile, size > guard); + managed_size = size - guard; + + pool = drmm_kzalloc(&xe->drm, sizeof(*pool), GFP_KERNEL); + if (!pool) + return ERR_PTR(-ENOMEM); + + bo = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) { + xe_tile_err(tile, "Failed to prepare %uKiB BO for mem pool (%pe)\n", + size / SZ_1K, bo); + return ERR_CAST(bo); + } + pool->bo = bo; + pool->is_iomem = bo->vmap.is_iomem; + + if (pool->is_iomem) { + pool->cpu_addr = kvzalloc(size, GFP_KERNEL); + if (!pool->cpu_addr) + return ERR_PTR(-ENOMEM); + } else { + pool->cpu_addr = bo->vmap.vaddr; + } + + if (flags & XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY) { + ret = pool_shadow_init(pool); + + if (ret) + goto out_err; + } + + drm_mm_init(&pool->base, 0, managed_size); + ret = drmm_add_action_or_reset(&xe->drm, fini_pool_action, pool); + if (ret) + return ERR_PTR(ret); + + return pool; + +out_err: + if (flags & XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY) + xe_tile_err(tile, + "Failed to initialize shadow BO for mem pool (%d)\n", ret); + if (bo->vmap.is_iomem) + kvfree(pool->cpu_addr); + return ERR_PTR(ret); +} + +/** + * xe_mem_pool_sync() - Copy the entire contents of the main pool to shadow pool. + * @pool: the memory pool containing the primary and shadow BOs. + * + * Copies the entire contents of the primary pool to the shadow pool. This must + * be done after xe_mem_pool_init() with the XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY + * flag to ensure that the shadow pool has the same initial contents as the primary + * pool. After this initial synchronization, clients can choose to synchronize the + * shadow pool with the primary pool on a node basis using + * xe_mem_pool_sync_shadow_locked() as needed. + * + * Return: None. + */ +void xe_mem_pool_sync(struct xe_mem_pool *pool) +{ + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + + xe_tile_assert(tile, pool->shadow); + + xe_map_memcpy_to(xe, &pool->shadow->vmap, 0, + pool->cpu_addr, xe_bo_size(pool->bo)); +} + +/** + * xe_mem_pool_swap_shadow_locked() - Swap the primary BO with the shadow BO. + * @pool: the memory pool containing the primary and shadow BOs. + * + * Swaps the primary buffer object with the shadow buffer object in the mem + * pool. This allows for atomic updates to the contents of the primary BO + * by first writing to the shadow BO and then swapping it with the primary BO. + * Swap_guard must be held to ensure synchronization with any concurrent swap + * operations. + * + * Return: None. + */ +void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool) +{ + struct xe_tile *tile = pool_to_tile(pool); + + xe_tile_assert(tile, pool->shadow); + lockdep_assert_held(&pool->swap_guard); + + swap(pool->bo, pool->shadow); + if (!pool->bo->vmap.is_iomem) + pool->cpu_addr = pool->bo->vmap.vaddr; +} + +/** + * xe_mem_pool_sync_shadow_locked() - Copy node from primary pool to shadow pool. + * @node: the node allocated in the memory pool. + * + * Copies the specified batch buffer from the primary pool to the shadow pool. + * Swap_guard must be held to ensure synchronization with any concurrent swap + * operations. + * + * Return: None. + */ +void xe_mem_pool_sync_shadow_locked(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + struct drm_mm_node *sa_node = &node->sa_node; + + xe_tile_assert(tile, pool->shadow); + lockdep_assert_held(&pool->swap_guard); + + xe_map_memcpy_to(xe, &pool->shadow->vmap, + sa_node->start, + pool->cpu_addr + sa_node->start, + sa_node->size); +} + +/** + * xe_mem_pool_gpu_addr() - Retrieve GPU address of memory pool. + * @pool: the memory pool + * + * Returns: GGTT address of the memory pool. + */ +u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool) +{ + return xe_bo_ggtt_addr(pool->bo); +} + +/** + * xe_mem_pool_cpu_addr() - Retrieve CPU address of manager pool. + * @pool: the memory pool + * + * Returns: CPU virtual address of memory pool. + */ +void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool) +{ + return pool->cpu_addr; +} + +/** + * xe_mem_pool_bo_swap_guard() - Retrieve the mutex used to guard swap + * operations on a memory pool. + * @pool: the memory pool + * + * Returns: Swap guard mutex or NULL if shadow pool is not created. + */ +struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool) +{ + if (!pool->shadow) + return NULL; + + return &pool->swap_guard; +} + +/** + * xe_mem_pool_bo_flush_write() - Copy the data from the sub-allocation + * to the GPU memory. + * @node: the node allocated in the memory pool to flush. + */ +void xe_mem_pool_bo_flush_write(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + struct drm_mm_node *sa_node = &node->sa_node; + + if (!pool->bo->vmap.is_iomem) + return; + + xe_map_memcpy_to(xe, &pool->bo->vmap, sa_node->start, + pool->cpu_addr + sa_node->start, + sa_node->size); +} + +/** + * xe_mem_pool_bo_sync_read() - Copy the data from GPU memory to the + * sub-allocation. + * @node: the node allocated in the memory pool to read back. + */ +void xe_mem_pool_bo_sync_read(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + struct drm_mm_node *sa_node = &node->sa_node; + + if (!pool->bo->vmap.is_iomem) + return; + + xe_map_memcpy_from(xe, pool->cpu_addr + sa_node->start, + &pool->bo->vmap, sa_node->start, sa_node->size); +} + +/** + * xe_mem_pool_alloc_node() - Allocate a new node for use with xe_mem_pool. + * + * Returns: node structure or an ERR_PTR(-ENOMEM). + */ +struct xe_mem_pool_node *xe_mem_pool_alloc_node(void) +{ + struct xe_mem_pool_node *node = kzalloc_obj(*node); + + if (!node) + return ERR_PTR(-ENOMEM); + + return node; +} + +/** + * xe_mem_pool_insert_node() - Insert a node into the memory pool. + * @pool: the memory pool to insert into + * @node: the node to insert + * @size: the size of the node to be allocated in bytes. + * + * Inserts a node into the specified memory pool using drm_mm for + * allocation. + * + * Returns: 0 on success or a negative error code on failure. + */ +int xe_mem_pool_insert_node(struct xe_mem_pool *pool, + struct xe_mem_pool_node *node, u32 size) +{ + if (!pool) + return -EINVAL; + + return drm_mm_insert_node(&pool->base, &node->sa_node, size); +} + +/** + * xe_mem_pool_free_node() - Free a node allocated from the memory pool. + * @node: the node to free + * + * Returns: None. + */ +void xe_mem_pool_free_node(struct xe_mem_pool_node *node) +{ + if (!node) + return; + + drm_mm_remove_node(&node->sa_node); + kfree(node); +} + +/** + * xe_mem_pool_node_cpu_addr() - Retrieve CPU address of the node. + * @node: the node allocated in the memory pool + * + * Returns: CPU virtual address of the node. + */ +void *xe_mem_pool_node_cpu_addr(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + + return xe_mem_pool_cpu_addr(pool) + node->sa_node.start; +} + +/** + * xe_mem_pool_dump() - Dump the state of the DRM MM manager for debugging. + * @pool: the memory pool info be dumped. + * @p: The DRM printer to use for output. + * + * Only the drm managed region is dumped, not the state of the BOs or any other + * pool information. + * + * Returns: None. + */ +void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p) +{ + drm_mm_print(&pool->base, p); +} diff --git a/drivers/gpu/drm/xe/xe_mem_pool.h b/drivers/gpu/drm/xe/xe_mem_pool.h new file mode 100644 index 000000000..89cd2555f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mem_pool.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ +#ifndef _XE_MEM_POOL_H_ +#define _XE_MEM_POOL_H_ + +#include +#include + +#include +#include "xe_mem_pool_types.h" + +struct drm_printer; +struct xe_mem_pool; +struct xe_tile; + +struct xe_mem_pool *xe_mem_pool_init(struct xe_tile *tile, u32 size, + u32 guard, int flags); +void xe_mem_pool_sync(struct xe_mem_pool *pool); +void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool); +void xe_mem_pool_sync_shadow_locked(struct xe_mem_pool_node *node); +u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool); +void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool); +struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool); +void xe_mem_pool_bo_flush_write(struct xe_mem_pool_node *node); +void xe_mem_pool_bo_sync_read(struct xe_mem_pool_node *node); +struct xe_mem_pool_node *xe_mem_pool_alloc_node(void); +int xe_mem_pool_insert_node(struct xe_mem_pool *pool, + struct xe_mem_pool_node *node, u32 size); +void xe_mem_pool_free_node(struct xe_mem_pool_node *node); +void *xe_mem_pool_node_cpu_addr(struct xe_mem_pool_node *node); +void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_mem_pool_types.h b/drivers/gpu/drm/xe/xe_mem_pool_types.h new file mode 100644 index 000000000..d5e926c93 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mem_pool_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_MEM_POOL_TYPES_H_ +#define _XE_MEM_POOL_TYPES_H_ + +#include + +#define XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY BIT(0) + +/** + * struct xe_mem_pool_node - Sub-range allocations from mem pool. + */ +struct xe_mem_pool_node { + /** @sa_node: drm_mm_node for this allocation. */ + struct drm_mm_node sa_node; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c new file mode 100644 index 000000000..9dfe965cb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_memirq.c @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include "regs/xe_guc_regs.h" +#include "regs/xe_irq_regs.h" + +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_hw_engine.h" +#include "xe_memirq.h" +#include "xe_tile_printk.h" + +#define memirq_assert(m, condition) xe_tile_assert(memirq_to_tile(m), condition) +#define memirq_printk(m, _level, _fmt, ...) \ + xe_tile_##_level(memirq_to_tile(m), "MEMIRQ: " _fmt, ##__VA_ARGS__) + +#ifdef CONFIG_DRM_XE_DEBUG_MEMIRQ +#define memirq_debug(m, _fmt, ...) memirq_printk(m, dbg, _fmt, ##__VA_ARGS__) +#else +#define memirq_debug(...) +#endif + +#define memirq_err(m, _fmt, ...) memirq_printk(m, err, _fmt, ##__VA_ARGS__) +#define memirq_err_ratelimited(m, _fmt, ...) \ + memirq_printk(m, err_ratelimited, _fmt, ##__VA_ARGS__) + +static struct xe_tile *memirq_to_tile(struct xe_memirq *memirq) +{ + return container_of(memirq, struct xe_tile, memirq); +} + +static struct xe_device *memirq_to_xe(struct xe_memirq *memirq) +{ + return tile_to_xe(memirq_to_tile(memirq)); +} + +static const char *guc_name(struct xe_guc *guc) +{ + return xe_gt_is_media_type(guc_to_gt(guc)) ? "media GuC" : "GuC"; +} + +/** + * DOC: Memory Based Interrupts + * + * MMIO register based interrupts infrastructure used for non-virtualized mode + * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently + * to allow delivering interrupts to a large number of Virtual machines or + * containers. Memory based interrupt status reporting provides an efficient + * and scalable infrastructure. + * + * For memory based interrupt status reporting hardware sequence is: + * * Engine writes the interrupt event to memory + * (Pointer to memory location is provided by SW. This memory surface must + * be mapped to system memory and must be marked as un-cacheable (UC) on + * Graphics IP Caches) + * * Engine triggers an interrupt to host. + */ + +/** + * DOC: Memory Based Interrupts Page Layout + * + * `Memory Based Interrupts`_ requires three different objects, which are + * called "page" in the specs, even if they aren't page-sized or aligned. + * + * To simplify the code we allocate a single page size object and then use + * offsets to embedded "pages". The address of those "pages" are then + * programmed in the HW via LRI and LRM in the context image. + * + * - _`Interrupt Status Report Page`: this page contains the interrupt + * status vectors for each unit. Each bit in the interrupt vectors is + * converted to a byte, with the byte being set to 0xFF when an + * interrupt is triggered; interrupt vectors are 16b big so each unit + * gets 16B. One space is reserved for each bit in one of the + * GT_INTR_DWx registers, so this object needs a total of 1024B. + * This object needs to be 4KiB aligned. + * + * - _`Interrupt Source Report Page`: this is the equivalent of the + * GT_INTR_DWx registers, with each bit in those registers being + * mapped to a byte here. The offsets are the same, just bytes instead + * of bits. This object needs to be cacheline aligned. + * + * - Interrupt Mask: the HW needs a location to fetch the interrupt + * mask vector to be used by the LRM in the context, so we just use + * the next available space in the interrupt page. + * + * :: + * + * 0x0000 +===========+ <== Interrupt Status Report Page + * | | + * | | ____ +----+----------------+ + * | | / | 0 | USER INTERRUPT | + * +-----------+ __/ | 1 | | + * | HWE(n) | __ | | CTX SWITCH | + * +-----------+ \ | | WAIT SEMAPHORE | + * | | \____ | 15 | | + * | | +----+----------------+ + * | | + * 0x0400 +===========+ <== Interrupt Source Report Page + * | HWE(0) | + * | HWE(1) | + * | | + * | HWE(x) | + * 0x0440 +===========+ <== Interrupt Enable Mask + * | | + * | | + * +-----------+ + * + * + * MSI-X use case + * + * When using MSI-X, hw engines report interrupt status and source to engine + * instance 0. For this scenario, in order to differentiate between the + * engines, we need to pass different status/source pointers in the LRC. + * + * The requirements on those pointers are: + * - Interrupt status should be 4KiB aligned + * - Interrupt source should be 64 bytes aligned + * + * To accommodate this, we duplicate the memirq page layout above - + * allocating a page for each engine instance and pass this page in the LRC. + * Note that the same page can be reused for different engine types. + * For example, an LRC executing on CCS #x will have pointers to page #x, + * and an LRC executing on BCS #x will have the same pointers. + * + * :: + * + * 0x0000 +==============================+ <== page for instance 0 (BCS0, CCS0, etc.) + * | Interrupt Status Report Page | + * 0x0400 +==============================+ + * | Interrupt Source Report Page | + * 0x0440 +==============================+ + * | Interrupt Enable Mask | + * +==============================+ + * | Not used | + * 0x1000 +==============================+ <== page for instance 1 (BCS1, CCS1, etc.) + * | Interrupt Status Report Page | + * 0x1400 +==============================+ + * | Interrupt Source Report Page | + * 0x1440 +==============================+ + * | Not used | + * 0x2000 +==============================+ <== page for instance 2 (BCS2, CCS2, etc.) + * | ... | + * +==============================+ + * + */ + +/* ISR */ +#define XE_MEMIRQ_STATUS_OFFSET(inst) ((inst) * SZ_4K + 0x0) +/* IIR */ +#define XE_MEMIRQ_SOURCE_OFFSET(inst) ((inst) * SZ_4K + 0x400) +/* IMR */ +#define XE_MEMIRQ_ENABLE_OFFSET 0x440 + +/* engine ISR vector offset */ +#define XE_MEMIRQ_VECTOR_OFFSET(page, source) (XE_MEMIRQ_STATUS_OFFSET(page) + (source) * SZ_16) + +static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq) +{ + /* + * When the HW engines are configured to use MSI-X, + * they report interrupt status and source to the offset of + * engine instance 0. + */ + return xe_device_has_msix(memirq_to_xe(memirq)); +} + +static unsigned int hwe_max_count(struct xe_tile *tile) +{ + unsigned int max_instance = 0; + unsigned int gtid, hweid; + struct xe_hw_engine *hwe; + struct xe_gt *gt; + + for_each_gt_on_tile(gt, tile, gtid) + for_each_hw_engine(hwe, gt, hweid) + max_instance = max(max_instance, hwe->instance); + + return max_instance + 1; +} + +static int memirq_alloc_pages(struct xe_memirq *memirq) +{ + struct xe_device *xe = memirq_to_xe(memirq); + struct xe_tile *tile = memirq_to_tile(memirq); + unsigned int num_pages; + struct xe_bo *bo; + size_t bo_size; + int err; + + BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET(0), SZ_64)); + BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET(0), SZ_4K)); + + num_pages = hw_reports_to_instance_zero(memirq) ? hwe_max_count(tile) : 1; + bo_size = num_pages * SZ_4K; + + bo = xe_managed_bo_create_pin_map(xe, tile, bo_size, + XE_BO_FLAG_SYSTEM | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_NEEDS_UC | + XE_BO_FLAG_NEEDS_CPU_ACCESS); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + goto out; + } + + memirq_assert(memirq, !xe_bo_is_vram(bo)); + memirq_assert(memirq, !memirq->bo); + + iosys_map_memset(&bo->vmap, 0, 0, bo_size); + + memirq->bo = bo; + memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0)); + memirq->num_pages = num_pages; + + memirq_assert(memirq, !memirq->source.is_iomem); + + memirq_debug(memirq, "pages: count %u size %zu\n", num_pages, bo_size); + memirq_debug(memirq, "page0: source %#x status %#x mask %#x\n", + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_SOURCE_OFFSET(0), + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_STATUS_OFFSET(0), + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_ENABLE_OFFSET); + + return 0; + +out: + memirq_err(memirq, "Failed to allocate memirq page (%pe)\n", ERR_PTR(err)); + return err; +} + +static void memirq_set_enable(struct xe_memirq *memirq, bool enable) +{ + /* + * We only care about the GT_MI_USER_INTERRUPT from the engines and + * the GuC does not look at the ENABLE mask at all. + */ + iosys_map_wr(&memirq->bo->vmap, XE_MEMIRQ_ENABLE_OFFSET, u32, + enable ? GT_MI_USER_INTERRUPT : 0); + + memirq->enabled = enable; +} + +/** + * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_. + * @memirq: the &xe_memirq to initialize + * + * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_ + * used by `Memory Based Interrupts`_. + * + * These allocations are managed and will be implicitly released on unload. + * + * If this function fails then the driver won't be able to operate correctly. + * If `Memory Based Interrupts`_ are not used this function will return 0. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_memirq_init(struct xe_memirq *memirq) +{ + struct xe_device *xe = memirq_to_xe(memirq); + int err; + + if (!xe_device_uses_memirq(xe)) + return 0; + + err = memirq_alloc_pages(memirq); + if (unlikely(err)) + return err; + + /* we need to start with all irqs enabled */ + memirq_set_enable(memirq, true); + + return 0; +} + +/** + * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_. + * @memirq: the &xe_memirq to query + * @hwe: the hw engine for which we want the report page + * + * Shall be called when `Memory Based Interrupts`_ are used + * and xe_memirq_init() didn't fail. + * + * Return: GGTT's offset of the `Interrupt Source Report Page`_. + */ +u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe) +{ + memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); + + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page); +} + +/** + * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_. + * @memirq: the &xe_memirq to query + * @hwe: the hw engine for which we want the report page + * + * Shall be called when `Memory Based Interrupts`_ are used + * and xe_memirq_init() didn't fail. + * + * Return: GGTT's offset of the `Interrupt Status Report Page`_. + */ +u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe) +{ + memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); + + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(hwe->irq_page); +} + +/** + * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask. + * @memirq: the &xe_memirq to query + * + * Shall be called when `Memory Based Interrupts`_ are used + * and xe_memirq_init() didn't fail. + * + * Return: GGTT's offset of the Interrupt Enable Mask. + */ +u32 xe_memirq_enable_ptr(struct xe_memirq *memirq) +{ + memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); + memirq_assert(memirq, memirq->bo); + + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET; +} + +/** + * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_. + * @memirq: the &xe_memirq + * @guc: the &xe_guc to setup + * + * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_ + * to be used by the GuC when `Memory Based Interrupts`_ are required. + * + * Shall be called when `Memory Based Interrupts`_ are used + * and xe_memirq_init() didn't fail. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc) +{ + bool is_media = xe_gt_is_media_type(guc_to_gt(guc)); + u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); + u64 source, status; + int err; + + memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); + + /* GuC expects exact locations, it doesn't add anything on its own */ + source = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(0) + offset; + status = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_VECTOR_OFFSET(0, offset); + + err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY, + source); + if (unlikely(err)) + goto failed; + + err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY, + status); + if (unlikely(err)) + goto failed; + + return 0; + +failed: + memirq_err(memirq, "Failed to setup report pages in %s (%pe)\n", + guc_name(guc), ERR_PTR(err)); + return err; +} + +/** + * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_. + * @memirq: struct xe_memirq + * + * This is part of the driver IRQ setup flow. + * + * This function shall only be used on platforms that use + * `Memory Based Interrupts`_. + */ +void xe_memirq_reset(struct xe_memirq *memirq) +{ + memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); + + if (memirq->bo) + memirq_set_enable(memirq, false); +} + +/** + * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_. + * @memirq: the &xe_memirq + * + * This is part of the driver IRQ setup flow. + * + * This function shall only be used on platforms that use + * `Memory Based Interrupts`_. + */ +void xe_memirq_postinstall(struct xe_memirq *memirq) +{ + memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); + + if (memirq->bo) + memirq_set_enable(memirq, true); +} + +static bool __memirq_received(struct xe_memirq *memirq, + struct iosys_map *vector, u16 offset, + const char *name, bool clear) +{ + u8 value; + + value = iosys_map_rd(vector, offset, u8); + if (value) { + if (value != 0xff) + memirq_err_ratelimited(memirq, + "Unexpected memirq value %#x from %s at %u\n", + value, name, offset); + if (clear) + iosys_map_wr(vector, offset, u8, 0x00); + } + + return value; +} + +static bool memirq_received_noclear(struct xe_memirq *memirq, + struct iosys_map *vector, + u16 offset, const char *name) +{ + return __memirq_received(memirq, vector, offset, name, false); +} + +static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector, + u16 offset, const char *name) +{ + return __memirq_received(memirq, vector, offset, name, true); +} + +static void memirq_assume_received(struct xe_memirq *memirq, const char *source, + u16 offset, const char *status) +{ + memirq_debug(memirq, "ASSUME %s %s(%u)\n", source, status, offset); +} + +static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status, + struct xe_hw_engine *hwe) +{ + memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr); + + /* + * The programming note says to assume that GT_MI_USER_INTERRUPT is always + * set. Check and clear related status byte just for a debug. + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) && + !memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name)) + memirq_assume_received(memirq, hwe->name, ilog2(GT_MI_USER_INTERRUPT), "USER"); + xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT); +} + +static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status, + struct xe_guc *guc) +{ + const char *name = guc_name(guc); + + memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr); + + /* + * The programming note says to assume that GUC_INTR_GUC2HOST is always + * set. Check and clear related status byte just for a debug. + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) && + !memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) + memirq_assume_received(memirq, name, ilog2(GUC_INTR_GUC2HOST), "GUC2HOST"); + xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); + + /* + * This is a software interrupt that must be cleared after it's consumed + * to avoid race conditions where xe_gt_sriov_vf_recovery_pending() + * returns false. + */ + if (memirq_received_noclear(memirq, status, ilog2(GUC_INTR_SW_INT_0), + name)) { + xe_guc_irq_handler(guc, GUC_INTR_SW_INT_0); + iosys_map_wr(status, ilog2(GUC_INTR_SW_INT_0), u8, 0x00); + } +} + +/** + * xe_memirq_hwe_handler - Check and process interrupts for a specific HW engine. + * @memirq: the &xe_memirq + * @hwe: the hw engine to process + * + * This function reads and dispatches `Memory Based Interrupts` for the provided HW engine. + */ +void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe) +{ + struct iosys_map source = + IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page)); + + if (memirq_received(memirq, &source, hwe->irq_offset, "SRC")) { + struct iosys_map status = + IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(hwe->irq_page, + hwe->irq_offset)); + memirq_dispatch_engine(memirq, &status, hwe); + } +} + +/** + * xe_memirq_guc_sw_int_0_irq_pending() - SW_INT_0 IRQ is pending + * @memirq: the &xe_memirq + * @guc: the &xe_guc to check for IRQ + * + * Return: True if SW_INT_0 IRQ is pending on @guc, False otherwise + */ +bool xe_memirq_guc_sw_int_0_irq_pending(struct xe_memirq *memirq, struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 offset = xe_gt_is_media_type(gt) ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); + struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, offset)); + + return memirq_received_noclear(memirq, &map, ilog2(GUC_INTR_SW_INT_0), + guc_name(guc)); +} + +static void memirq_dump_source_pages(struct xe_memirq *memirq) +{ + memirq_assert(memirq, !memirq->bo->vmap.is_iomem); + + for (int n = 0; n < memirq->num_pages; n++) { + memirq_debug(memirq, "SOURCE %*ph\n", 32, + memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n)); + memirq_debug(memirq, "SOURCE %*ph\n", 32, + memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n) + 32); + } +} + +/** + * xe_memirq_handler - The `Memory Based Interrupts`_ Handler. + * @memirq: the &xe_memirq + * + * This function reads and dispatches `Memory Based Interrupts`. + */ +void xe_memirq_handler(struct xe_memirq *memirq) +{ + struct xe_device *xe = memirq_to_xe(memirq); + struct xe_tile *tile = memirq_to_tile(memirq); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct iosys_map map; + unsigned int gtid; + struct xe_gt *gt; + + if (!memirq->bo) + return; + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ)) + memirq_dump_source_pages(memirq); + + for_each_gt(gt, xe, gtid) { + if (gt->tile != tile) + continue; + + for_each_hw_engine(hwe, gt, id) + xe_memirq_hwe_handler(memirq, hwe); + } + + /* GuC and media GuC (if present) must be checked separately */ + + if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) { + map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_GUC))); + memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc); + } + + if (!tile->media_gt) + return; + + if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) { + map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_MGUC))); + memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc); + } +} diff --git a/drivers/gpu/drm/xe/xe_memirq.h b/drivers/gpu/drm/xe/xe_memirq.h new file mode 100644 index 000000000..e25d2234a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_memirq.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_MEMIRQ_H_ +#define _XE_MEMIRQ_H_ + +#include + +struct xe_guc; +struct xe_hw_engine; +struct xe_memirq; + +int xe_memirq_init(struct xe_memirq *memirq); + +u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe); +u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe); +u32 xe_memirq_enable_ptr(struct xe_memirq *memirq); + +void xe_memirq_reset(struct xe_memirq *memirq); +void xe_memirq_postinstall(struct xe_memirq *memirq); +void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe); +void xe_memirq_handler(struct xe_memirq *memirq); + +int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc); + +bool xe_memirq_guc_sw_int_0_irq_pending(struct xe_memirq *memirq, struct xe_guc *guc); + +#endif diff --git a/drivers/gpu/drm/xe/xe_memirq_types.h b/drivers/gpu/drm/xe/xe_memirq_types.h new file mode 100644 index 000000000..c2b0b33d5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_memirq_types.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_MEMIRQ_TYPES_H_ +#define _XE_MEMIRQ_TYPES_H_ + +#include + +struct xe_bo; + +/** + * struct xe_memirq - Data used by the `Memory Based Interrupts`_. + * + * @bo: buffer object with `Memory Based Interrupts Page Layout`_. + * @num_pages: number of per-instance source/status pages. + * @source: iosys pointer to `Interrupt Source Report Page`_. + * @enabled: internal flag used to control processing of the interrupts. + */ +struct xe_memirq { + struct xe_bo *bo; + unsigned int num_pages; + struct iosys_map source; + bool enabled; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_mert.c b/drivers/gpu/drm/xe/xe_mert.c new file mode 100644 index 000000000..f637df954 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mert.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2025, Intel Corporation. All rights reserved. + */ + +#include "regs/xe_irq_regs.h" +#include "regs/xe_mert_regs.h" + +#include "xe_device.h" +#include "xe_mert.h" +#include "xe_mmio.h" +#include "xe_sriov_printk.h" +#include "xe_tile.h" + +/** + * xe_mert_init_early() - Initialize MERT data + * @xe: the &xe_device with MERT to init + */ +void xe_mert_init_early(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_mert *mert = &tile->mert; + + spin_lock_init(&mert->lock); + init_completion(&mert->tlb_inv_done); +} + +/** + * xe_mert_invalidate_lmtt() - Invalidate MERT LMTT + * @xe: the &xe_device with MERT + * + * Trigger invalidation of the MERT LMTT and wait for completion. + * + * Return: 0 on success or -ETIMEDOUT in case of a timeout. + */ +int xe_mert_invalidate_lmtt(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_mert *mert = &tile->mert; + const long timeout = HZ / 4; + unsigned long flags; + + xe_assert(xe, xe_device_has_mert(xe)); + + spin_lock_irqsave(&mert->lock, flags); + if (!mert->tlb_inv_triggered) { + mert->tlb_inv_triggered = true; + reinit_completion(&mert->tlb_inv_done); + xe_mmio_write32(&tile->mmio, MERT_TLB_INV_DESC_A, MERT_TLB_INV_DESC_A_VALID); + } + spin_unlock_irqrestore(&mert->lock, flags); + + if (!wait_for_completion_timeout(&mert->tlb_inv_done, timeout)) + return -ETIMEDOUT; + + return 0; +} + +static void mert_handle_cat_error(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + u32 reg_val, vfid, code; + + reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT); + if (!reg_val) + return; + xe_mmio_write32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT, 0); + + vfid = FIELD_GET(CATERR_VFID, reg_val); + code = FIELD_GET(CATERR_CODES, reg_val); + + switch (code) { + case CATERR_NO_ERROR: + break; + case CATERR_UNMAPPED_GGTT: + xe_sriov_err(xe, "MERT: CAT_ERR: Access to an unmapped GGTT!\n"); + xe_device_declare_wedged(xe); + break; + case CATERR_LMTT_FAULT: + xe_sriov_dbg(xe, "MERT: CAT_ERR: VF%u LMTT fault!\n", vfid); + /* XXX: track/report malicious VF activity */ + break; + default: + xe_sriov_err(xe, "MERT: Unexpected CAT_ERR code=%#x!\n", code); + xe_device_declare_wedged(xe); + break; + } +} + +/** + * xe_mert_irq_handler - Handler for MERT interrupts + * @xe: the &xe_device + * @master_ctl: interrupt register + * + * Handle interrupts generated by MERT. + */ +void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_mert *mert = &tile->mert; + unsigned long flags; + u32 reg_val; + + if (!(master_ctl & SOC_H2DMEMINT_IRQ)) + return; + + mert_handle_cat_error(xe); + + spin_lock_irqsave(&mert->lock, flags); + if (mert->tlb_inv_triggered) { + reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_INV_DESC_A); + if (!(reg_val & MERT_TLB_INV_DESC_A_VALID)) { + mert->tlb_inv_triggered = false; + complete_all(&mert->tlb_inv_done); + } + } + spin_unlock_irqrestore(&mert->lock, flags); +} diff --git a/drivers/gpu/drm/xe/xe_mert.h b/drivers/gpu/drm/xe/xe_mert.h new file mode 100644 index 000000000..fc9772036 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mert.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2025, Intel Corporation. All rights reserved. + */ + +#ifndef _XE_MERT_H_ +#define _XE_MERT_H_ + +#include +#include +#include + +struct xe_device; + +/** + * struct xe_mert - MERT related data + */ +struct xe_mert { + /** @lock: protects the TLB invalidation status */ + spinlock_t lock; + /** @tlb_inv_triggered: indicates if TLB invalidation was triggered */ + bool tlb_inv_triggered; + /** @tlb_inv_done: completion of TLB invalidation */ + struct completion tlb_inv_done; +}; + +#ifdef CONFIG_PCI_IOV +void xe_mert_init_early(struct xe_device *xe); +int xe_mert_invalidate_lmtt(struct xe_device *xe); +void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl); +#else +static inline void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c new file mode 100644 index 000000000..f79d0047b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -0,0 +1,2637 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "xe_migrate.h" + +#include +#include + +#include +#include +#include +#include + +#include + +#include "instructions/xe_gpu_commands.h" +#include "instructions/xe_mi_commands.h" +#include "regs/xe_gtt_defs.h" +#include "tests/xe_test.h" +#include "xe_assert.h" +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_exec_queue.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_mem_pool.h" +#include "xe_mocs.h" +#include "xe_pat.h" +#include "xe_printk.h" +#include "xe_pt.h" +#include "xe_res_cursor.h" +#include "xe_sa.h" +#include "xe_sched_job.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_svm.h" +#include "xe_sync.h" +#include "xe_trace_bo.h" +#include "xe_validation.h" +#include "xe_vm.h" +#include "xe_vram.h" + +/** + * struct xe_migrate - migrate context. + */ +struct xe_migrate { + /** @q: Default exec queue used for migration */ + struct xe_exec_queue *q; + /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ + struct xe_tile *tile; + /** @job_mutex: Timeline mutex for @eng. */ + struct mutex job_mutex; + /** @pt_bo: Page-table buffer object. */ + struct xe_bo *pt_bo; + /** @batch_base_ofs: VM offset of the migration batch buffer */ + u64 batch_base_ofs; + /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ + u64 usm_batch_base_ofs; + /** @cleared_mem_ofs: VM offset of @cleared_bo. */ + u64 cleared_mem_ofs; + /** @large_page_copy_ofs: VM offset of 2M pages used for large copies */ + u64 large_page_copy_ofs; + /** + * @large_page_copy_pdes: BO offset to writeout 2M pages (PDEs) used for + * large copies + */ + u64 large_page_copy_pdes; + /** + * @fence: dma-fence representing the last migration job batch. + * Protected by @job_mutex. + */ + struct dma_fence *fence; + /** + * @vm_update_sa: For integrated, used to suballocate page-tables + * out of the pt_bo. + */ + struct drm_suballoc_manager vm_update_sa; + /** @min_chunk_size: For dgfx, Minimum chunk size */ + u64 min_chunk_size; +}; + +#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ +#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ +#define NUM_KERNEL_PDE 15 +#define NUM_PT_SLOTS 32 +#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M +#define MAX_NUM_PTE 512 +#define IDENTITY_OFFSET 256ULL + +/* + * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest + * legal value accepted. Since that instruction field is always stored in + * (val-2) format, this translates to 0x400 dwords for the true maximum length + * of the instruction. Subtracting the instruction header (1 dword) and + * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. + */ +#define MAX_PTE_PER_SDI 0x1FEU + +static void xe_migrate_fini(void *arg) +{ + struct xe_migrate *m = arg; + + xe_vm_lock(m->q->vm, false); + xe_bo_unpin(m->pt_bo); + xe_vm_unlock(m->q->vm); + + dma_fence_put(m->fence); + xe_bo_put(m->pt_bo); + drm_suballoc_manager_fini(&m->vm_update_sa); + mutex_destroy(&m->job_mutex); + xe_vm_close_and_put(m->q->vm); + xe_exec_queue_put(m->q); +} + +static inline u16 xe_migrate_pat_index(struct xe_device *xe, + enum ttm_caching caching, + bool is_comp_pte) +{ + enum xe_cache_level cache_level; + + /* + * Select the appropriate PAT index for buffer object PTEs programmed + * by emit_pte(). We choose not to mess with xe_migrate_prepare_vm() + * yet, for simplicity. + */ + if (is_comp_pte && GRAPHICS_VERx100(xe) >= 2000) + cache_level = XE_CACHE_NONE_COMPRESSION; + else if (caching == ttm_cached) + cache_level = XE_CACHE_WB; + else + cache_level = XE_CACHE_NONE; + + return xe_cache_pat_idx(xe, cache_level); +} + +static u64 xe_migrate_vm_addr(u64 slot, u32 level) +{ + XE_WARN_ON(slot >= NUM_PT_SLOTS); + + /* First slot is reserved for mapping of PT bo and bb, start from 1 */ + return (slot + 1ULL) << xe_pt_shift(level + 1); +} + +static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte) +{ + /* + * Remove the DPA to get a correct offset into identity table for the + * migrate offset + */ + u64 identity_offset = IDENTITY_OFFSET; + + if (GRAPHICS_VER(xe) >= 20 && is_comp_pte) + identity_offset += DIV_ROUND_UP_ULL(xe_vram_region_actual_physical_size + (xe->mem.vram), SZ_1G); + + addr -= xe_vram_region_dpa_base(xe->mem.vram); + return addr + (identity_offset << xe_pt_shift(2)); +} + +static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo, + u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs) +{ + struct xe_vram_region *vram = xe->mem.vram; + resource_size_t dpa_base = xe_vram_region_dpa_base(vram); + u64 pos, ofs, flags; + u64 entry; + /* XXX: Unclear if this should be usable_size? */ + u64 vram_limit = xe_vram_region_actual_physical_size(vram) + dpa_base; + u32 level = 2; + + ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8; + flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, + true, 0); + + xe_assert(xe, IS_ALIGNED(xe_vram_region_usable_size(vram), SZ_2M)); + + /* + * Use 1GB pages when possible, last chunk always use 2M + * pages as mixing reserved memory (stolen, WOCPM) with a single + * mapping is not allowed on certain platforms. + */ + for (pos = dpa_base; pos < vram_limit; + pos += SZ_1G, ofs += 8) { + if (pos + SZ_1G >= vram_limit) { + entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs); + xe_map_wr(xe, &bo->vmap, ofs, u64, entry); + + flags = vm->pt_ops->pte_encode_addr(xe, 0, + pat_index, + level - 1, + true, 0); + + for (ofs = pt_2m_ofs; pos < vram_limit; + pos += SZ_2M, ofs += 8) + xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); + break; /* Ensure pos == vram_limit assert correct */ + } + + xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); + } + + xe_assert(xe, pos == vram_limit); +} + +static int xe_migrate_pt_bo_alloc(struct xe_tile *tile, struct xe_migrate *m, + struct xe_vm *vm, struct drm_exec *exec) +{ + struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; + u32 num_entries = NUM_PT_SLOTS; + + /* Can't bump NUM_PT_SLOTS too high */ + BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); + /* Must be a multiple of 64K to support all platforms */ + BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); + /* And one slot reserved for the 4KiB page table updates */ + BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); + + /* Need to be sure everything fits in the first PT, or create more */ + xe_tile_assert(tile, m->batch_base_ofs + xe_bo_size(batch) < SZ_2M); + + bo = xe_bo_create_pin_map(vm->xe, tile, vm, + num_entries * XE_PAGE_SIZE, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_PAGETABLE, exec); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + m->pt_bo = bo; + return 0; +} + +static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, + struct xe_vm *vm, u32 *ofs) +{ + struct xe_device *xe = tile_to_xe(tile); + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); + u8 id = tile->id; + u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; +#define VRAM_IDENTITY_MAP_COUNT 2 + u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT; +#undef VRAM_IDENTITY_MAP_COUNT + u32 map_ofs, level, i; + struct xe_bo *bo = m->pt_bo, *batch = tile->mem.kernel_bb_pool->bo; + u64 entry, pt29_ofs; + + /* PT30 & PT31 reserved for 2M identity map */ + pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE; + entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs); + xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); + + map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; + + /* Map the entire BO in our level 0 pt */ + for (i = 0, level = 0; i < num_entries; level++) { + entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, + pat_index, 0); + + xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); + + if (vm->flags & XE_VM_FLAG_64K) + i += 16; + else + i += 1; + } + + if (!IS_DGFX(xe)) { + /* Write out batch too */ + m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; + for (i = 0; i < xe_bo_size(batch); + i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : + XE_PAGE_SIZE) { + entry = vm->pt_ops->pte_encode_bo(batch, i, + pat_index, 0); + + xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, + entry); + level++; + } + if (xe->info.has_usm) { + xe_tile_assert(tile, xe_bo_size(batch) == SZ_1M); + + batch = tile->primary_gt->usm.bb_pool->bo; + m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; + xe_tile_assert(tile, xe_bo_size(batch) == SZ_512K); + + for (i = 0; i < xe_bo_size(batch); + i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : + XE_PAGE_SIZE) { + entry = vm->pt_ops->pte_encode_bo(batch, i, + pat_index, 0); + + xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, + entry); + level++; + } + } + } else { + u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); + + m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); + + if (xe->info.has_usm) { + batch = tile->primary_gt->usm.bb_pool->bo; + batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); + m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); + } + } + + for (level = 1; level < num_level; level++) { + u32 flags = 0; + + if (vm->flags & XE_VM_FLAG_64K && level == 1) + flags = XE_PDE_64K; + + entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * + XE_PAGE_SIZE); + xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, + entry | flags); + } + + /* Write PDE's that point to our BO. */ + for (i = 0; i < map_ofs / XE_PAGE_SIZE; i++) { + entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE); + + xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + + (i + 1) * 8, u64, entry); + } + + /* Reserve 2M PDEs */ + level = 1; + m->large_page_copy_ofs = NUM_PT_SLOTS << xe_pt_shift(level); + m->large_page_copy_pdes = map_ofs + XE_PAGE_SIZE * level + + NUM_PT_SLOTS * 8; + + /* Set up a 1GiB NULL mapping at 255GiB offset. */ + level = 2; + xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, + vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) + | XE_PTE_NULL); + m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); + + /* Identity map the entire vram at 256GiB offset */ + if (IS_DGFX(xe)) { + u64 pt30_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE; + resource_size_t actual_phy_size = xe_vram_region_actual_physical_size(xe->mem.vram); + + xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET, + pat_index, pt30_ofs); + xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); + + /* + * Identity map the entire vram for compressed pat_index for xe2+ + * if flat ccs is enabled. + */ + if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { + u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION); + u64 vram_offset = IDENTITY_OFFSET + + DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G); + u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; + + xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET - + IDENTITY_OFFSET / 2) * SZ_1G); + xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset, + comp_pat_index, pt31_ofs); + } + } + + if (ofs) + *ofs = map_ofs; +} + +static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs) +{ + /* + * Example layout created above, with root level = 3: + * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's + * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's + * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's + * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map] + * + * This makes the lowest part of the VM point to the pagetables. + * Hence the lowest 2M in the vm should point to itself, with a few writes + * and flushes, other parts of the VM can be used either for copying and + * clearing. + * + * For performance, the kernel reserves PDE's, so about 20 are left + * for async VM updates. + * + * To make it easier to work, each scratch PT is put in slot (1 + PT #) + * everywhere, this allows lockless updates to scratch pages by using + * the different addresses in VM. + */ +#define NUM_VMUSA_UNIT_PER_PAGE 32 +#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) +#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) + drm_suballoc_manager_init(&m->vm_update_sa, + (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * + NUM_VMUSA_UNIT_PER_PAGE, 0); +} + +static bool xe_migrate_needs_ccs_emit(struct xe_device *xe) +{ + return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)); +} + +/** + * xe_migrate_alloc - Allocate a migrate struct for a given &xe_tile + * @tile: &xe_tile + * + * Allocates a &xe_migrate for a given tile. + * + * Return: &xe_migrate on success, or NULL when out of memory. + */ +struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile) +{ + struct xe_migrate *m = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*m), GFP_KERNEL); + + if (m) + m->tile = tile; + return m; +} + +static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, struct xe_vm *vm) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_validation_ctx ctx; + struct drm_exec exec; + u32 map_ofs; + int err = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { + err = xe_vm_drm_exec_lock(vm, &exec); + if (err) + return err; + + drm_exec_retry_on_contention(&exec); + + err = xe_migrate_pt_bo_alloc(tile, m, vm, &exec); + if (err) + return err; + + xe_migrate_prepare_vm(tile, m, vm, &map_ofs); + xe_migrate_suballoc_manager_init(m, map_ofs); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + } + + return err; +} + +/** + * xe_migrate_init() - Initialize a migrate context + * @m: The migration context + * + * Return: 0 if successful, negative error code on failure + */ +int xe_migrate_init(struct xe_migrate *m) +{ + struct xe_tile *tile = m->tile; + struct xe_gt *primary_gt = tile->primary_gt; + struct xe_device *xe = tile_to_xe(tile); + struct xe_vm *vm; + int err; + + /* Special layout, prepared below.. */ + vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | + XE_VM_FLAG_SET_TILE_ID(tile), NULL); + if (IS_ERR(vm)) + return PTR_ERR(vm); + + err = xe_migrate_lock_prepare_vm(tile, m, vm); + if (err) + goto err_out; + + if (xe->info.has_usm) { + struct xe_hw_engine *hwe0 = primary_gt->usm.paging_hwe0; + u32 logical_mask = primary_gt->usm.paging_logical_mask; + + if (!hwe0 || !logical_mask) { + err = -EINVAL; + goto err_out; + } + + /* + * XXX: Currently only reserving 1 (likely slow) BCS instance on + * PVC, may want to revisit if performance is needed. + */ + m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, + EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_PERMANENT | + EXEC_QUEUE_FLAG_HIGH_PRIORITY | + EXEC_QUEUE_FLAG_MIGRATE | + EXEC_QUEUE_FLAG_LOW_LATENCY, 0); + } else { + m->q = xe_exec_queue_create_class(xe, primary_gt, vm, + XE_ENGINE_CLASS_COPY, + EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_PERMANENT | + EXEC_QUEUE_FLAG_MIGRATE, 0); + } + if (IS_ERR(m->q)) { + err = PTR_ERR(m->q); + goto err_out; + } + + mutex_init(&m->job_mutex); + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&m->job_mutex); + fs_reclaim_release(GFP_KERNEL); + + err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m); + if (err) + return err; + + if (IS_DGFX(xe)) { + if (xe_migrate_needs_ccs_emit(xe)) + /* min chunk size corresponds to 4K of CCS Metadata */ + m->min_chunk_size = SZ_4K * SZ_64K / + xe_device_ccs_bytes(xe, SZ_64K); + else + /* Somewhat arbitrary to avoid a huge amount of blits */ + m->min_chunk_size = SZ_64K; + m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); + drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", + (unsigned long long)m->min_chunk_size); + } + + return err; + +err_out: + xe_vm_close_and_put(vm); + return err; + +} + +static u64 max_mem_transfer_per_pass(struct xe_device *xe) +{ + if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) + return MAX_CCS_LIMITED_TRANSFER; + + return MAX_PREEMPTDISABLE_TRANSFER; +} + +static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) +{ + struct xe_device *xe = tile_to_xe(m->tile); + u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); + + if (mem_type_is_vram(cur->mem_type)) { + /* + * VRAM we want to blit in chunks with sizes aligned to + * min_chunk_size in order for the offset to CCS metadata to be + * page-aligned. If it's the last chunk it may be smaller. + * + * Another constraint is that we need to limit the blit to + * the VRAM block size, unless size is smaller than + * min_chunk_size. + */ + u64 chunk = max_t(u64, cur->size, m->min_chunk_size); + + size = min_t(u64, size, chunk); + if (size > m->min_chunk_size) + size = round_down(size, m->min_chunk_size); + } + + return size; +} + +static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) +{ + /* If the chunk is not fragmented, allow identity map. */ + return cur->size >= size; +} + +#define PTE_UPDATE_FLAG_IS_VRAM BIT(0) +#define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1) + +static u32 pte_update_size(struct xe_migrate *m, + u32 flags, + struct ttm_resource *res, + struct xe_res_cursor *cur, + u64 *L0, u64 *L0_ofs, u32 *L0_pt, + u32 cmd_size, u32 pt_ofs, u32 avail_pts) +{ + u32 cmds = 0; + bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags; + bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags; + + *L0_pt = pt_ofs; + if (is_vram && xe_migrate_allow_identity(*L0, cur)) { + /* Offset into identity map. */ + *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), + cur->start + vram_region_gpu_offset(res), + is_comp_pte); + cmds += cmd_size; + } else { + /* Clip L0 to available size */ + u64 size = min(*L0, (u64)avail_pts * SZ_2M); + u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; + + *L0 = size; + *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); + + /* MI_STORE_DATA_IMM */ + cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); + + /* PDE qwords */ + cmds += num_4k_pages * 2; + + /* Each chunk has a single blit command */ + cmds += cmd_size; + } + + return cmds; +} + +static void emit_pte(struct xe_migrate *m, + struct xe_bb *bb, u32 at_pt, + bool is_vram, bool is_comp_pte, + struct xe_res_cursor *cur, + u32 size, struct ttm_resource *res) +{ + struct xe_device *xe = tile_to_xe(m->tile); + struct xe_vm *vm = m->q->vm; + struct xe_bo *bo = ttm_to_xe_bo(res->bo); + enum ttm_caching caching = ttm_cached; + u16 pat_index; + u32 ptes; + u64 ofs = (u64)at_pt * XE_PAGE_SIZE; + u64 cur_ofs; + + if (!is_vram && bo->ttm.ttm) + caching = bo->ttm.ttm->caching; + + pat_index = xe_migrate_pat_index(xe, caching, is_comp_pte); + + ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); + + while (ptes) { + u32 chunk = min(MAX_PTE_PER_SDI, ptes); + + bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); + bb->cs[bb->len++] = ofs; + bb->cs[bb->len++] = 0; + + cur_ofs = ofs; + ofs += chunk * 8; + ptes -= chunk; + + while (chunk--) { + u64 addr, flags = 0; + bool devmem = false; + + addr = xe_res_dma(cur) & PAGE_MASK; + if (is_vram) { + if (vm->flags & XE_VM_FLAG_64K) { + u64 va = cur_ofs * XE_PAGE_SIZE / 8; + + xe_assert(xe, (va & (SZ_64K - 1)) == + (addr & (SZ_64K - 1))); + + flags |= XE_PTE_PS64; + } + + addr += vram_region_gpu_offset(res); + devmem = true; + } + + addr = vm->pt_ops->pte_encode_addr(m->tile->xe, + addr, pat_index, + 0, devmem, flags); + bb->cs[bb->len++] = lower_32_bits(addr); + bb->cs[bb->len++] = upper_32_bits(addr); + + xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); + cur_ofs += 8; + } + } +} + +#define EMIT_COPY_CCS_DW 5 +static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, + u64 dst_ofs, bool dst_is_indirect, + u64 src_ofs, bool src_is_indirect, + u32 size) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 *cs = bb->cs + bb->len; + u32 num_ccs_blks; + u32 num_pages; + u32 ccs_copy_size; + u32 mocs; + + if (GRAPHICS_VERx100(xe) >= 2000) { + num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); + xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); + + ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); + mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); + + } else { + num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), + NUM_CCS_BYTES_PER_BLOCK); + xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); + + ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); + mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); + } + + *cs++ = XY_CTRL_SURF_COPY_BLT | + (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | + (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | + ccs_copy_size; + *cs++ = lower_32_bits(src_ofs); + *cs++ = upper_32_bits(src_ofs) | mocs; + *cs++ = lower_32_bits(dst_ofs); + *cs++ = upper_32_bits(dst_ofs) | mocs; + + bb->len = cs - bb->cs; +} + +static u32 blt_fast_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 blt_mem_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 emit_copy_cmd_len(struct xe_device *xe) +{ + return (xe->info.has_mem_copy_instr) ? blt_mem_copy_cmd_len(xe) : + blt_fast_copy_cmd_len(xe); +} + +static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, + u64 dst_ofs, unsigned int size, + unsigned int pitch) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 mocs = 0; + u32 tile_y = 0; + u32 len; + + xe_gt_assert(gt, !(pitch & 3)); + xe_gt_assert(gt, size / pitch <= S16_MAX); + xe_gt_assert(gt, pitch / 4 <= S16_MAX); + xe_gt_assert(gt, pitch <= U16_MAX); + + if (GRAPHICS_VER(xe) >= 20) + mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); + + if (GRAPHICS_VERx100(xe) >= 1250) + tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; + + len = blt_fast_copy_cmd_len(xe); + bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (len - 2); + bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; + bb->cs[bb->len++] = 0; + bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; + bb->cs[bb->len++] = lower_32_bits(dst_ofs); + bb->cs[bb->len++] = upper_32_bits(dst_ofs); + bb->cs[bb->len++] = 0; + bb->cs[bb->len++] = pitch | mocs; + bb->cs[bb->len++] = lower_32_bits(src_ofs); + bb->cs[bb->len++] = upper_32_bits(src_ofs); +} + +#define PAGE_COPY_MODE_PS SZ_256 /* hw uses 256 bytes as the page-size */ +static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, + u64 dst_ofs, unsigned int size, unsigned int pitch) +{ + u32 mode, copy_type, width; + u32 len; + + xe_gt_assert(gt, IS_ALIGNED(size, pitch)); + xe_gt_assert(gt, pitch <= U16_MAX); + xe_gt_assert(gt, pitch); + xe_gt_assert(gt, size); + + if (IS_ALIGNED(size, PAGE_COPY_MODE_PS) && + IS_ALIGNED(lower_32_bits(src_ofs), PAGE_COPY_MODE_PS) && + IS_ALIGNED(lower_32_bits(dst_ofs), PAGE_COPY_MODE_PS)) { + mode = MEM_COPY_PAGE_COPY_MODE; + copy_type = 0; /* linear copy */ + width = size / PAGE_COPY_MODE_PS; + } else if (pitch > 1) { + xe_gt_assert(gt, size / pitch <= U16_MAX); + mode = 0; /* BYTE_COPY */ + copy_type = MEM_COPY_MATRIX_COPY; + width = pitch; + } else { + mode = 0; /* BYTE_COPY */ + copy_type = 0; /* linear copy */ + width = size; + } + + xe_gt_assert(gt, width <= U16_MAX); + + len = blt_mem_copy_cmd_len(gt_to_xe(gt)); + + bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type | (len - 2); + bb->cs[bb->len++] = width - 1; + bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */ + bb->cs[bb->len++] = pitch - 1; + bb->cs[bb->len++] = pitch - 1; + bb->cs[bb->len++] = lower_32_bits(src_ofs); + bb->cs[bb->len++] = upper_32_bits(src_ofs); + bb->cs[bb->len++] = lower_32_bits(dst_ofs); + bb->cs[bb->len++] = upper_32_bits(dst_ofs); + bb->cs[bb->len++] = FIELD_PREP(MEM_COPY_SRC_MOCS_INDEX_MASK, gt->mocs.uc_index) | + FIELD_PREP(MEM_COPY_DST_MOCS_INDEX_MASK, gt->mocs.uc_index); +} + +static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, + u64 src_ofs, u64 dst_ofs, unsigned int size, + unsigned int pitch) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (xe->info.has_mem_copy_instr) + emit_mem_copy(gt, bb, src_ofs, dst_ofs, size, pitch); + else + emit_xy_fast_copy(gt, bb, src_ofs, dst_ofs, size, pitch); +} + +static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) +{ + return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; +} + +static u32 xe_migrate_ccs_copy(struct xe_migrate *m, + struct xe_bb *bb, + u64 src_ofs, bool src_is_indirect, + u64 dst_ofs, bool dst_is_indirect, u32 dst_size, + u64 ccs_ofs, bool copy_ccs) +{ + struct xe_gt *gt = m->tile->primary_gt; + u32 flush_flags = 0; + + if (!copy_ccs && dst_is_indirect) { + /* + * If the src is already in vram, then it should already + * have been cleared by us, or has been populated by the + * user. Make sure we copy the CCS aux state as-is. + * + * Otherwise if the bo doesn't have any CCS metadata attached, + * we still need to clear it for security reasons. + */ + u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; + + emit_copy_ccs(gt, bb, + dst_ofs, true, + ccs_src_ofs, src_is_indirect, dst_size); + + flush_flags = MI_FLUSH_DW_CCS; + } else if (copy_ccs) { + if (!src_is_indirect) + src_ofs = ccs_ofs; + else if (!dst_is_indirect) + dst_ofs = ccs_ofs; + + xe_gt_assert(gt, src_is_indirect || dst_is_indirect); + + emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, + src_is_indirect, dst_size); + if (dst_is_indirect) + flush_flags = MI_FLUSH_DW_CCS; + } + + return flush_flags; +} + +static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + struct xe_bo *dst_bo, + struct ttm_resource *src, + struct ttm_resource *dst, + bool copy_only_ccs, + bool is_vram_resolve) +{ + struct xe_gt *gt = m->tile->primary_gt; + struct xe_device *xe = gt_to_xe(gt); + struct dma_fence *fence = NULL; + u64 size = xe_bo_size(src_bo); + struct xe_res_cursor src_it, dst_it, ccs_it; + u64 src_L0_ofs, dst_L0_ofs; + u32 src_L0_pt, dst_L0_pt; + u64 src_L0, dst_L0; + int pass = 0; + int err; + bool src_is_pltt = src->mem_type == XE_PL_TT; + bool dst_is_pltt = dst->mem_type == XE_PL_TT; + bool src_is_vram = mem_type_is_vram(src->mem_type); + bool dst_is_vram = mem_type_is_vram(dst->mem_type); + bool type_device = src_bo->ttm.type == ttm_bo_type_device; + bool needs_ccs_emit = type_device && xe_migrate_needs_ccs_emit(xe); + bool copy_ccs = xe_device_has_flat_ccs(xe) && + xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); + bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); + + /* + * For decompression operation, always use the compression PAT index. + * Otherwise, only use the compression PAT index for device memory + * when copying from VRAM to system memory. + */ + bool use_comp_pat = is_vram_resolve || (type_device && + xe_device_has_flat_ccs(xe) && + GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram); + + /* Copying CCS between two different BOs is not supported yet. */ + if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) + return ERR_PTR(-EINVAL); + + if (src_bo != dst_bo && XE_WARN_ON(xe_bo_size(src_bo) != xe_bo_size(dst_bo))) + return ERR_PTR(-EINVAL); + + if (!src_is_vram) + xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); + else + xe_res_first(src, 0, size, &src_it); + if (!dst_is_vram) + xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); + else + xe_res_first(dst, 0, size, &dst_it); + + if (copy_system_ccs) + xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), + PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), + &ccs_it); + + while (size) { + u32 batch_size = 1; /* MI_BATCH_BUFFER_END */ + struct xe_sched_job *job; + struct xe_bb *bb; + u32 flush_flags = 0; + u32 update_idx; + u64 ccs_ofs, ccs_size; + u32 ccs_pt; + u32 pte_flags; + + bool usm = xe->info.has_usm; + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + src_L0 = xe_migrate_res_sizes(m, &src_it); + dst_L0 = xe_migrate_res_sizes(m, &dst_it); + + drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", + pass++, src_L0, dst_L0); + + src_L0 = min(src_L0, dst_L0); + + pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; + pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0; + batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + avail_pts); + if (copy_only_ccs) { + dst_L0_ofs = src_L0_ofs; + } else { + pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; + batch_size += pte_update_size(m, pte_flags, dst, + &dst_it, &src_L0, + &dst_L0_ofs, &dst_L0_pt, + 0, avail_pts, avail_pts); + } + + if (copy_system_ccs) { + xe_assert(xe, type_device); + ccs_size = xe_device_ccs_bytes(xe, src_L0); + batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, + &ccs_ofs, &ccs_pt, 0, + 2 * avail_pts, + avail_pts); + xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); + } + + /* Add copy commands size here */ + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + + ((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0)); + + bb = xe_bb_new(gt, batch_size, usm); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + goto err_sync; + } + + if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) + xe_res_next(&src_it, src_L0); + else + emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs || use_comp_pat, + &src_it, src_L0, src); + + if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) + xe_res_next(&dst_it, src_L0); + else if (!copy_only_ccs) + emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, + &dst_it, src_L0, dst); + + if (copy_system_ccs) + emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + if (!copy_only_ccs) + emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); + + if (needs_ccs_emit) + flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, + IS_DGFX(xe) ? src_is_vram : src_is_pltt, + dst_L0_ofs, + IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, + src_L0, ccs_ofs, copy_ccs); + + job = xe_bb_create_migration_job(m->q, bb, + xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err; + } + + xe_sched_job_add_migrate_flush(job, flush_flags | MI_INVALIDATE_TLB); + if (!fence) { + err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP); + if (!err && src_bo->ttm.base.resv != dst_bo->ttm.base.resv) + err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP); + if (err) + goto err_job; + } + + mutex_lock(&m->job_mutex); + xe_sched_job_arm(job); + dma_fence_put(fence); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + size -= src_L0; + continue; + +err_job: + xe_sched_job_put(job); +err: + xe_bb_free(bb, NULL); + +err_sync: + /* Sync partial copy if any. FIXME: under job_mutex? */ + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + + return ERR_PTR(err); + } + + return fence; +} + +/** + * xe_migrate_copy() - Copy content of TTM resources. + * @m: The migration context. + * @src_bo: The buffer object @src is currently bound to. + * @dst_bo: If copying between resources created for the same bo, set this to + * the same value as @src_bo. If copying between buffer objects, set it to + * the buffer object @dst is currently bound to. + * @src: The source TTM resource. + * @dst: The dst TTM resource. + * @copy_only_ccs: If true copy only CCS metadata + * + * Copies the contents of @src to @dst: On flat CCS devices, + * the CCS metadata is copied as well if needed, or if not present, + * the CCS metadata of @dst is cleared for security reasons. + * + * Return: Pointer to a dma_fence representing the last copy batch, or + * an error pointer on failure. If there is a failure, any copy operation + * started by the function call has been synced. + */ +struct dma_fence *xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + struct xe_bo *dst_bo, + struct ttm_resource *src, + struct ttm_resource *dst, + bool copy_only_ccs) +{ + return __xe_migrate_copy(m, src_bo, dst_bo, src, dst, copy_only_ccs, false); +} + +/** + * xe_migrate_resolve() - Resolve and decompress a buffer object if required. + * @m: The migrate context + * @bo: The buffer object to resolve + * @res: The reservation object + * + * Wrapper around __xe_migrate_copy() with is_vram_resolve set to true + * to trigger decompression if needed. + * + * Return: A dma_fence that signals on completion, or an ERR_PTR on failure. + */ +struct dma_fence *xe_migrate_resolve(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *res) +{ + return __xe_migrate_copy(m, bo, bo, res, res, false, true); +} + +/** + * xe_migrate_lrc() - Get the LRC from migrate context. + * @migrate: Migrate context. + * + * Return: Pointer to LRC on success, error on failure + */ +struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate) +{ + return migrate->q->lrc[0]; +} + +static u64 migrate_vm_ppgtt_addr_tlb_inval(void) +{ + /* + * The migrate VM is self-referential so it can modify its own PTEs (see + * pte_update_size() or emit_pte() functions). We reserve NUM_KERNEL_PDE + * entries for kernel operations (copies, clears, CCS migrate), and + * suballocate the rest to user operations (binds/unbinds). With + * NUM_KERNEL_PDE = 15, NUM_KERNEL_PDE - 1 is already used for PTE updates, + * so assign NUM_KERNEL_PDE - 2 for TLB invalidation. + */ + return (NUM_KERNEL_PDE - 2) * XE_PAGE_SIZE; +} + +static int emit_flush_invalidate(u32 *dw, int i, u32 flags) +{ + u64 addr = migrate_vm_ppgtt_addr_tlb_inval(); + + dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | + MI_FLUSH_IMM_DW | flags; + dw[i++] = lower_32_bits(addr); + dw[i++] = upper_32_bits(addr); + dw[i++] = MI_NOOP; + dw[i++] = MI_NOOP; + + return i; +} + +/** + * xe_migrate_ccs_rw_copy() - Copy content of TTM resources. + * @tile: Tile whose migration context to be used. + * @q : Execution to be used along with migration context. + * @src_bo: The buffer object @src is currently bound to. + * @new_mem: The (not yet committed) destination resource @src_bo is being + * moved into; src_bo->ttm.resource is still the old resource. + * @read_write : Creates BB commands for CCS read/write. + * + * Creates batch buffer instructions to copy CCS metadata from CCS pool to + * memory and vice versa. + * + * This function should only be called for IGPU. + * + * Return: 0 if successful, negative error code on failure. + */ +int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, + struct xe_bo *src_bo, + struct ttm_resource *new_mem, + enum xe_sriov_vf_ccs_rw_ctxs read_write) + +{ + bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX; + bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX; + struct ttm_resource *src = new_mem; + struct xe_migrate *m = tile->migrate; + struct xe_gt *gt = tile->primary_gt; + u32 batch_size, batch_size_allocated; + struct xe_device *xe = gt_to_xe(gt); + struct xe_res_cursor src_it, ccs_it; + struct xe_mem_pool *bb_pool; + struct xe_sriov_vf_ccs_ctx *ctx; + u64 size = xe_bo_size(src_bo); + struct xe_mem_pool_node *bb; + u64 src_L0, src_L0_ofs; + struct xe_bb xe_bb_tmp; + u32 src_L0_pt; + int err; + + ctx = &xe->sriov.vf.ccs.contexts[read_write]; + + xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); + + xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), + PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), + &ccs_it); + + /* Calculate Batch buffer size */ + batch_size = 0; + while (size) { + batch_size += 10; /* Flush + ggtt addr + 2 NOP */ + u64 ccs_ofs, ccs_size; + u32 ccs_pt; + + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + src_L0 = min_t(u64, max_mem_transfer_per_pass(xe), size); + + batch_size += pte_update_size(m, false, src, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + avail_pts); + + ccs_size = xe_device_ccs_bytes(xe, src_L0); + batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs, + &ccs_pt, 0, avail_pts, avail_pts); + xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); + + /* Add copy commands size here */ + batch_size += EMIT_COPY_CCS_DW; + + size -= src_L0; + } + + bb = xe_mem_pool_alloc_node(); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + bb_pool = ctx->mem.ccs_bb_pool; + scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) { + xe_mem_pool_swap_shadow_locked(bb_pool); + + err = xe_mem_pool_insert_node(bb_pool, bb, batch_size * sizeof(u32)); + if (err) { + xe_gt_err(gt, "BB allocation failed.\n"); + kfree(bb); + return err; + } + + batch_size_allocated = batch_size; + size = xe_bo_size(src_bo); + batch_size = 0; + + xe_bb_tmp = (struct xe_bb){ .cs = xe_mem_pool_node_cpu_addr(bb), .len = 0 }; + /* + * Emit PTE and copy commands here. + * The CCS copy command can only support limited size. If the size to be + * copied is more than the limit, divide copy into chunks. So, calculate + * sizes here again before copy command is emitted. + */ + + while (size) { + batch_size += 10; /* Flush + ggtt addr + 2 NOP */ + u32 flush_flags = 0; + u64 ccs_ofs, ccs_size; + u32 ccs_pt; + + u32 avail_pts = max_mem_transfer_per_pass(xe) / + LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + src_L0 = xe_migrate_res_sizes(m, &src_it); + + batch_size += pte_update_size(m, false, src, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + avail_pts); + + ccs_size = xe_device_ccs_bytes(xe, src_L0); + batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs, + &ccs_pt, 0, avail_pts, avail_pts); + xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); + batch_size += EMIT_COPY_CCS_DW; + + emit_pte(m, &xe_bb_tmp, src_L0_pt, false, true, &src_it, src_L0, src); + + emit_pte(m, &xe_bb_tmp, ccs_pt, false, false, &ccs_it, ccs_size, src); + + xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len, + flush_flags); + flush_flags = xe_migrate_ccs_copy(m, &xe_bb_tmp, src_L0_ofs, src_is_pltt, + src_L0_ofs, dst_is_pltt, + src_L0, ccs_ofs, true); + xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len, + flush_flags); + + size -= src_L0; + } + + xe_assert(xe, (batch_size_allocated == xe_bb_tmp.len)); + xe_assert(xe, bb->sa_node.size == xe_bb_tmp.len * sizeof(u32)); + src_bo->bb_ccs[read_write] = bb; + + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); + xe_mem_pool_sync_shadow_locked(bb); + } + + return 0; +} + +/** + * xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer + * content. + * @src_bo: The buffer object @src is currently bound to. + * @read_write : Creates BB commands for CCS read/write. + * @bound: Device is bound + * + * Directly clearing the BB lacks atomicity and can lead to undefined + * behavior if the vCPU is halted mid-operation during the clearing + * process. To avoid this issue, we use a shadow buffer object approach. + * + * First swap the SA BO address with the shadow BO, perform the clearing + * operation on the BB, update the shadow BO in the ring buffer, then + * sync the shadow and the actual buffer to maintain consistency. + * + * Returns: None. + */ +void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write, + bool bound) +{ + struct xe_mem_pool_node *bb = src_bo->bb_ccs[read_write]; + struct xe_device *xe = xe_bo_device(src_bo); + struct xe_mem_pool *bb_pool; + struct xe_sriov_vf_ccs_ctx *ctx; + u32 *cs; + + xe_assert(xe, IS_SRIOV_VF(xe)); + + ctx = &xe->sriov.vf.ccs.contexts[read_write]; + bb_pool = ctx->mem.ccs_bb_pool; + + scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) { + if (bound) { + xe_mem_pool_swap_shadow_locked(bb_pool); + + cs = xe_mem_pool_node_cpu_addr(bb); + memset(cs, MI_NOOP, bb->sa_node.size); + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); + + xe_mem_pool_sync_shadow_locked(bb); + } + xe_mem_pool_free_node(bb); + src_bo->bb_ccs[read_write] = NULL; + } +} + +/** + * xe_migrate_exec_queue() - Get the execution queue from migrate context. + * @migrate: Migrate context. + * + * Return: Pointer to execution queue on success, error on failure + */ +struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate) +{ + return migrate->q; +} + +/** + * xe_migrate_vram_copy_chunk() - Copy a chunk of a VRAM buffer object. + * @vram_bo: The VRAM buffer object. + * @vram_offset: The VRAM offset. + * @sysmem_bo: The sysmem buffer object. + * @sysmem_offset: The sysmem offset. + * @size: The size of VRAM chunk to copy. + * @dir: The direction of the copy operation. + * + * Copies a portion of a buffer object between VRAM and system memory. + * On Xe2 platforms that support flat CCS, VRAM data is decompressed when + * copying to system memory. + * + * Return: Pointer to a dma_fence representing the last copy batch, or + * an error pointer on failure. If there is a failure, any copy operation + * started by the function call has been synced. + */ +struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_offset, + struct xe_bo *sysmem_bo, u64 sysmem_offset, + u64 size, enum xe_migrate_copy_dir dir) +{ + struct xe_device *xe = xe_bo_device(vram_bo); + struct xe_tile *tile = vram_bo->tile; + struct xe_gt *gt = tile->primary_gt; + struct xe_migrate *m = tile->migrate; + struct dma_fence *fence = NULL; + struct ttm_resource *vram = vram_bo->ttm.resource; + struct ttm_resource *sysmem = sysmem_bo->ttm.resource; + struct xe_res_cursor vram_it, sysmem_it; + u64 vram_L0_ofs, sysmem_L0_ofs; + u32 vram_L0_pt, sysmem_L0_pt; + u64 vram_L0, sysmem_L0; + bool to_sysmem = (dir == XE_MIGRATE_COPY_TO_SRAM); + bool use_comp_pat = to_sysmem && + GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe); + int pass = 0; + int err; + + xe_assert(xe, IS_ALIGNED(vram_offset | sysmem_offset | size, PAGE_SIZE)); + xe_assert(xe, xe_bo_is_vram(vram_bo)); + xe_assert(xe, !xe_bo_is_vram(sysmem_bo)); + xe_assert(xe, !range_overflows(vram_offset, size, (u64)vram_bo->ttm.base.size)); + xe_assert(xe, !range_overflows(sysmem_offset, size, (u64)sysmem_bo->ttm.base.size)); + + xe_res_first(vram, vram_offset, size, &vram_it); + xe_res_first_sg(xe_bo_sg(sysmem_bo), sysmem_offset, size, &sysmem_it); + + while (size) { + u32 pte_flags = PTE_UPDATE_FLAG_IS_VRAM; + u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ + struct xe_sched_job *job; + struct xe_bb *bb; + u32 update_idx; + bool usm = xe->info.has_usm; + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + sysmem_L0 = xe_migrate_res_sizes(m, &sysmem_it); + vram_L0 = min(xe_migrate_res_sizes(m, &vram_it), sysmem_L0); + + xe_dbg(xe, "Pass %u, size: %llu\n", pass++, vram_L0); + + pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0; + batch_size += pte_update_size(m, pte_flags, vram, &vram_it, &vram_L0, + &vram_L0_ofs, &vram_L0_pt, 0, 0, avail_pts); + + batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs, + &sysmem_L0_pt, 0, avail_pts, avail_pts); + batch_size += emit_copy_cmd_len(xe); + + bb = xe_bb_new(gt, batch_size, usm); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + return ERR_PTR(err); + } + + if (xe_migrate_allow_identity(vram_L0, &vram_it)) + xe_res_next(&vram_it, vram_L0); + else + emit_pte(m, bb, vram_L0_pt, true, use_comp_pat, &vram_it, vram_L0, vram); + + emit_pte(m, bb, sysmem_L0_pt, false, false, &sysmem_it, vram_L0, sysmem); + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + if (to_sysmem) + emit_copy(gt, bb, vram_L0_ofs, sysmem_L0_ofs, vram_L0, XE_PAGE_SIZE); + else + emit_copy(gt, bb, sysmem_L0_ofs, vram_L0_ofs, vram_L0, XE_PAGE_SIZE); + + job = xe_bb_create_migration_job(m->q, bb, xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + xe_bb_free(bb, NULL); + err = PTR_ERR(job); + return ERR_PTR(err); + } + + xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); + + xe_assert(xe, dma_resv_test_signaled(vram_bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP)); + xe_assert(xe, dma_resv_test_signaled(sysmem_bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP)); + + scoped_guard(mutex, &m->job_mutex) { + xe_sched_job_arm(job); + dma_fence_put(fence); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + } + + xe_bb_free(bb, fence); + size -= vram_L0; + } + + return fence; +} + +static u32 blt_mem_set_cmd_len(struct xe_device *xe) +{ + return 7; +} + +static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, + u32 size, u32 pitch) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 *cs = bb->cs + bb->len; + u32 len = blt_mem_set_cmd_len(xe); + + *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); + *cs++ = pitch - 1; + *cs++ = (size / pitch) - 1; + *cs++ = pitch - 1; + *cs++ = lower_32_bits(src_ofs); + *cs++ = upper_32_bits(src_ofs); + if (GRAPHICS_VERx100(xe) >= 2000) + *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); + else + *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); + + xe_gt_assert(gt, cs - bb->cs == len + bb->len); + + bb->len += len; +} + +static u32 blt_fast_color_cmd_len(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1250) + return 16; + else + return 11; +} + +static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, + u64 src_ofs, u32 size, u32 pitch, bool is_vram) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 *cs = bb->cs + bb->len; + u32 len = blt_fast_color_cmd_len(xe); + + + *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | + (len - 2); + if (GRAPHICS_VERx100(xe) >= 2000) + *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | + (pitch - 1); + else + *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | + (pitch - 1); + *cs++ = 0; + *cs++ = (size / pitch) << 16 | pitch / 4; + *cs++ = lower_32_bits(src_ofs); + *cs++ = upper_32_bits(src_ofs); + *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + + if (len > 11) { + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + } + + xe_gt_assert(gt, cs - bb->cs == len + bb->len); + + bb->len += len; +} + +static u32 emit_clear_cmd_len(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (gt->info.has_xe2_blt_instructions) + return blt_mem_set_cmd_len(xe); + else + return blt_fast_color_cmd_len(xe); +} + +static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, + u32 size, u32 pitch, bool is_vram) +{ + if (gt->info.has_xe2_blt_instructions) + emit_clear_link_copy(gt, bb, src_ofs, size, pitch); + else + emit_clear_main_copy(gt, bb, src_ofs, size, pitch, + is_vram); +} + +/** + * xe_migrate_clear() - Copy content of TTM resources. + * @m: The migration context. + * @bo: The buffer object @dst is currently bound to. + * @dst: The dst TTM resource to be cleared. + * @clear_flags: flags to specify which data to clear: CCS, BO, or both. + * + * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set. + * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA. + * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata. + * TODO: Eliminate the @bo argument. + * + * Return: Pointer to a dma_fence representing the last clear batch, or + * an error pointer on failure. If there is a failure, any clear operation + * started by the function call has been synced. + */ +struct dma_fence *xe_migrate_clear(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *dst, + u32 clear_flags) +{ + bool clear_vram = mem_type_is_vram(dst->mem_type); + bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags; + bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags; + struct xe_gt *gt = m->tile->primary_gt; + struct xe_device *xe = gt_to_xe(gt); + bool clear_only_system_ccs = false; + struct dma_fence *fence = NULL; + u64 size = xe_bo_size(bo); + struct xe_res_cursor src_it; + struct ttm_resource *src = dst; + int err; + + if (WARN_ON(!clear_bo_data && !clear_ccs)) + return NULL; + + if (!clear_bo_data && clear_ccs && !IS_DGFX(xe)) + clear_only_system_ccs = true; + + if (!clear_vram) + xe_res_first_sg(xe_bo_sg(bo), 0, xe_bo_size(bo), &src_it); + else + xe_res_first(src, 0, xe_bo_size(bo), &src_it); + + while (size) { + u64 clear_L0_ofs; + u32 clear_L0_pt; + u32 flush_flags = 0; + u64 clear_L0; + struct xe_sched_job *job; + struct xe_bb *bb; + u32 batch_size, update_idx; + u32 pte_flags; + + bool usm = xe->info.has_usm; + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + clear_L0 = xe_migrate_res_sizes(m, &src_it); + + /* Calculate final sizes and batch size.. */ + pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; + batch_size = 1 + + pte_update_size(m, pte_flags, src, &src_it, + &clear_L0, &clear_L0_ofs, &clear_L0_pt, + clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0, + avail_pts); + + if (xe_migrate_needs_ccs_emit(xe)) + batch_size += EMIT_COPY_CCS_DW; + + /* Clear commands */ + + if (WARN_ON_ONCE(!clear_L0)) + break; + + bb = xe_bb_new(gt, batch_size, usm); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + goto err_sync; + } + + size -= clear_L0; + /* Preemption is enabled again by the ring ops. */ + if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) { + xe_res_next(&src_it, clear_L0); + } else { + emit_pte(m, bb, clear_L0_pt, clear_vram, + clear_only_system_ccs, &src_it, clear_L0, dst); + flush_flags |= MI_INVALIDATE_TLB; + } + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + if (clear_bo_data) + emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); + + if (xe_migrate_needs_ccs_emit(xe)) { + emit_copy_ccs(gt, bb, clear_L0_ofs, true, + m->cleared_mem_ofs, false, clear_L0); + flush_flags |= MI_FLUSH_DW_CCS; + } + + job = xe_bb_create_migration_job(m->q, bb, + xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err; + } + + xe_sched_job_add_migrate_flush(job, flush_flags); + if (!fence) { + /* + * There can't be anything userspace related at this + * point, so we just need to respect any potential move + * fences, which are always tracked as + * DMA_RESV_USAGE_KERNEL. + */ + err = xe_sched_job_add_deps(job, bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL); + if (err) + goto err_job; + } + + mutex_lock(&m->job_mutex); + xe_sched_job_arm(job); + dma_fence_put(fence); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + continue; + +err_job: + xe_sched_job_put(job); +err: + xe_bb_free(bb, NULL); +err_sync: + /* Sync partial copies if any. FIXME: job_mutex? */ + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + + return ERR_PTR(err); + } + + if (clear_ccs) + bo->ccs_cleared = true; + + return fence; +} + +static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, + const struct xe_vm_pgtable_update_op *pt_op, + const struct xe_vm_pgtable_update *update, + struct xe_migrate_pt_update *pt_update) +{ + const struct xe_migrate_pt_update_ops *ops = pt_update->ops; + u32 chunk; + u32 ofs = update->ofs, size = update->qwords; + + /* + * If we have 512 entries (max), we would populate it ourselves, + * and update the PDE above it to the new pointer. + * The only time this can only happen if we have to update the top + * PDE. This requires a BO that is almost vm->size big. + * + * This shouldn't be possible in practice.. might change when 16K + * pages are used. Hence the assert. + */ + xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); + if (!ppgtt_ofs) + ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), + xe_bo_addr(update->pt_bo, 0, + XE_PAGE_SIZE), false); + + do { + u64 addr = ppgtt_ofs + ofs * 8; + + chunk = min(size, MAX_PTE_PER_SDI); + + /* Ensure populatefn can do memset64 by aligning bb->cs */ + if (!(bb->len & 1)) + bb->cs[bb->len++] = MI_NOOP; + + bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); + bb->cs[bb->len++] = lower_32_bits(addr); + bb->cs[bb->len++] = upper_32_bits(addr); + if (pt_op->bind) + ops->populate(pt_update, tile, NULL, bb->cs + bb->len, + ofs, chunk, update); + else + ops->clear(pt_update, tile, NULL, bb->cs + bb->len, + ofs, chunk, update); + + bb->len += chunk * 2; + ofs += chunk; + size -= chunk; + } while (size); +} + +struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) +{ + return xe_vm_get(m->q->vm); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +struct migrate_test_params { + struct xe_test_priv base; + bool force_gpu; +}; + +#define to_migrate_test_params(_priv) \ + container_of(_priv, struct migrate_test_params, base) +#endif + +static struct dma_fence * +xe_migrate_update_pgtables_cpu(struct xe_migrate *m, + struct xe_migrate_pt_update *pt_update) +{ + XE_TEST_DECLARE(struct migrate_test_params *test = + to_migrate_test_params + (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) + const struct xe_migrate_pt_update_ops *ops = pt_update->ops; + struct xe_vm *vm = pt_update->vops->vm; + struct xe_vm_pgtable_update_ops *pt_update_ops = + &pt_update->vops->pt_update_ops[pt_update->tile_id]; + int err; + u32 i, j; + + if (XE_TEST_ONLY(test && test->force_gpu)) + return ERR_PTR(-ETIME); + + if (ops->pre_commit) { + pt_update->job = NULL; + err = ops->pre_commit(pt_update); + if (err) + return ERR_PTR(err); + } + + for (i = 0; i < pt_update_ops->num_ops; ++i) { + const struct xe_vm_pgtable_update_op *pt_op = + &pt_update_ops->ops[i]; + + for (j = 0; j < pt_op->num_entries; j++) { + const struct xe_vm_pgtable_update *update = + &pt_op->entries[j]; + + if (pt_op->bind) + ops->populate(pt_update, m->tile, + &update->pt_bo->vmap, NULL, + update->ofs, update->qwords, + update); + else + ops->clear(pt_update, m->tile, + &update->pt_bo->vmap, NULL, + update->ofs, update->qwords, update); + } + } + + trace_xe_vm_cpu_bind(vm); + xe_device_wmb(vm->xe); + + return dma_fence_get_stub(); +} + +static struct dma_fence * +__xe_migrate_update_pgtables(struct xe_migrate *m, + struct xe_migrate_pt_update *pt_update, + struct xe_vm_pgtable_update_ops *pt_update_ops) +{ + const struct xe_migrate_pt_update_ops *ops = pt_update->ops; + struct xe_tile *tile = m->tile; + struct xe_gt *gt = tile->primary_gt; + struct xe_device *xe = tile_to_xe(tile); + struct xe_sched_job *job; + struct dma_fence *fence; + struct drm_suballoc *sa_bo = NULL; + struct xe_bb *bb; + u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0; + u32 num_updates = 0, current_update = 0; + u64 addr; + int err = 0; + bool is_migrate = pt_update_ops->q == m->q; + bool usm = is_migrate && xe->info.has_usm; + + for (i = 0; i < pt_update_ops->num_ops; ++i) { + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; + struct xe_vm_pgtable_update *updates = pt_op->entries; + + num_updates += pt_op->num_entries; + for (j = 0; j < pt_op->num_entries; ++j) { + u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, + MAX_PTE_PER_SDI); + + /* align noop + MI_STORE_DATA_IMM cmd prefix */ + batch_size += 4 * num_cmds + updates[j].qwords * 2; + } + } + + /* fixed + PTE entries */ + if (IS_DGFX(xe)) + batch_size += 2; + else + batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) + + num_updates * 2; + + bb = xe_bb_new(gt, batch_size, usm); + if (IS_ERR(bb)) + return ERR_CAST(bb); + + /* For sysmem PTE's, need to map them in our hole.. */ + if (!IS_DGFX(xe)) { + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); + u32 ptes, ofs; + + ppgtt_ofs = NUM_KERNEL_PDE - 1; + if (!is_migrate) { + u32 num_units = DIV_ROUND_UP(num_updates, + NUM_VMUSA_WRITES_PER_UNIT); + + if (num_units > m->vm_update_sa.size) { + err = -ENOBUFS; + goto err_bb; + } + sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units, + GFP_KERNEL, true, 0); + if (IS_ERR(sa_bo)) { + err = PTR_ERR(sa_bo); + goto err_bb; + } + + ppgtt_ofs = NUM_KERNEL_PDE + + (drm_suballoc_soffset(sa_bo) / + NUM_VMUSA_UNIT_PER_PAGE); + page_ofs = (drm_suballoc_soffset(sa_bo) % + NUM_VMUSA_UNIT_PER_PAGE) * + VM_SA_UPDATE_UNIT_SIZE; + } + + /* Map our PT's to gtt */ + i = 0; + j = 0; + ptes = num_updates; + ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; + while (ptes) { + u32 chunk = min(MAX_PTE_PER_SDI, ptes); + u32 idx = 0; + + bb->cs[bb->len++] = MI_STORE_DATA_IMM | + MI_SDI_NUM_QW(chunk); + bb->cs[bb->len++] = ofs; + bb->cs[bb->len++] = 0; /* upper_32_bits */ + + for (; i < pt_update_ops->num_ops; ++i) { + struct xe_vm_pgtable_update_op *pt_op = + &pt_update_ops->ops[i]; + struct xe_vm_pgtable_update *updates = pt_op->entries; + + for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) { + struct xe_vm *vm = pt_update->vops->vm; + struct xe_bo *pt_bo = updates[j].pt_bo; + + if (idx == chunk) + goto next_cmd; + + xe_tile_assert(tile, xe_bo_size(pt_bo) == SZ_4K); + + /* Map a PT at most once */ + if (pt_bo->update_index < 0) + pt_bo->update_index = current_update; + + addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, + pat_index, 0); + bb->cs[bb->len++] = lower_32_bits(addr); + bb->cs[bb->len++] = upper_32_bits(addr); + } + + j = 0; + } + +next_cmd: + ptes -= chunk; + ofs += chunk * sizeof(u64); + } + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + + (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; + for (i = 0; i < pt_update_ops->num_ops; ++i) { + struct xe_vm_pgtable_update_op *pt_op = + &pt_update_ops->ops[i]; + struct xe_vm_pgtable_update *updates = pt_op->entries; + + for (j = 0; j < pt_op->num_entries; ++j) { + struct xe_bo *pt_bo = updates[j].pt_bo; + + write_pgtable(tile, bb, addr + + pt_bo->update_index * XE_PAGE_SIZE, + pt_op, &updates[j], pt_update); + } + } + } else { + /* phys pages, no preamble required */ + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + for (i = 0; i < pt_update_ops->num_ops; ++i) { + struct xe_vm_pgtable_update_op *pt_op = + &pt_update_ops->ops[i]; + struct xe_vm_pgtable_update *updates = pt_op->entries; + + for (j = 0; j < pt_op->num_entries; ++j) + write_pgtable(tile, bb, 0, pt_op, &updates[j], + pt_update); + } + } + + job = xe_bb_create_migration_job(pt_update_ops->q, bb, + xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err_sa; + } + + xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); + + if (ops->pre_commit) { + pt_update->job = job; + err = ops->pre_commit(pt_update); + if (err) + goto err_job; + } + if (is_migrate) + mutex_lock(&m->job_mutex); + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + if (is_migrate) + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + drm_suballoc_free(sa_bo, fence); + + return fence; + +err_job: + xe_sched_job_put(job); +err_sa: + drm_suballoc_free(sa_bo, NULL); +err_bb: + xe_bb_free(bb, NULL); + return ERR_PTR(err); +} + +/** + * xe_migrate_update_pgtables() - Pipelined page-table update + * @m: The migrate context. + * @pt_update: PT update arguments + * + * Perform a pipelined page-table update. The update descriptors are typically + * built under the same lock critical section as a call to this function. If + * using the default engine for the updates, they will be performed in the + * order they grab the job_mutex. If different engines are used, external + * synchronization is needed for overlapping updates to maintain page-table + * consistency. Note that the meaning of "overlapping" is that the updates + * touch the same page-table, which might be a higher-level page-directory. + * If no pipelining is needed, then updates may be performed by the cpu. + * + * Return: A dma_fence that, when signaled, indicates the update completion. + */ +struct dma_fence * +xe_migrate_update_pgtables(struct xe_migrate *m, + struct xe_migrate_pt_update *pt_update) + +{ + struct xe_vm_pgtable_update_ops *pt_update_ops = + &pt_update->vops->pt_update_ops[pt_update->tile_id]; + struct dma_fence *fence; + + fence = xe_migrate_update_pgtables_cpu(m, pt_update); + + /* -ETIME indicates a job is needed, anything else is legit error */ + if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) + return fence; + + return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops); +} + +/** + * xe_migrate_wait() - Complete all operations using the xe_migrate context + * @m: Migrate context to wait for. + * + * Waits until the GPU no longer uses the migrate context's default engine + * or its page-table objects. FIXME: What about separate page-table update + * engines? + */ +void xe_migrate_wait(struct xe_migrate *m) +{ + if (m->fence) + dma_fence_wait(m->fence, false); +} + +static u32 pte_update_cmd_size(u64 size) +{ + u32 num_dword; + u64 entries = DIV_U64_ROUND_UP(size, XE_PAGE_SIZE); + + XE_WARN_ON(size > MAX_PREEMPTDISABLE_TRANSFER); + + /* + * MI_STORE_DATA_IMM command is used to update page table. Each + * instruction can update maximumly MAX_PTE_PER_SDI pte entries. To + * update n (n <= MAX_PTE_PER_SDI) pte entries, we need: + * + * - 1 dword for the MI_STORE_DATA_IMM command header (opcode etc) + * - 2 dword for the page table's physical location + * - 2*n dword for value of pte to fill (each pte entry is 2 dwords) + */ + num_dword = (1 + 2) * DIV_U64_ROUND_UP(entries, MAX_PTE_PER_SDI); + num_dword += entries * 2; + + return num_dword; +} + +static void build_pt_update_batch_sram(struct xe_migrate *m, + struct xe_bb *bb, u32 pt_offset, + struct drm_pagemap_addr *sram_addr, + u32 size, int level) +{ + u16 pat_index = xe_cache_pat_idx(tile_to_xe(m->tile), XE_CACHE_WB); + u64 gpu_page_size = 0x1ull << xe_pt_shift(level); + u32 ptes; + int i = 0; + + xe_tile_assert(m->tile, PAGE_ALIGNED(size)); + + ptes = DIV_ROUND_UP(size, gpu_page_size); + while (ptes) { + u32 chunk = min(MAX_PTE_PER_SDI, ptes); + + if (!level) + chunk = ALIGN_DOWN(chunk, PAGE_SIZE / XE_PAGE_SIZE); + + bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); + bb->cs[bb->len++] = pt_offset; + bb->cs[bb->len++] = 0; + + pt_offset += chunk * 8; + ptes -= chunk; + + while (chunk--) { + u64 addr = sram_addr[i].addr; + u64 pte; + + xe_tile_assert(m->tile, sram_addr[i].proto == + DRM_INTERCONNECT_SYSTEM || + sram_addr[i].proto == XE_INTERCONNECT_P2P); + xe_tile_assert(m->tile, addr); + xe_tile_assert(m->tile, PAGE_ALIGNED(addr)); + +again: + pte = m->q->vm->pt_ops->pte_encode_addr(m->tile->xe, + addr, pat_index, + level, false, 0); + bb->cs[bb->len++] = lower_32_bits(pte); + bb->cs[bb->len++] = upper_32_bits(pte); + + if (gpu_page_size < PAGE_SIZE) { + addr += XE_PAGE_SIZE; + if (!PAGE_ALIGNED(addr)) { + chunk--; + goto again; + } + i++; + } else { + i += gpu_page_size / PAGE_SIZE; + } + } + } +} + +static bool xe_migrate_vram_use_pde(struct drm_pagemap_addr *sram_addr, + unsigned long size) +{ + u32 large_size = (0x1 << xe_pt_shift(1)); + unsigned long i, incr = large_size / PAGE_SIZE; + + for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE); i += incr) + if (PAGE_SIZE << sram_addr[i].order != large_size) + return false; + + return true; +} + +#define XE_CACHELINE_BYTES 64ull +#define XE_CACHELINE_MASK (XE_CACHELINE_BYTES - 1) + +static u32 xe_migrate_copy_pitch(struct xe_device *xe, u32 len) +{ + u32 pitch; + + if (IS_ALIGNED(len, PAGE_SIZE)) + pitch = PAGE_SIZE; + else if (IS_ALIGNED(len, SZ_4K)) + pitch = SZ_4K; + else if (IS_ALIGNED(len, SZ_256)) + pitch = SZ_256; + else if (IS_ALIGNED(len, 4)) + pitch = 4; + else + pitch = 1; + + xe_assert(xe, pitch > 1 || xe->info.has_mem_copy_instr); + return pitch; +} + +static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, + unsigned long len, + unsigned long sram_offset, + struct drm_pagemap_addr *sram_addr, + u64 vram_addr, + struct dma_fence *deps, + const enum xe_migrate_copy_dir dir) +{ + struct xe_gt *gt = m->tile->primary_gt; + struct xe_device *xe = gt_to_xe(gt); + bool use_usm_batch = xe->info.has_usm; + struct dma_fence *fence = NULL; + u32 batch_size = 1; + u64 src_L0_ofs, dst_L0_ofs; + struct xe_sched_job *job; + struct xe_bb *bb; + u32 update_idx, pt_slot = 0; + unsigned long npages = DIV_ROUND_UP(len + sram_offset, PAGE_SIZE); + unsigned int pitch = xe_migrate_copy_pitch(xe, len); + int err; + unsigned long i, j; + bool use_pde = xe_migrate_vram_use_pde(sram_addr, len + sram_offset); + + if (!xe->info.has_mem_copy_instr && + drm_WARN_ON(&xe->drm, + (!IS_ALIGNED(len, pitch)) || (sram_offset | vram_addr) & XE_CACHELINE_MASK)) + return ERR_PTR(-EOPNOTSUPP); + + xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER); + + batch_size += pte_update_cmd_size(npages << PAGE_SHIFT); + batch_size += emit_copy_cmd_len(xe); + + bb = xe_bb_new(gt, batch_size, use_usm_batch); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + return ERR_PTR(err); + } + + /* + * If the order of a struct drm_pagemap_addr entry is greater than 0, + * the entry is populated by GPU pagemap but subsequent entries within + * the range of that order are not populated. + * build_pt_update_batch_sram() expects a fully populated array of + * struct drm_pagemap_addr. Ensure this is the case even with higher + * orders. + */ + for (i = 0; !use_pde && i < npages;) { + unsigned int order = sram_addr[i].order; + + for (j = 1; j < NR_PAGES(order) && i + j < npages; j++) + if (!sram_addr[i + j].addr) + sram_addr[i + j].addr = sram_addr[i].addr + j * PAGE_SIZE; + + i += NR_PAGES(order); + } + + if (use_pde) + build_pt_update_batch_sram(m, bb, m->large_page_copy_pdes, + sram_addr, npages << PAGE_SHIFT, 1); + else + build_pt_update_batch_sram(m, bb, pt_slot * XE_PAGE_SIZE, + sram_addr, npages << PAGE_SHIFT, 0); + + if (dir == XE_MIGRATE_COPY_TO_VRAM) { + if (use_pde) + src_L0_ofs = m->large_page_copy_ofs + sram_offset; + else + src_L0_ofs = xe_migrate_vm_addr(pt_slot, 0) + sram_offset; + dst_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr, false); + + } else { + src_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr, false); + if (use_pde) + dst_L0_ofs = m->large_page_copy_ofs + sram_offset; + else + dst_L0_ofs = xe_migrate_vm_addr(pt_slot, 0) + sram_offset; + } + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, len, pitch); + + job = xe_bb_create_migration_job(m->q, bb, + xe_migrate_batch_base(m, use_usm_batch), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err; + } + + xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); + + if (deps && !dma_fence_is_signaled(deps)) { + dma_fence_get(deps); + err = drm_sched_job_add_dependency(&job->drm, deps); + if (err) + dma_fence_wait(deps, false); + err = 0; + } + + mutex_lock(&m->job_mutex); + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + + return fence; + +err: + xe_bb_free(bb, NULL); + + return ERR_PTR(err); +} + +/** + * xe_migrate_to_vram() - Migrate to VRAM + * @m: The migration context. + * @npages: Number of pages to migrate. + * @src_addr: Array of DMA information (source of migrate) + * @dst_addr: Device physical address of VRAM (destination of migrate) + * @deps: struct dma_fence representing the dependencies that need + * to be signaled before migration. + * + * Copy from an array dma addresses to a VRAM device physical address + * + * Return: dma fence for migrate to signal completion on success, ERR_PTR on + * failure + */ +struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, + unsigned long npages, + struct drm_pagemap_addr *src_addr, + u64 dst_addr, + struct dma_fence *deps) +{ + return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr, + deps, XE_MIGRATE_COPY_TO_VRAM); +} + +/** + * xe_migrate_from_vram() - Migrate from VRAM + * @m: The migration context. + * @npages: Number of pages to migrate. + * @src_addr: Device physical address of VRAM (source of migrate) + * @dst_addr: Array of DMA information (destination of migrate) + * @deps: struct dma_fence representing the dependencies that need + * to be signaled before migration. + * + * Copy from a VRAM device physical address to an array dma addresses + * + * Return: dma fence for migrate to signal completion on success, ERR_PTR on + * failure + */ +struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, + unsigned long npages, + u64 src_addr, + struct drm_pagemap_addr *dst_addr, + struct dma_fence *deps) +{ + return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr, + deps, XE_MIGRATE_COPY_TO_SRAM); +} + +static void xe_migrate_dma_unmap(struct xe_device *xe, + struct drm_pagemap_addr *pagemap_addr, + int len, int write) +{ + unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE); + + for (i = 0; i < npages; ++i) { + if (!pagemap_addr[i].addr) + break; + + dma_unmap_page(xe->drm.dev, pagemap_addr[i].addr, PAGE_SIZE, + write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); + } + kfree(pagemap_addr); +} + +static struct drm_pagemap_addr *xe_migrate_dma_map(struct xe_device *xe, + void *buf, int len, + int write) +{ + struct drm_pagemap_addr *pagemap_addr; + unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE); + + pagemap_addr = kzalloc_objs(*pagemap_addr, npages); + if (!pagemap_addr) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < npages; ++i) { + dma_addr_t addr; + struct page *page; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : + DMA_FROM_DEVICE; + + if (is_vmalloc_addr(buf)) + page = vmalloc_to_page(buf); + else + page = virt_to_page(buf); + + addr = dma_map_page(xe->drm.dev, page, 0, PAGE_SIZE, dir); + if (dma_mapping_error(xe->drm.dev, addr)) + goto err_fault; + + pagemap_addr[i] = + drm_pagemap_addr_encode(addr, + DRM_INTERCONNECT_SYSTEM, + 0, dir); + buf += PAGE_SIZE; + } + + return pagemap_addr; + +err_fault: + xe_migrate_dma_unmap(xe, pagemap_addr, len, write); + return ERR_PTR(-EFAULT); +} + +/** + * xe_migrate_access_memory - Access memory of a BO via GPU + * + * @m: The migration context. + * @bo: buffer object + * @offset: access offset into buffer object + * @buf: pointer to caller memory to read into or write from + * @len: length of access + * @write: write access + * + * Access memory of a BO via GPU either reading in or writing from a passed in + * pointer. Pointer is dma mapped for GPU access and GPU commands are issued to + * read to or write from pointer. + * + * Returns: + * 0 if successful, negative error code on failure. + */ +int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, + unsigned long offset, void *buf, int len, + int write) +{ + struct xe_tile *tile = m->tile; + struct xe_device *xe = tile_to_xe(tile); + struct xe_res_cursor cursor; + struct dma_fence *fence = NULL; + struct drm_pagemap_addr *pagemap_addr; + unsigned long page_offset = (unsigned long)buf & ~PAGE_MASK; + int bytes_left = len, current_page = 0; + void *orig_buf = buf; + + xe_bo_assert_held(bo); + + /* Use bounce buffer for small access and unaligned access */ + if (!xe->info.has_mem_copy_instr && + (!IS_ALIGNED(len, 4) || + !IS_ALIGNED(page_offset, XE_CACHELINE_BYTES) || + !IS_ALIGNED(offset, XE_CACHELINE_BYTES))) { + int buf_offset = 0; + void *bounce; + int err; + + BUILD_BUG_ON(!is_power_of_2(XE_CACHELINE_BYTES)); + bounce = kmalloc(XE_CACHELINE_BYTES, GFP_KERNEL); + if (!bounce) + return -ENOMEM; + + /* + * Less than ideal for large unaligned access but this should be + * fairly rare, can fixup if this becomes common. + */ + do { + int copy_bytes = min_t(int, bytes_left, + XE_CACHELINE_BYTES - + (offset & XE_CACHELINE_MASK)); + int ptr_offset = offset & XE_CACHELINE_MASK; + + err = xe_migrate_access_memory(m, bo, + offset & + ~XE_CACHELINE_MASK, + bounce, + XE_CACHELINE_BYTES, 0); + if (err) + break; + + if (write) { + memcpy(bounce + ptr_offset, buf + buf_offset, copy_bytes); + + err = xe_migrate_access_memory(m, bo, + offset & ~XE_CACHELINE_MASK, + bounce, + XE_CACHELINE_BYTES, write); + if (err) + break; + } else { + memcpy(buf + buf_offset, bounce + ptr_offset, + copy_bytes); + } + + bytes_left -= copy_bytes; + buf_offset += copy_bytes; + offset += copy_bytes; + } while (bytes_left); + + kfree(bounce); + return err; + } + + pagemap_addr = xe_migrate_dma_map(xe, buf, len + page_offset, write); + if (IS_ERR(pagemap_addr)) + return PTR_ERR(pagemap_addr); + + xe_res_first(bo->ttm.resource, offset, xe_bo_size(bo) - offset, &cursor); + + do { + struct dma_fence *__fence; + u64 vram_addr = vram_region_gpu_offset(bo->ttm.resource) + + cursor.start; + int current_bytes; + u32 pitch; + + if (cursor.size > MAX_PREEMPTDISABLE_TRANSFER) + current_bytes = min_t(int, bytes_left, + MAX_PREEMPTDISABLE_TRANSFER); + else + current_bytes = min_t(int, bytes_left, cursor.size); + + pitch = xe_migrate_copy_pitch(xe, current_bytes); + if (xe->info.has_mem_copy_instr) + current_bytes = min_t(int, current_bytes, U16_MAX * pitch); + else + current_bytes = min_t(int, current_bytes, + round_down(S16_MAX * pitch, + XE_CACHELINE_BYTES)); + + __fence = xe_migrate_vram(m, current_bytes, + (unsigned long)buf & ~PAGE_MASK, + &pagemap_addr[current_page], + vram_addr, NULL, write ? + XE_MIGRATE_COPY_TO_VRAM : + XE_MIGRATE_COPY_TO_SRAM); + if (IS_ERR(__fence)) { + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + fence = __fence; + goto out_err; + } + + dma_fence_put(fence); + fence = __fence; + + buf += current_bytes; + offset += current_bytes; + current_page = (int)(buf - orig_buf) / PAGE_SIZE; + bytes_left -= current_bytes; + if (bytes_left) + xe_res_next(&cursor, current_bytes); + } while (bytes_left); + + dma_fence_wait(fence, false); + dma_fence_put(fence); + +out_err: + xe_migrate_dma_unmap(xe, pagemap_addr, len + page_offset, write); + return IS_ERR(fence) ? PTR_ERR(fence) : 0; +} + +/** + * xe_migrate_job_lock() - Lock migrate job lock + * @m: The migration context. + * @q: Queue associated with the operation which requires a lock + * + * Lock the migrate job lock if the queue is a migration queue, otherwise + * assert the VM's dma-resv is held (user queue's have own locking). + */ +void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q) +{ + bool is_migrate = q == m->q; + + if (is_migrate) + mutex_lock(&m->job_mutex); + else + xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ +} + +/** + * xe_migrate_job_unlock() - Unlock migrate job lock + * @m: The migration context. + * @q: Queue associated with the operation which requires a lock + * + * Unlock the migrate job lock if the queue is a migration queue, otherwise + * assert the VM's dma-resv is held (user queue's have own locking). + */ +void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q) +{ + bool is_migrate = q == m->q; + + if (is_migrate) + mutex_unlock(&m->job_mutex); + else + xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ +} + +#if IS_ENABLED(CONFIG_PROVE_LOCKING) +/** + * xe_migrate_job_lock_assert() - Assert migrate job lock held of queue + * @q: Migrate queue + */ +void xe_migrate_job_lock_assert(struct xe_exec_queue *q) +{ + struct xe_migrate *m = gt_to_tile(q->gt)->migrate; + + xe_gt_assert(q->gt, q == m->q); + lockdep_assert_held(&m->job_mutex); +} +#endif + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_migrate.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h new file mode 100644 index 000000000..c3a268b01 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef _XE_MIGRATE_H_ +#define _XE_MIGRATE_H_ + +#include + +struct dma_fence; +struct drm_pagemap_addr; +struct iosys_map; +struct ttm_resource; + +struct xe_bo; +struct xe_gt; +struct xe_tlb_inval_job; +struct xe_exec_queue; +struct xe_migrate; +struct xe_migrate_pt_update; +struct xe_sync_entry; +struct xe_pt; +struct xe_tile; +struct xe_vm; +struct xe_vm_pgtable_update; +struct xe_vma; + +enum xe_sriov_vf_ccs_rw_ctxs; + +enum xe_migrate_copy_dir { + XE_MIGRATE_COPY_TO_VRAM, + XE_MIGRATE_COPY_TO_SRAM, +}; + +/** + * struct xe_migrate_pt_update_ops - Callbacks for the + * xe_migrate_update_pgtables() function. + */ +struct xe_migrate_pt_update_ops { + /** + * @populate: Populate a command buffer or page-table with ptes. + * @pt_update: Embeddable callback argument. + * @tile: The tile for the current operation. + * @map: struct iosys_map into the memory to be populated. + * @pos: If @map is NULL, map into the memory to be populated. + * @ofs: qword offset into @map, unused if @map is NULL. + * @num_qwords: Number of qwords to write. + * @update: Information about the PTEs to be inserted. + * + * This interface is intended to be used as a callback into the + * page-table system to populate command buffers or shared + * page-tables with PTEs. + */ + void (*populate)(struct xe_migrate_pt_update *pt_update, + struct xe_tile *tile, struct iosys_map *map, + void *pos, u32 ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update); + /** + * @clear: Clear a command buffer or page-table with ptes. + * @pt_update: Embeddable callback argument. + * @tile: The tile for the current operation. + * @map: struct iosys_map into the memory to be populated. + * @pos: If @map is NULL, map into the memory to be populated. + * @ofs: qword offset into @map, unused if @map is NULL. + * @num_qwords: Number of qwords to write. + * @update: Information about the PTEs to be inserted. + * + * This interface is intended to be used as a callback into the + * page-table system to populate command buffers or shared + * page-tables with PTEs. + */ + void (*clear)(struct xe_migrate_pt_update *pt_update, + struct xe_tile *tile, struct iosys_map *map, + void *pos, u32 ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update); + + /** + * @pre_commit: Callback to be called just before arming the + * sched_job. + * @pt_update: Pointer to embeddable callback argument. + * + * Return: 0 on success, negative error code on error. + */ + int (*pre_commit)(struct xe_migrate_pt_update *pt_update); +}; + +/** + * struct xe_migrate_pt_update - Argument to the + * struct xe_migrate_pt_update_ops callbacks. + * + * Intended to be subclassed to support additional arguments if necessary. + */ +struct xe_migrate_pt_update { + /** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */ + const struct xe_migrate_pt_update_ops *ops; + /** @vops: VMA operations */ + struct xe_vma_ops *vops; + /** @job: The job if a GPU page-table update. NULL otherwise */ + struct xe_sched_job *job; + /** + * @ijob: The TLB invalidation job for primary GT. NULL otherwise + */ + struct xe_tlb_inval_job *ijob; + /** + * @mjob: The TLB invalidation job for media GT. NULL otherwise + */ + struct xe_tlb_inval_job *mjob; + /** @tile_id: Tile ID of the update */ + u8 tile_id; +}; + +struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile); +int xe_migrate_init(struct xe_migrate *m); + +struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, + unsigned long npages, + struct drm_pagemap_addr *src_addr, + u64 dst_addr, + struct dma_fence *deps); + +struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, + unsigned long npages, + u64 src_addr, + struct drm_pagemap_addr *dst_addr, + struct dma_fence *deps); + +struct dma_fence *xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + struct xe_bo *dst_bo, + struct ttm_resource *src, + struct ttm_resource *dst, + bool copy_only_ccs); + +struct dma_fence *xe_migrate_resolve(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *res); + +int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, + struct xe_bo *src_bo, + struct ttm_resource *new_mem, + enum xe_sriov_vf_ccs_rw_ctxs read_write); + +void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write, + bool bound); + +struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate); +struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate); +struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_offset, + struct xe_bo *sysmem_bo, u64 sysmem_offset, + u64 size, enum xe_migrate_copy_dir dir); +int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, + unsigned long offset, void *buf, int len, + int write); + +#define XE_MIGRATE_CLEAR_FLAG_BO_DATA BIT(0) +#define XE_MIGRATE_CLEAR_FLAG_CCS_DATA BIT(1) +#define XE_MIGRATE_CLEAR_FLAG_FULL (XE_MIGRATE_CLEAR_FLAG_BO_DATA | \ + XE_MIGRATE_CLEAR_FLAG_CCS_DATA) +struct dma_fence *xe_migrate_clear(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *dst, + u32 clear_flags); + +struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m); + +struct dma_fence * +xe_migrate_update_pgtables(struct xe_migrate *m, + struct xe_migrate_pt_update *pt_update); + +void xe_migrate_wait(struct xe_migrate *m); + +#if IS_ENABLED(CONFIG_PROVE_LOCKING) +void xe_migrate_job_lock_assert(struct xe_exec_queue *q); +#else +static inline void xe_migrate_job_lock_assert(struct xe_exec_queue *q) +{ +} +#endif + +void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q); +void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q); + +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate_doc.h b/drivers/gpu/drm/xe/xe_migrate_doc.h new file mode 100644 index 000000000..c082bc0b7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_migrate_doc.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_MIGRATE_DOC_H_ +#define _XE_MIGRATE_DOC_H_ + +/** + * DOC: Migrate Layer + * + * The Xe migrate layer is used generate jobs which can copy memory (eviction), + * clear memory, or program tables (binds). This layer exists in every GT, has + * a migrate engine, and uses a special VM for all generated jobs. + * + * Special VM details + * ================== + * + * The special VM is configured with a page structure where we can dynamically + * map BOs which need to be copied and cleared, dynamically map other VM's page + * table BOs for updates, and identity map the entire device's VRAM with 1 GB + * pages. + * + * Currently the page structure consists of 32 physical pages with 16 being + * reserved for BO mapping during copies and clear, 1 reserved for kernel binds, + * several pages are needed to setup the identity mappings (exact number based + * on how many bits of address space the device has), and the rest are reserved + * user bind operations. + * + * TODO: Diagram of layout + * + * Bind jobs + * ========= + * + * A bind job consist of two batches and runs either on the migrate engine + * (kernel binds) or the bind engine passed in (user binds). In both cases the + * VM of the engine is the migrate VM. + * + * The first batch is used to update the migration VM page structure to point to + * the bind VM page table BOs which need to be updated. A physical page is + * required for this. If it is a user bind, the page is allocated from pool of + * pages reserved user bind operations with drm_suballoc managing this pool. If + * it is a kernel bind, the page reserved for kernel binds is used. + * + * The first batch is only required for devices without VRAM as when the device + * has VRAM the bind VM page table BOs are in VRAM and the identity mapping can + * be used. + * + * The second batch is used to program page table updated in the bind VM. Why + * not just one batch? Well the TLBs need to be invalidated between these two + * batches and that only can be done from the ring. + * + * When the bind job complete, the page allocated is returned the pool of pages + * reserved for user bind operations if a user bind. No need do this for kernel + * binds as the reserved kernel page is serially used by each job. + * + * Copy / clear jobs + * ================= + * + * A copy or clear job consist of two batches and runs on the migrate engine. + * + * Like binds, the first batch is used update the migration VM page structure. + * In copy jobs, we need to map the source and destination of the BO into page + * the structure. In clear jobs, we just need to add 1 mapping of BO into the + * page structure. We use the 16 reserved pages in migration VM for mappings, + * this gives us a maximum copy size of 16 MB and maximum clear size of 32 MB. + * + * The second batch is used do either do the copy or clear. Again similar to + * binds, two batches are required as the TLBs need to be invalidated from the + * ring between the batches. + * + * More than one job will be generated if the BO is larger than maximum copy / + * clear size. + * + * Future work + * =========== + * + * Update copy and clear code to use identity mapped VRAM. + * + * Can we rework the use of the pages async binds to use all the entries in each + * page? + * + * Using large pages for sysmem mappings. + * + * Is it possible to identity map the sysmem? We should explore this. + */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c new file mode 100644 index 000000000..7fa18dfcb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2023 Intel Corporation + */ + +#include "xe_mmio.h" + +#include +#include +#include +#include + +#include + +#include "regs/xe_bars.h" +#include "xe_device.h" +#include "xe_gt_sriov_vf.h" +#include "xe_printk.h" +#include "xe_sriov.h" +#include "xe_tile_printk.h" +#include "xe_trace.h" +#include "xe_wa.h" + +#include "generated/xe_device_wa_oob.h" + +/* + * On multi-tile devices, partition the BAR space for MMIO on each tile, + * possibly accounting for register override on the number of tiles available. + * tile_mmio_size contains both the tile's 4MB register space, as well as + * additional space for the GTT and other (possibly unused) regions). + * Resulting memory layout is like below: + * + * .----------------------. <- tile_count * tile_mmio_size + * | .... | + * |----------------------| <- 2 * tile_mmio_size + * | tile1 GTT + other | + * |----------------------| <- 1 * tile_mmio_size + 4MB + * | tile1->mmio.regs | + * |----------------------| <- 1 * tile_mmio_size + * | tile0 GTT + other | + * |----------------------| <- 4MB + * | tile0->mmio.regs | + * '----------------------' <- 0MB + */ +static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) +{ + struct xe_tile *tile; + u8 id; + + for_each_remote_tile(tile, xe, id) + xe_mmio_init(&tile->mmio, tile, xe->mmio.regs + id * tile_mmio_size, SZ_4M); +} + +/** + * xe_mmio_probe_tiles() - Initialize all tiles' MMIO + * @xe: the &xe_device + * + * Initialize the remaining tiles' MMIO instances. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_mmio_probe_tiles(struct xe_device *xe) +{ + size_t tile_mmio_size = SZ_16M; + + /* + * Nothing to be done as tile 0 has already been setup earlier with the + * entire BAR mapped - see xe_mmio_probe_early() + */ + if (xe->info.tile_count == 1) + return 0; + + if (xe->mmio.size < xe->info.tile_count * tile_mmio_size) { + xe_err(xe, "GTTMMADR_BAR is too small for %d tiles: %zu\n", + xe->info.tile_count, xe->mmio.size); + return -EIO; + } + + mmio_multi_tile_setup(xe, tile_mmio_size); + return 0; +} + +static void mmio_fini(void *arg) +{ + struct xe_device *xe = arg; + + xe->mmio.regs = NULL; +} + +/** + * xe_mmio_probe_early() - Probe and initialize device's MMIO + * @xe: the &xe_device + * + * Map the entire GTTMMADR_BAR and initialize the first tile's MMIO instance. + * + * The first 16MB of the GTTMMADR_BAR always belongs to the root tile, and + * includes: registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB). + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_mmio_probe_early(struct xe_device *xe) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(xe); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + xe->mmio.regs = pcim_iomap(pdev, GTTMMADR_BAR, 0); + if (!xe->mmio.regs) { + xe_err(xe, "Failed to map GTTMMADR_BAR\n"); + return -EIO; + } + + xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR); + if (xe->mmio.size < SZ_16M) { + xe_err(xe, "GTTMMADR_BAR is too small: %zu\n", xe->mmio.size); + return -EIO; + } + + /* Setup first tile; other tiles (if present) will be setup later. */ + xe_mmio_init(&root_tile->mmio, root_tile, xe->mmio.regs, SZ_4M); + + return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe); +} +ALLOW_ERROR_INJECTION(xe_mmio_probe_early, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_mmio_init() - Initialize an MMIO instance + * @mmio: Pointer to the MMIO instance to initialize + * @tile: The tile to which the MMIO region belongs + * @ptr: Pointer to the start of the MMIO region + * @size: The size of the MMIO region in bytes + * + * This is a convenience function for minimal initialization of struct xe_mmio. + */ +void xe_mmio_init(struct xe_mmio *mmio, struct xe_tile *tile, void __iomem *ptr, u32 size) +{ + xe_tile_assert(tile, size <= XE_REG_ADDR_MAX); + + mmio->regs = ptr; + mmio->regs_size = size; + mmio->tile = tile; +} + +static bool mmio_available(struct xe_mmio *mmio) +{ + return !xe_tile_WARN_ON_ONCE(mmio->tile, !mmio->tile->xe->mmio.regs); +} + +static void mmio_flush_pending_writes(struct xe_mmio *mmio) +{ +#define DUMMY_REG_OFFSET 0x130030 + int i; + + if (!XE_DEVICE_WA(mmio->tile->xe, 15015404425)) + return; + + /* 4 dummy writes */ + for (i = 0; i < 4; i++) + writel(0, mmio->regs + DUMMY_REG_OFFSET); +} + +u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg) +{ + u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); + u8 val; + + if (!mmio_available(mmio)) + return 0; + + mmio_flush_pending_writes(mmio); + + val = readb(mmio->regs + addr); + trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); + + return val; +} + +void xe_mmio_write8(struct xe_mmio *mmio, struct xe_reg reg, u8 val) +{ + u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); + + if (!mmio_available(mmio)) + return; + + trace_xe_reg_rw(mmio, true, addr, val, sizeof(val)); + + writeb(val, mmio->regs + addr); +} + +u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg) +{ + u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); + u16 val; + + if (!mmio_available(mmio)) + return 0; + + mmio_flush_pending_writes(mmio); + + val = readw(mmio->regs + addr); + trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); + + return val; +} + +void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val) +{ + u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); + + if (!mmio_available(mmio)) + return; + + trace_xe_reg_rw(mmio, true, addr, val, sizeof(val)); + + if (!reg.vf && IS_SRIOV_VF(mmio->tile->xe)) + xe_gt_sriov_vf_write32(mmio->sriov_vf_gt ?: + mmio->tile->primary_gt, reg, val); + else + writel(val, mmio->regs + addr); +} + +u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg) +{ + u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); + u32 val; + + if (!mmio_available(mmio)) + return 0; + + mmio_flush_pending_writes(mmio); + + if (!reg.vf && IS_SRIOV_VF(mmio->tile->xe)) + val = xe_gt_sriov_vf_read32(mmio->sriov_vf_gt ?: + mmio->tile->primary_gt, reg); + else + val = readl(mmio->regs + addr); + + trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); + + return val; +} + +u32 xe_mmio_rmw32(struct xe_mmio *mmio, struct xe_reg reg, u32 clr, u32 set) +{ + u32 old, reg_val; + + old = xe_mmio_read32(mmio, reg); + reg_val = (old & ~clr) | set; + xe_mmio_write32(mmio, reg, reg_val); + + return old; +} + +int xe_mmio_write32_and_verify(struct xe_mmio *mmio, + struct xe_reg reg, u32 val, u32 mask, u32 eval) +{ + u32 reg_val; + + xe_mmio_write32(mmio, reg, val); + reg_val = xe_mmio_read32(mmio, reg); + + return (reg_val & mask) != eval ? -EINVAL : 0; +} + +bool xe_mmio_in_range(const struct xe_mmio *mmio, + const struct xe_mmio_range *range, + struct xe_reg reg) +{ + u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); + + return range && addr >= range->start && addr <= range->end; +} + +/** + * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads + * @mmio: MMIO target + * @reg: register to read value from + * + * Although Intel GPUs have some 64-bit registers, the hardware officially + * only supports GTTMMADR register reads of 32 bits or smaller. Even if + * a readq operation may return a reasonable value, that violation of the + * spec shouldn't be relied upon and all 64-bit register reads should be + * performed as two 32-bit reads of the upper and lower dwords. + * + * When reading registers that may be changing (such as + * counters), a rollover of the lower dword between the two 32-bit reads + * can be problematic. This function attempts to ensure the upper dword has + * stabilized before returning the 64-bit value. + * + * Note that because this function may re-read the register multiple times + * while waiting for the value to stabilize it should not be used to read + * any registers where read operations have side effects. + * + * Returns the value of the 64-bit register. + */ +u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) +{ + struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; + u32 ldw, udw, oldudw, retries; + + /* + * The two dwords of a 64-bit register can never straddle the offset + * adjustment cutoff. + */ + xe_tile_assert(mmio->tile, !in_range(mmio->adj_limit, reg.addr + 1, 7)); + + oldudw = xe_mmio_read32(mmio, reg_udw); + for (retries = 5; retries; --retries) { + ldw = xe_mmio_read32(mmio, reg); + udw = xe_mmio_read32(mmio, reg_udw); + + if (udw == oldudw) + break; + + oldudw = udw; + } + + xe_tile_WARN(mmio->tile, retries == 0, + "MMIO: 64-bit read of %#x did not stabilize\n", reg.addr); + + return (u64)udw << 32 | ldw; +} + +static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, + u32 timeout_us, u32 *out_val, bool atomic, bool expect_match) +{ + ktime_t cur = ktime_get_raw(); + const ktime_t end = ktime_add_us(cur, timeout_us); + int ret = -ETIMEDOUT; + s64 wait = 10; + u32 read; + bool check; + + for (;;) { + read = xe_mmio_read32(mmio, reg); + + check = (read & mask) == val; + if (!expect_match) + check = !check; + + if (check) { + ret = 0; + break; + } + + cur = ktime_get_raw(); + if (!ktime_before(cur, end)) + break; + + if (ktime_after(ktime_add_us(cur, wait), end)) + wait = ktime_us_delta(end, cur); + + if (atomic) + udelay(wait); + else + usleep_range(wait, wait << 1); + wait <<= 1; + } + + if (ret != 0) { + read = xe_mmio_read32(mmio, reg); + + check = (read & mask) == val; + if (!expect_match) + check = !check; + + if (check) + ret = 0; + } + + if (out_val) + *out_val = read; + + return ret; +} + +/** + * xe_mmio_wait32() - Wait for a register to match the desired masked value + * @mmio: MMIO target + * @reg: register to read value from + * @mask: mask to be applied to the value read from the register + * @val: desired value after applying the mask + * @timeout_us: time out after this period of time. Wait logic tries to be + * smart, applying an exponential backoff until @timeout_us is reached. + * @out_val: if not NULL, points where to store the last unmasked value + * @atomic: needs to be true if calling from an atomic context + * + * This function polls for the desired masked value and returns zero on success + * or -ETIMEDOUT if timed out. + * + * Note that @timeout_us represents the minimum amount of time to wait before + * giving up. The actual time taken by this function can be a little more than + * @timeout_us for different reasons, specially in non-atomic contexts. Thus, + * it is possible that this function succeeds even after @timeout_us has passed. + */ +int xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, + u32 *out_val, bool atomic) +{ + return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, true); +} + +/** + * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value + * @mmio: MMIO target + * @reg: register to read value from + * @mask: mask to be applied to the value read from the register + * @val: value not to be matched after applying the mask + * @timeout_us: time out after this period of time + * @out_val: if not NULL, points where to store the last unmasked value + * @atomic: needs to be true if calling from an atomic context + * + * This function works exactly like xe_mmio_wait32() with the exception that + * @val is expected not to be matched. + */ +int xe_mmio_wait32_not(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, + u32 *out_val, bool atomic) +{ + return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, false); +} + +#ifdef CONFIG_PCI_IOV +static size_t vf_regs_stride(struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) > 1200 ? 0x400 : 0x1000; +} + +/** + * xe_mmio_init_vf_view() - Initialize an MMIO instance for accesses like the VF + * @mmio: the target &xe_mmio to initialize as VF's view + * @base: the source &xe_mmio to initialize from + * @vfid: the VF identifier + */ +void xe_mmio_init_vf_view(struct xe_mmio *mmio, const struct xe_mmio *base, unsigned int vfid) +{ + struct xe_tile *tile = base->tile; + struct xe_device *xe = tile->xe; + size_t offset = vf_regs_stride(xe) * vfid; + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid); + xe_assert(xe, !base->sriov_vf_gt); + xe_assert(xe, base->regs_size > offset); + + *mmio = *base; + mmio->regs += offset; + mmio->regs_size -= offset; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_mmio.h b/drivers/gpu/drm/xe/xe_mmio.h new file mode 100644 index 000000000..befe021f2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021-2023 Intel Corporation + */ + +#ifndef _XE_MMIO_H_ +#define _XE_MMIO_H_ + +#include "xe_mmio_types.h" + +struct xe_device; +struct xe_reg; + +int xe_mmio_probe_early(struct xe_device *xe); +int xe_mmio_probe_tiles(struct xe_device *xe); + +void xe_mmio_init(struct xe_mmio *mmio, struct xe_tile *tile, void __iomem *ptr, u32 size); + +u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg); +void xe_mmio_write8(struct xe_mmio *mmio, struct xe_reg reg, u8 val); +u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg); +void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val); +u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg); +u32 xe_mmio_rmw32(struct xe_mmio *mmio, struct xe_reg reg, u32 clr, u32 set); +int xe_mmio_write32_and_verify(struct xe_mmio *mmio, struct xe_reg reg, u32 val, u32 mask, u32 eval); +bool xe_mmio_in_range(const struct xe_mmio *mmio, const struct xe_mmio_range *range, struct xe_reg reg); + +u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg); +int xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, + u32 timeout_us, u32 *out_val, bool atomic); +int xe_mmio_wait32_not(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, + u32 val, u32 timeout_us, u32 *out_val, bool atomic); + +static inline u32 xe_mmio_adjusted_addr(const struct xe_mmio *mmio, u32 addr) +{ + if (addr < mmio->adj_limit) + addr += mmio->adj_offset; + return addr; +} + +#ifdef CONFIG_PCI_IOV +void xe_mmio_init_vf_view(struct xe_mmio *mmio, const struct xe_mmio *base, unsigned int vfid); +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c new file mode 100644 index 000000000..3741ae60f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_mmio_gem.h" + +#include +#include +#include + +#include "xe_device_types.h" + +/** + * DOC: Exposing MMIO regions to userspace + * + * In certain cases, the driver may allow userspace to mmap a portion of the hardware registers. + * + * This can be done as follows: + * 1. Call xe_mmio_gem_create() to create a GEM object with an mmap-able fake offset. + * 2. Use xe_mmio_gem_mmap_offset() on the created GEM object to retrieve the fake offset. + * 3. Provide the fake offset to userspace. + * 4. Userspace can call mmap with the fake offset. The length provided to mmap + * must match the size of the GEM object. + * 5. When the region is no longer needed, call xe_mmio_gem_destroy() to release the GEM object. + * + * NOTE: The exposed MMIO region must be page-aligned with regards to its BAR offset and size. + * + * WARNING: Exposing MMIO regions to userspace can have security and stability implications. + * Make sure not to expose any sensitive registers. + */ + +static void xe_mmio_gem_free(struct drm_gem_object *); +static int xe_mmio_gem_mmap(struct drm_gem_object *, struct vm_area_struct *); +static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *); + +struct xe_mmio_gem { + struct drm_gem_object base; + phys_addr_t phys_addr; +}; + +static const struct vm_operations_struct vm_ops = { + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, + .fault = xe_mmio_gem_vm_fault, +}; + +static const struct drm_gem_object_funcs xe_mmio_gem_funcs = { + .free = xe_mmio_gem_free, + .mmap = xe_mmio_gem_mmap, + .vm_ops = &vm_ops, +}; + +static inline struct xe_mmio_gem *to_xe_mmio_gem(struct drm_gem_object *obj) +{ + return container_of(obj, struct xe_mmio_gem, base); +} + +/** + * xe_mmio_gem_create - Expose an MMIO region to userspace + * @xe: The xe device + * @file: DRM file descriptor + * @phys_addr: Start of the exposed MMIO region + * @size: The size of the exposed MMIO region + * + * This function creates a GEM object that exposes an MMIO region with an mmap-able + * fake offset. + * + * See: "Exposing MMIO regions to userspace" + */ +struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file, + phys_addr_t phys_addr, size_t size) +{ + struct xe_mmio_gem *obj; + struct drm_gem_object *base; + int err; + + if ((phys_addr % PAGE_SIZE != 0) || (size % PAGE_SIZE != 0)) + return ERR_PTR(-EINVAL); + + obj = kzalloc_obj(*obj); + if (!obj) + return ERR_PTR(-ENOMEM); + + base = &obj->base; + base->funcs = &xe_mmio_gem_funcs; + obj->phys_addr = phys_addr; + + drm_gem_private_object_init(&xe->drm, base, size); + + err = drm_gem_create_mmap_offset(base); + if (err) + goto free_gem; + + err = drm_vma_node_allow(&base->vma_node, file); + if (err) + goto free_gem; + + return obj; + +free_gem: + xe_mmio_gem_free(base); + return ERR_PTR(err); +} + +/** + * xe_mmio_gem_mmap_offset - Return the mmap-able fake offset + * @gem: the GEM object created with xe_mmio_gem_create() + * + * This function returns the mmap-able fake offset allocated during + * xe_mmio_gem_create(). + * + * See: "Exposing MMIO regions to userspace" + */ +u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem) +{ + return drm_vma_node_offset_addr(&gem->base.vma_node); +} + +static void xe_mmio_gem_free(struct drm_gem_object *base) +{ + struct xe_mmio_gem *obj = to_xe_mmio_gem(base); + + drm_gem_object_release(base); + kfree(obj); +} + +/** + * xe_mmio_gem_destroy - Destroy the GEM object that exposes an MMIO region + * @gem: the GEM object to destroy + * + * This function releases resources associated with the GEM object created by + * xe_mmio_gem_create(). + * + * See: "Exposing MMIO regions to userspace" + */ +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem) +{ + xe_mmio_gem_free(&gem->base); +} + +static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct *vma) +{ + if (vma->vm_end - vma->vm_start != base->size) + return -EINVAL; + + if ((vma->vm_flags & VM_SHARED) == 0) + return -EINVAL; + + /* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 */ + vma->vm_pgoff = 0; + vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); + vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | + VM_DONTCOPY | VM_NORESERVE); + + /* Defer actual mapping to the fault handler. */ + return 0; +} + +static void xe_mmio_gem_release_dummy_page(struct drm_device *dev, void *res) +{ + __free_page((struct page *)res); +} + +static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_area_struct *vma) +{ + struct drm_gem_object *base = vma->vm_private_data; + struct drm_device *dev = base->dev; + vm_fault_t ret = VM_FAULT_NOPAGE; + struct page *page; + unsigned long pfn; + unsigned long i; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + if (drmm_add_action_or_reset(dev, xe_mmio_gem_release_dummy_page, page)) + return VM_FAULT_OOM; + + pfn = page_to_pfn(page); + + /* Map the entire VMA to the same dummy page */ + for (i = 0; i < base->size; i += PAGE_SIZE) { + unsigned long addr = vma->vm_start + i; + + ret = vmf_insert_pfn(vma, addr, pfn); + if (ret & VM_FAULT_ERROR) + break; + } + + return ret; +} + +static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *base = vma->vm_private_data; + struct xe_mmio_gem *obj = to_xe_mmio_gem(base); + struct drm_device *dev = base->dev; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long i; + int idx; + + if (!drm_dev_enter(dev, &idx)) { + /* + * Provide a dummy page to avoid SIGBUS for events such as hot-unplug. + * This gives the userspace the option to recover instead of crashing. + * It is assumed the userspace will receive the notification via some + * other channel (e.g. drm uevent). + */ + return xe_mmio_gem_vm_fault_dummy_page(vma); + } + + for (i = 0; i < base->size; i += PAGE_SIZE) { + unsigned long addr = vma->vm_start + i; + unsigned long phys_addr = obj->phys_addr + i; + + ret = vmf_insert_pfn(vma, addr, PHYS_PFN(phys_addr)); + if (ret & VM_FAULT_ERROR) + break; + } + + drm_dev_exit(idx); + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.h b/drivers/gpu/drm/xe/xe_mmio_gem.h new file mode 100644 index 000000000..4b76d5586 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio_gem.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_MMIO_GEM_H_ +#define _XE_MMIO_GEM_H_ + +#include + +struct drm_file; +struct xe_device; +struct xe_mmio_gem; + +struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file, + phys_addr_t phys_addr, size_t size); +u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem); +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem); + +#endif /* _XE_MMIO_GEM_H_ */ diff --git a/drivers/gpu/drm/xe/xe_mmio_types.h b/drivers/gpu/drm/xe/xe_mmio_types.h new file mode 100644 index 000000000..99e8f269e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio_types.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022-2026 Intel Corporation + */ + +#ifndef _XE_MMIO_TYPES_H_ +#define _XE_MMIO_TYPES_H_ + +#include + +struct xe_gt; +struct xe_tile; + +/** + * struct xe_mmio - register mmio structure + * + * Represents an MMIO region that the CPU may use to access registers. A + * region may share its IO map with other regions (e.g., all GTs within a + * tile share the same map with their parent tile, but represent different + * subregions of the overall IO space). + */ +struct xe_mmio { + /** @tile: Backpointer to tile, used for tracing */ + struct xe_tile *tile; + + /** @regs: Map used to access registers. */ + void __iomem *regs; + + /** + * @sriov_vf_gt: Backpointer to GT. + * + * This pointer is only set for GT MMIO regions and only when running + * as an SRIOV VF structure + */ + struct xe_gt *sriov_vf_gt; + + /** + * @regs_size: Length of the register region within the map. + * + * The size of the iomap set in *regs is generally larger than the + * register mmio space since it includes unused regions and/or + * non-register regions such as the GGTT PTEs. + */ + size_t regs_size; + + /** @adj_limit: adjust MMIO address if address is below this value */ + u32 adj_limit; + + /** @adj_offset: offset to add to MMIO address when adjusting */ + u32 adj_offset; +}; + +/** + * struct xe_mmio_range - register range structure + * + * @start: first register offset in the range. + * @end: last register offset in the range. + */ +struct xe_mmio_range { + u32 start; + u32 end; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_mocs.c b/drivers/gpu/drm/xe/xe_mocs.c new file mode 100644 index 000000000..1d19df860 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mocs.c @@ -0,0 +1,831 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_mocs.h" + +#include "regs/xe_gt_regs.h" +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "xe_pm.h" +#include "xe_sriov.h" + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define mocs_dbg xe_gt_dbg +#else +__printf(2, 3) +static inline void mocs_dbg(const struct xe_gt *gt, + const char *format, ...) +{ /* noop */ } +#endif + +enum { + HAS_GLOBAL_MOCS = BIT(0), + HAS_LNCF_MOCS = BIT(1), +}; + +struct xe_mocs_entry { + u32 control_value; + u16 l3cc_value; + u16 used; +}; + +struct xe_mocs_info; + +struct xe_mocs_ops { + void (*dump)(struct xe_mocs_info *mocs, unsigned int flags, + struct xe_gt *gt, struct drm_printer *p); +}; + +struct xe_mocs_info { + /* + * Size of the spec's suggested MOCS programming table. The list of + * table entries from the spec can potentially be smaller than the + * number of hardware registers used to program the MOCS table; in such + * cases the registers for the remaining indices will be programmed to + * match unused_entries_index. + */ + unsigned int table_size; + /* Number of MOCS entries supported by the hardware */ + unsigned int num_mocs_regs; + const struct xe_mocs_entry *table; + const struct xe_mocs_ops *ops; + u8 uc_index; + u8 wb_index; + u8 unused_entries_index; +}; + +/* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */ +#define IG_PAT REG_BIT(8) +#define L3_CACHE_POLICY_MASK REG_GENMASK(5, 4) +#define L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) + +/* Helper defines */ +#define XELP_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ +#define PVC_NUM_MOCS_ENTRIES 3 +#define MTL_NUM_MOCS_ENTRIES 16 +#define XE2_NUM_MOCS_ENTRIES 16 + +/* (e)LLC caching options */ +/* + * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means + * the same as LE_UC + */ +#define LE_0_PAGETABLE LE_CACHEABILITY(0) +#define LE_1_UC LE_CACHEABILITY(1) +#define LE_2_WT LE_CACHEABILITY(2) +#define LE_3_WB LE_CACHEABILITY(3) + +/* Target cache */ +#define LE_TC_0_PAGETABLE LE_TGT_CACHE(0) +#define LE_TC_1_LLC LE_TGT_CACHE(1) +#define LE_TC_2_LLC_ELLC LE_TGT_CACHE(2) +#define LE_TC_3_LLC_ELLC_ALT LE_TGT_CACHE(3) + +/* L3 caching options */ +#define L3_0_DIRECT L3_CACHEABILITY(0) +#define L3_1_UC L3_CACHEABILITY(1) +#define L3_2_RESERVED L3_CACHEABILITY(2) +#define L3_3_WB L3_CACHEABILITY(3) + +/* L4 caching options */ +#define L4_0_WB REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 0) +#define L4_1_WT REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 1) +#define L4_3_UC REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 3) + +#define XE2_L3_0_WB REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 0) +/* XD: WB Transient Display */ +#define XE2_L3_1_XD REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 1) +#define XE2_L3_3_UC REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 3) + +#define XE2_L3_CLOS_MASK REG_GENMASK(7, 6) + +#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ + [__idx] = { \ + .control_value = __control_value, \ + .l3cc_value = __l3cc_value, \ + .used = 1, \ + } + +/* + * MOCS tables + * + * These are the MOCS tables that are programmed across all the rings. + * The control value is programmed to all the rings that support the + * MOCS registers. While the l3cc_values are only programmed to the + * LNCFCMOCS0 - LNCFCMOCS32 registers. + * + * These tables are intended to be kept reasonably consistent across + * HW platforms, and for ICL+, be identical across OSes. To achieve + * that, the list of entries is published as part of bspec. + * + * Entries not part of the following tables are undefined as far as userspace is + * concerned and shouldn't be relied upon. The last few entries are reserved by + * the hardware. They should be initialized according to bspec and never used. + * + * NOTE1: These tables are part of bspec and defined as part of the hardware + * interface. It is expected that, for specific hardware platform, existing + * entries will remain constant and the table will only be updated by adding new + * entries, filling unused positions. + * + * NOTE2: Reserved and unspecified MOCS indices have been set to L3 WB. These + * reserved entries should never be used. They may be changed to low performant + * variants with better coherency in the future if more entries are needed. + */ + +static const struct xe_mocs_entry gen12_mocs_desc[] = { + /* Base - L3 + LLC */ + MOCS_ENTRY(2, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_3_WB), + /* Base - Uncached */ + MOCS_ENTRY(3, + LE_1_UC | LE_TC_1_LLC, + L3_1_UC), + /* Base - L3 */ + MOCS_ENTRY(4, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), + /* Base - LLC */ + MOCS_ENTRY(5, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* Age 0 - LLC */ + MOCS_ENTRY(6, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), + L3_1_UC), + /* Age 0 - L3 + LLC */ + MOCS_ENTRY(7, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), + L3_3_WB), + /* Age: Don't Chg. - LLC */ + MOCS_ENTRY(8, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), + L3_1_UC), + /* Age: Don't Chg. - L3 + LLC */ + MOCS_ENTRY(9, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), + L3_3_WB), + /* No AOM - LLC */ + MOCS_ENTRY(10, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), + L3_1_UC), + /* No AOM - L3 + LLC */ + MOCS_ENTRY(11, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), + L3_3_WB), + /* No AOM; Age 0 - LLC */ + MOCS_ENTRY(12, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), + L3_1_UC), + /* No AOM; Age 0 - L3 + LLC */ + MOCS_ENTRY(13, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), + L3_3_WB), + /* No AOM; Age:DC - LLC */ + MOCS_ENTRY(14, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), + L3_1_UC), + /* No AOM; Age:DC - L3 + LLC */ + MOCS_ENTRY(15, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), + L3_3_WB), + /* Self-Snoop - L3 + LLC */ + MOCS_ENTRY(18, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), + L3_3_WB), + /* Skip Caching - L3 + LLC(12.5%) */ + MOCS_ENTRY(19, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), + L3_3_WB), + /* Skip Caching - L3 + LLC(25%) */ + MOCS_ENTRY(20, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), + L3_3_WB), + /* Skip Caching - L3 + LLC(50%) */ + MOCS_ENTRY(21, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), + L3_3_WB), + /* Skip Caching - L3 + LLC(75%) */ + MOCS_ENTRY(22, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), + L3_3_WB), + /* Skip Caching - L3 + LLC(87.5%) */ + MOCS_ENTRY(23, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ + MOCS_ENTRY(48, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + L3 */ + MOCS_ENTRY(49, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + LLC */ + MOCS_ENTRY(50, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* Implicitly enable L1 - HDC:L1 */ + MOCS_ENTRY(51, + LE_1_UC | LE_TC_1_LLC, + L3_1_UC), + /* HW Special Case (CCS) */ + MOCS_ENTRY(60, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* HW Special Case (Displayable) */ + MOCS_ENTRY(61, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), + /* HW Reserved - SW program but never use */ + MOCS_ENTRY(62, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* HW Reserved - SW program but never use */ + MOCS_ENTRY(63, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC) +}; + +static bool regs_are_mcr(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (xe_gt_is_media_type(gt)) + return MEDIA_VER(xe) >= 20; + else + return GRAPHICS_VERx100(xe) >= 1250; +} + +static void xelp_lncf_dump(struct xe_mocs_info *info, struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int i, j; + u32 reg_val; + + drm_printf(p, "LNCFCMOCS[idx] = [ESC, SCC, L3CC] (value)\n\n"); + + for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i)); + + drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", + j++, + !!(reg_val & L3_ESC_MASK), + REG_FIELD_GET(L3_SCC_MASK, reg_val), + REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), + reg_val); + + drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", + j, + !!(reg_val & L3_UPPER_IDX_ESC_MASK), + REG_FIELD_GET(L3_UPPER_IDX_SCC_MASK, reg_val), + REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), + reg_val); + } +} + +static void xelp_mocs_dump(struct xe_mocs_info *info, unsigned int flags, + struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int i; + u32 reg_val; + + if (flags & HAS_GLOBAL_MOCS) { + drm_printf(p, "Global mocs table configuration:\n"); + drm_printf(p, "GLOB_MOCS[idx] = [LeCC, TC, LRUM, AOM, RSC, SCC, PFM, SCF, CoS, SSE] (value)\n\n"); + + for (i = 0; i < info->num_mocs_regs; i++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); + + drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u ] (%#8x)\n", + i, + REG_FIELD_GET(LE_CACHEABILITY_MASK, reg_val), + REG_FIELD_GET(LE_TGT_CACHE_MASK, reg_val), + REG_FIELD_GET(LE_LRUM_MASK, reg_val), + !!(reg_val & LE_AOM_MASK), + !!(reg_val & LE_RSC_MASK), + REG_FIELD_GET(LE_SCC_MASK, reg_val), + REG_FIELD_GET(LE_PFM_MASK, reg_val), + !!(reg_val & LE_SCF_MASK), + REG_FIELD_GET(LE_COS_MASK, reg_val), + REG_FIELD_GET(LE_SSE_MASK, reg_val), + reg_val); + } + } + + xelp_lncf_dump(info, gt, p); +} + +static const struct xe_mocs_ops xelp_mocs_ops = { + .dump = xelp_mocs_dump, +}; + +static const struct xe_mocs_entry dg1_mocs_desc[] = { + /* UC */ + MOCS_ENTRY(1, 0, L3_1_UC), + /* WB - L3 */ + MOCS_ENTRY(5, 0, L3_3_WB), + /* WB - L3 50% */ + MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB), + /* WB - L3 25% */ + MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB), + /* WB - L3 12.5% */ + MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB), + + /* HDC:L1 + L3 */ + MOCS_ENTRY(48, 0, L3_3_WB), + /* HDC:L1 */ + MOCS_ENTRY(49, 0, L3_1_UC), + + /* HW Reserved */ + MOCS_ENTRY(60, 0, L3_1_UC), + MOCS_ENTRY(61, 0, L3_1_UC), + MOCS_ENTRY(62, 0, L3_1_UC), + MOCS_ENTRY(63, 0, L3_1_UC), +}; + +static const struct xe_mocs_entry dg2_mocs_desc[] = { + /* UC - Coherent; GO:L3 */ + MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), + /* UC - Coherent; GO:Memory */ + MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Non-Coherent; GO:Memory */ + MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), + + /* WB - LC */ + MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), +}; + +static void xehp_lncf_dump(struct xe_mocs_info *info, unsigned int flags, + struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int i, j; + u32 reg_val; + + drm_printf(p, "LNCFCMOCS[idx] = [UCL3LOOKUP, GLBGO, L3CC] (value)\n\n"); + + for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i)); + + drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", + j++, + !!(reg_val & L3_LKUP_MASK), + !!(reg_val & L3_GLBGO_MASK), + REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), + reg_val); + + drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", + j, + !!(reg_val & L3_UPPER_LKUP_MASK), + !!(reg_val & L3_UPPER_GLBGO_MASK), + REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), + reg_val); + } +} + +static const struct xe_mocs_ops xehp_mocs_ops = { + .dump = xehp_lncf_dump, +}; + +static const struct xe_mocs_entry pvc_mocs_desc[] = { + /* Error */ + MOCS_ENTRY(0, 0, L3_3_WB), + + /* UC */ + MOCS_ENTRY(1, 0, L3_1_UC), + + /* WB */ + MOCS_ENTRY(2, 0, L3_3_WB), +}; + +static void pvc_mocs_dump(struct xe_mocs_info *info, unsigned int flags, struct xe_gt *gt, + struct drm_printer *p) +{ + unsigned int i, j; + u32 reg_val; + + drm_printf(p, "LNCFCMOCS[idx] = [ L3CC ] (value)\n\n"); + + for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i)); + + drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", + j++, + REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), + reg_val); + + drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", + j, + REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), + reg_val); + } +} + +static const struct xe_mocs_ops pvc_mocs_ops = { + .dump = pvc_mocs_dump, +}; + +static const struct xe_mocs_entry mtl_mocs_desc[] = { + /* Error - Reserved for Non-Use */ + MOCS_ENTRY(0, + 0, + L3_LKUP(1) | L3_3_WB), + /* Cached - L3 + L4 */ + MOCS_ENTRY(1, + IG_PAT, + L3_LKUP(1) | L3_3_WB), + /* L4 - GO:L3 */ + MOCS_ENTRY(2, + IG_PAT, + L3_LKUP(1) | L3_1_UC), + /* Uncached - GO:L3 */ + MOCS_ENTRY(3, + IG_PAT | L4_3_UC, + L3_LKUP(1) | L3_1_UC), + /* L4 - GO:Mem */ + MOCS_ENTRY(4, + IG_PAT, + L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), + /* Uncached - GO:Mem */ + MOCS_ENTRY(5, + IG_PAT | L4_3_UC, + L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), + /* L4 - L3:NoLKUP; GO:L3 */ + MOCS_ENTRY(6, + IG_PAT, + L3_1_UC), + /* Uncached - L3:NoLKUP; GO:L3 */ + MOCS_ENTRY(7, + IG_PAT | L4_3_UC, + L3_1_UC), + /* L4 - L3:NoLKUP; GO:Mem */ + MOCS_ENTRY(8, + IG_PAT, + L3_GLBGO(1) | L3_1_UC), + /* Uncached - L3:NoLKUP; GO:Mem */ + MOCS_ENTRY(9, + IG_PAT | L4_3_UC, + L3_GLBGO(1) | L3_1_UC), + /* Display - L3; L4:WT */ + MOCS_ENTRY(14, + IG_PAT | L4_1_WT, + L3_LKUP(1) | L3_3_WB), + /* CCS - Non-Displayable */ + MOCS_ENTRY(15, + IG_PAT, + L3_GLBGO(1) | L3_1_UC), +}; + +static void mtl_mocs_dump(struct xe_mocs_info *info, unsigned int flags, + struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int i; + u32 reg_val; + + drm_printf(p, "Global mocs table configuration:\n"); + drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L4_CACHE_POLICY] (value)\n\n"); + + for (i = 0; i < info->num_mocs_regs; i++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); + + drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u] (%#8x)\n", + i, + !!(reg_val & IG_PAT), + REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), + reg_val); + } + + /* MTL lncf mocs table pattern is similar to that of xehp */ + xehp_lncf_dump(info, flags, gt, p); +} + +static const struct xe_mocs_ops mtl_mocs_ops = { + .dump = mtl_mocs_dump, +}; + +static const struct xe_mocs_entry xe2_mocs_table[] = { + /* Defer to PAT */ + MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0), + /* Cached L3, Uncached L4 */ + MOCS_ENTRY(1, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0), + /* Uncached L3, Cached L4 */ + MOCS_ENTRY(2, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0), + /* Uncached L3 + L4 */ + MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0), + /* Cached L3 + L4 */ + MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0), +}; + +static void xe2_mocs_dump(struct xe_mocs_info *info, unsigned int flags, + struct xe_gt *gt, struct drm_printer *p) +{ + unsigned int i; + u32 reg_val; + + drm_printf(p, "Global mocs table configuration:\n"); + drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L3_CLOS, L3_CACHE_POLICY, L4_CACHE_POLICY] (value)\n\n"); + + for (i = 0; i < info->num_mocs_regs; i++) { + if (regs_are_mcr(gt)) + reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); + else + reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); + + drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u] (%#8x)\n", + i, + !!(reg_val & IG_PAT), + REG_FIELD_GET(XE2_L3_CLOS_MASK, reg_val), + REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), + reg_val); + } +} + +static const struct xe_mocs_ops xe2_mocs_ops = { + .dump = xe2_mocs_dump, +}; + +/* + * Note that the "L3" and "L4" register fields actually control the L2 and L3 + * caches respectively on this platform. + */ +static const struct xe_mocs_entry xe3p_xpc_mocs_table[] = { + /* Defer to PAT */ + MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0), + /* UC */ + MOCS_ENTRY(1, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0), + /* L2 */ + MOCS_ENTRY(2, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0), + /* L3 */ + MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0), + /* L2 + L3 */ + MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0), +}; + +static unsigned int get_mocs_settings(struct xe_device *xe, + struct xe_mocs_info *info) +{ + unsigned int flags = 0; + + memset(info, 0, sizeof(struct xe_mocs_info)); + + switch (xe->info.platform) { + case XE_CRESCENTISLAND: + info->ops = &xe2_mocs_ops; + info->table_size = ARRAY_SIZE(xe3p_xpc_mocs_table); + info->table = xe3p_xpc_mocs_table; + info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES; + info->uc_index = 1; + info->wb_index = 4; + info->unused_entries_index = 4; + break; + case XE_NOVALAKE_P: + case XE_NOVALAKE_S: + case XE_PANTHERLAKE: + case XE_LUNARLAKE: + case XE_BATTLEMAGE: + info->ops = &xe2_mocs_ops; + info->table_size = ARRAY_SIZE(xe2_mocs_table); + info->table = xe2_mocs_table; + info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES; + info->uc_index = 3; + info->wb_index = 4; + info->unused_entries_index = 4; + break; + case XE_PVC: + info->ops = &pvc_mocs_ops; + info->table_size = ARRAY_SIZE(pvc_mocs_desc); + info->table = pvc_mocs_desc; + info->num_mocs_regs = PVC_NUM_MOCS_ENTRIES; + info->uc_index = 1; + info->wb_index = 2; + info->unused_entries_index = 2; + break; + case XE_METEORLAKE: + info->ops = &mtl_mocs_ops; + info->table_size = ARRAY_SIZE(mtl_mocs_desc); + info->table = mtl_mocs_desc; + info->num_mocs_regs = MTL_NUM_MOCS_ENTRIES; + info->uc_index = 9; + info->unused_entries_index = 1; + break; + case XE_DG2: + info->ops = &xehp_mocs_ops; + info->table_size = ARRAY_SIZE(dg2_mocs_desc); + info->table = dg2_mocs_desc; + info->uc_index = 1; + /* + * Last entry is RO on hardware, don't bother with what was + * written when checking later + */ + info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES - 1; + info->unused_entries_index = 3; + break; + case XE_DG1: + info->ops = &xelp_mocs_ops; + info->table_size = ARRAY_SIZE(dg1_mocs_desc); + info->table = dg1_mocs_desc; + info->uc_index = 1; + info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES; + info->unused_entries_index = 5; + break; + case XE_TIGERLAKE: + case XE_ROCKETLAKE: + case XE_ALDERLAKE_S: + case XE_ALDERLAKE_P: + case XE_ALDERLAKE_N: + info->ops = &xelp_mocs_ops; + info->table_size = ARRAY_SIZE(gen12_mocs_desc); + info->table = gen12_mocs_desc; + info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES; + info->uc_index = 3; + info->unused_entries_index = 2; + break; + default: + drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n"); + return 0; + } + + /* + * Index 0 is a reserved/unused table entry on most platforms, but + * even on those where it does represent a legitimate MOCS entry, it + * never represents the "most cached, least coherent" behavior we want + * to populate undefined table rows with. So if unused_entries_index + * is still 0 at this point, we'll assume that it was omitted by + * mistake in the switch statement above. + */ + xe_assert(xe, info->unused_entries_index != 0); + + xe_assert(xe, info->ops && info->ops->dump); + xe_assert(xe, info->table_size <= info->num_mocs_regs); + + if (!IS_DGFX(xe) || GRAPHICS_VER(xe) >= 20) + flags |= HAS_GLOBAL_MOCS; + if (GRAPHICS_VER(xe) < 20) + flags |= HAS_LNCF_MOCS; + + return flags; +} + +/* + * Get control_value from MOCS entry. If the table entry is not defined, the + * settings from unused_entries_index will be returned. + */ +static u32 get_entry_control(const struct xe_mocs_info *info, + unsigned int index) +{ + if (index < info->table_size && info->table[index].used) + return info->table[index].control_value; + return info->table[info->unused_entries_index].control_value; +} + +static void __init_mocs_table(struct xe_gt *gt, + const struct xe_mocs_info *info) +{ + unsigned int i; + u32 mocs; + + mocs_dbg(gt, "mocs entries: %d\n", info->num_mocs_regs); + + for (i = 0; i < info->num_mocs_regs; i++) { + mocs = get_entry_control(info, i); + + mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i, + XELP_GLOBAL_MOCS(i).addr, mocs); + + if (regs_are_mcr(gt)) + xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs); + else + xe_mmio_write32(>->mmio, XELP_GLOBAL_MOCS(i), mocs); + } +} + +/* + * Get l3cc_value from MOCS entry taking into account when it's not used + * then if unused_entries_index is not zero then its value will be returned + * otherwise I915_MOCS_PTE's value is returned in this case. + */ +static u16 get_entry_l3cc(const struct xe_mocs_info *info, + unsigned int index) +{ + if (index < info->table_size && info->table[index].used) + return info->table[index].l3cc_value; + return info->table[info->unused_entries_index].l3cc_value; +} + +static u32 l3cc_combine(u16 low, u16 high) +{ + return low | (u32)high << 16; +} + +static void init_l3cc_table(struct xe_gt *gt, + const struct xe_mocs_info *info) +{ + unsigned int i; + u32 l3cc; + + mocs_dbg(gt, "l3cc entries: %d\n", info->num_mocs_regs); + + for (i = 0; i < (info->num_mocs_regs + 1) / 2; i++) { + l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i), + get_entry_l3cc(info, 2 * i + 1)); + + mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i, + XELP_LNCFCMOCS(i).addr, l3cc); + + if (regs_are_mcr(gt)) + xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc); + else + xe_mmio_write32(>->mmio, XELP_LNCFCMOCS(i), l3cc); + } +} + +void xe_mocs_init_early(struct xe_gt *gt) +{ + struct xe_mocs_info table; + + get_mocs_settings(gt_to_xe(gt), &table); + gt->mocs.uc_index = table.uc_index; + gt->mocs.wb_index = table.wb_index; +} + +void xe_mocs_init(struct xe_gt *gt) +{ + struct xe_mocs_info table; + unsigned int flags; + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return; + + /* + * MOCS settings are split between "GLOB_MOCS" and/or "LNCFCMOCS" + * registers depending on platform. + * + * These registers should be programmed before GuC initialization + * since their values will affect some of the memory transactions + * performed by the GuC. + */ + flags = get_mocs_settings(gt_to_xe(gt), &table); + mocs_dbg(gt, "flag:0x%x\n", flags); + + if (IS_SRIOV_VF(gt_to_xe(gt))) + return; + + if (flags & HAS_GLOBAL_MOCS) + __init_mocs_table(gt, &table); + if (flags & HAS_LNCF_MOCS) + init_l3cc_table(gt, &table); +} + +/** + * xe_mocs_dump() - Dump MOCS table. + * @gt: the &xe_gt with MOCS table + * @p: the &drm_printer to dump info to + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + enum xe_force_wake_domains domain; + struct xe_mocs_info table; + unsigned int flags; + + flags = get_mocs_settings(xe, &table); + + domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT; + + guard(xe_pm_runtime_noresume)(xe); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain)) + return -ETIMEDOUT; + + table.ops->dump(&table, flags, gt, p); + + return 0; +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_mocs.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_mocs.h b/drivers/gpu/drm/xe/xe_mocs.h new file mode 100644 index 000000000..f00bbb269 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mocs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_MOCS_H_ +#define _XE_MOCS_H_ + +struct drm_printer; +struct xe_gt; + +void xe_mocs_init_early(struct xe_gt *gt); +void xe_mocs_init(struct xe_gt *gt); +int xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c new file mode 100644 index 000000000..848d65265 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_module.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_module.h" + +#include +#include +#include + +#include + +#include "xe_defaults.h" +#include "xe_device_types.h" +#include "xe_drv.h" +#include "xe_configfs.h" +#include "xe_hw_fence.h" +#include "xe_pci.h" +#include "xe_pm.h" +#include "xe_observation.h" +#include "xe_sched_job.h" + +struct xe_modparam xe_modparam = { + .probe_display = XE_DEFAULT_PROBE_DISPLAY, + .guc_log_level = XE_DEFAULT_GUC_LOG_LEVEL, + .force_probe = XE_DEFAULT_FORCE_PROBE, +#ifdef CONFIG_PCI_IOV + .max_vfs = XE_DEFAULT_MAX_VFS, +#endif + .wedged_mode = XE_DEFAULT_WEDGED_MODE, + .svm_notifier_size = XE_DEFAULT_SVM_NOTIFIER_SIZE, + /* the rest are 0 by default */ +}; + +module_param_named(svm_notifier_size, xe_modparam.svm_notifier_size, uint, 0600); +MODULE_PARM_DESC(svm_notifier_size, "Set the svm notifier size in MiB, must be power of 2 " + "[default=" __stringify(XE_DEFAULT_SVM_NOTIFIER_SIZE) "]"); + +#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) +module_param_named(probe_display, xe_modparam.probe_display, bool, 0444); +MODULE_PARM_DESC(probe_display, "Probe display HW, otherwise it's left untouched " + "[default=" __stringify(XE_DEFAULT_PROBE_DISPLAY) "])"); +#endif + +module_param_named(vram_bar_size, xe_modparam.force_vram_bar_size, int, 0600); +MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size in MiB (<0=disable-resize, 0=max-needed-size, >0=force-size " + "[default=" __stringify(XE_DEFAULT_VRAM_BAR_SIZE) "])"); + +module_param_named(guc_log_level, xe_modparam.guc_log_level, int, 0600); +MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1=normal, 2..5=verbose-levels " + "[default=" __stringify(XE_DEFAULT_GUC_LOG_LEVEL) "])"); + +module_param_named_unsafe(guc_firmware_path, xe_modparam.guc_firmware_path, charp, 0400); +MODULE_PARM_DESC(guc_firmware_path, + "GuC firmware path to use instead of the default one"); + +module_param_named_unsafe(huc_firmware_path, xe_modparam.huc_firmware_path, charp, 0400); +MODULE_PARM_DESC(huc_firmware_path, + "HuC firmware path to use instead of the default one - empty string disables"); + +module_param_named_unsafe(gsc_firmware_path, xe_modparam.gsc_firmware_path, charp, 0400); +MODULE_PARM_DESC(gsc_firmware_path, + "GSC firmware path to use instead of the default one - empty string disables"); + +module_param_named_unsafe(force_probe, xe_modparam.force_probe, charp, 0400); +MODULE_PARM_DESC(force_probe, + "Force probe options for specified devices. See CONFIG_DRM_XE_FORCE_PROBE for details " + "[default=" XE_DEFAULT_FORCE_PROBE "])"); + +#ifdef CONFIG_PCI_IOV +module_param_named(max_vfs, xe_modparam.max_vfs, uint, 0400); +MODULE_PARM_DESC(max_vfs, + "Limit number of Virtual Functions (VFs) that could be managed. " + "(0=no VFs; N=allow up to N VFs " + "[default=" XE_DEFAULT_MAX_VFS_STR "])"); +#endif + +module_param_named_unsafe(wedged_mode, xe_modparam.wedged_mode, uint, 0600); +MODULE_PARM_DESC(wedged_mode, + "Module's default policy for the wedged mode (0=never, 1=upon-critical-error, 2=upon-any-hang-no-reset " + "[default=" XE_DEFAULT_WEDGED_MODE_STR "])"); + +static int xe_check_nomodeset(void) +{ + if (drm_firmware_drivers_only()) + return -ENODEV; + + return 0; +} + +static struct workqueue_struct *xe_destroy_wq; + +static int __init xe_destroy_wq_module_init(void) +{ + xe_destroy_wq = alloc_workqueue("xe-guc-destroy-wq", WQ_UNBOUND, 0); + if (!xe_destroy_wq) + return -ENOMEM; + return 0; +} + +static void xe_destroy_wq_module_exit(void) +{ + if (xe_destroy_wq) + destroy_workqueue(xe_destroy_wq); + xe_destroy_wq = NULL; +} + +/** + * xe_destroy_wq_queue() - Queue work on the destroy workqueue + * @work: work item to queue + * + * The destroy workqueue has module lifetime and is used for GuC exec queue + * teardown that can outlive a single xe_device. SVM pagemap destroy uses the + * per-device xe->destroy_wq instead. + * + * Return: %true if @work was queued, %false if it was already pending. + */ +bool xe_destroy_wq_queue(struct work_struct *work) +{ + return queue_work(xe_destroy_wq, work); +} + +/** + * xe_destroy_wq_flush() - Flush the destroy workqueue + * + * Drains all pending destroy work. Called from PCI remove to ensure + * teardown ordering before the device is destroyed. + */ +void xe_destroy_wq_flush(void) +{ + if (xe_destroy_wq) + flush_workqueue(xe_destroy_wq); +} + +struct init_funcs { + int (*init)(void); + void (*exit)(void); +}; + +static const struct init_funcs init_funcs[] = { + { + .init = xe_check_nomodeset, + }, + { + .init = xe_configfs_init, + .exit = xe_configfs_exit, + }, + { + .init = xe_hw_fence_module_init, + .exit = xe_hw_fence_module_exit, + }, + { + .init = xe_sched_job_module_init, + .exit = xe_sched_job_module_exit, + }, + { + .init = xe_destroy_wq_module_init, + .exit = xe_destroy_wq_module_exit, + }, + { + .init = xe_register_pci_driver, + .exit = xe_unregister_pci_driver, + }, + { + .init = xe_observation_sysctl_register, + .exit = xe_observation_sysctl_unregister, + }, + { + .init = xe_pm_module_init, + }, +}; + +static int __init xe_call_init_func(const struct init_funcs *func) +{ + if (func->init) + return func->init(); + return 0; +} + +static void xe_call_exit_func(const struct init_funcs *func) +{ + if (func->exit) + func->exit(); +} + +static int __init xe_init(void) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(init_funcs); i++) { + err = xe_call_init_func(init_funcs + i); + if (err) { + pr_info("%s: module_init aborted at %ps %pe\n", + DRIVER_NAME, init_funcs[i].init, ERR_PTR(err)); + while (i--) + xe_call_exit_func(init_funcs + i); + return err; + } + } + + return 0; +} + +static void __exit xe_exit(void) +{ + int i; + + for (i = ARRAY_SIZE(init_funcs) - 1; i >= 0; i--) + xe_call_exit_func(init_funcs + i); +} + +module_init(xe_init); +module_exit(xe_exit); + +MODULE_AUTHOR("Intel Corporation"); + +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h new file mode 100644 index 000000000..a0eb7db07 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_module.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_MODULE_H_ +#define _XE_MODULE_H_ + +#include + +struct work_struct; + +/* Module modprobe variables */ +struct xe_modparam { + bool probe_display; + int force_vram_bar_size; + int guc_log_level; + char *guc_firmware_path; + char *huc_firmware_path; + char *gsc_firmware_path; + char *force_probe; +#ifdef CONFIG_PCI_IOV + unsigned int max_vfs; +#endif + unsigned int wedged_mode; + u32 svm_notifier_size; +}; + +extern struct xe_modparam xe_modparam; + +bool xe_destroy_wq_queue(struct work_struct *work); +void xe_destroy_wq_flush(void); + +#endif + diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c new file mode 100644 index 000000000..1ea67eaea --- /dev/null +++ b/drivers/gpu/drm/xe/xe_nvm.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2019-2025, Intel Corporation. All rights reserved. + */ + +#include +#include + +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_nvm.h" +#include "xe_pcode_api.h" +#include "regs/xe_gsc_regs.h" +#include "xe_sriov.h" + +#define GEN12_GUNIT_NVM_BASE 0x00102040 +#define GEN12_DEBUG_NVM_BASE 0x00101018 + +#define GEN12_CNTL_PROTECTED_NVM_REG 0x0010100C + +#define GEN12_GUNIT_NVM_SIZE 0x80 +#define GEN12_DEBUG_NVM_SIZE 0x4 + +#define NVM_NON_POSTED_ERASE_CHICKEN_BIT BIT(13) + +#define HECI_FW_STATUS_2_NVM_ACCESS_MODE BIT(3) + +static const struct intel_dg_nvm_region regions[INTEL_DG_NVM_REGIONS] = { + [0] = { .name = "DESCRIPTOR", }, + [2] = { .name = "GSC", }, + [9] = { .name = "PADDING", }, + [11] = { .name = "OptionROM", }, + [12] = { .name = "DAM", }, +}; + +static void xe_nvm_release_dev(struct device *dev) +{ + struct auxiliary_device *aux = container_of(dev, struct auxiliary_device, dev); + struct intel_dg_nvm_dev *nvm = container_of(aux, struct intel_dg_nvm_dev, aux_dev); + + kfree(nvm); +} + +static bool xe_nvm_non_posted_erase(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + + switch (xe->info.platform) { + case XE_CRESCENTISLAND: + case XE_BATTLEMAGE: + return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) & + NVM_NON_POSTED_ERASE_CHICKEN_BIT); + default: + return false; + } +} + +static bool xe_nvm_writable_override(struct xe_device *xe) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + bool writable_override; + struct xe_reg reg; + u32 test_bit, test_val; + + switch (xe->info.platform) { + case XE_CRESCENTISLAND: + reg = PCODE_SCRATCH(0); + test_bit = FDO_MODE; + test_val = FDO_MODE; + break; + case XE_BATTLEMAGE: + reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; + break; + case XE_PVC: + reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; + break; + case XE_DG2: + reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; + break; + case XE_DG1: + reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; + break; + default: + drm_err(&xe->drm, "Unknown platform\n"); + return true; + } + + writable_override = (xe_mmio_read32(mmio, reg) & test_bit) == test_val; + if (writable_override) + drm_info(&xe->drm, "NVM access overridden by jumper\n"); + return writable_override; +} + +static void xe_nvm_fini(void *arg) +{ + struct xe_device *xe = arg; + struct intel_dg_nvm_dev *nvm = xe->nvm; + + if (!xe->info.has_gsc_nvm) + return; + + /* No access to internal NVM from VFs */ + if (IS_SRIOV_VF(xe)) + return; + + /* Nvm pointer should not be NULL here */ + if (WARN_ON(!nvm)) + return; + + auxiliary_device_delete(&nvm->aux_dev); + auxiliary_device_uninit(&nvm->aux_dev); + xe->nvm = NULL; +} + +int xe_nvm_init(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct auxiliary_device *aux_dev; + struct intel_dg_nvm_dev *nvm; + int ret; + + if (!xe->info.has_gsc_nvm) + return 0; + + /* No access to internal NVM from VFs */ + if (IS_SRIOV_VF(xe)) + return 0; + + /* Nvm pointer should be NULL here */ + if (WARN_ON(xe->nvm)) + return -EFAULT; + + nvm = kzalloc_obj(*nvm); + if (!nvm) + return -ENOMEM; + + nvm->writable_override = xe_nvm_writable_override(xe); + nvm->non_posted_erase = xe_nvm_non_posted_erase(xe); + nvm->bar.parent = &pdev->resource[0]; + nvm->bar.start = GEN12_GUNIT_NVM_BASE + pdev->resource[0].start; + nvm->bar.end = nvm->bar.start + GEN12_GUNIT_NVM_SIZE - 1; + nvm->bar.flags = IORESOURCE_MEM; + nvm->bar.desc = IORES_DESC_NONE; + nvm->regions = regions; + + nvm->bar2.parent = &pdev->resource[0]; + nvm->bar2.start = GEN12_DEBUG_NVM_BASE + pdev->resource[0].start; + nvm->bar2.end = nvm->bar2.start + GEN12_DEBUG_NVM_SIZE - 1; + nvm->bar2.flags = IORESOURCE_MEM; + nvm->bar2.desc = IORES_DESC_NONE; + + aux_dev = &nvm->aux_dev; + + aux_dev->name = "nvm"; + aux_dev->id = (pci_domain_nr(pdev->bus) << 16) | pci_dev_id(pdev); + aux_dev->dev.parent = &pdev->dev; + aux_dev->dev.release = xe_nvm_release_dev; + + ret = auxiliary_device_init(aux_dev); + if (ret) { + drm_err(&xe->drm, "xe-nvm aux init failed %d\n", ret); + kfree(nvm); + return ret; + } + + ret = auxiliary_device_add(aux_dev); + if (ret) { + drm_err(&xe->drm, "xe-nvm aux add failed %d\n", ret); + auxiliary_device_uninit(aux_dev); + return ret; + } + + xe->nvm = nvm; + return devm_add_action_or_reset(xe->drm.dev, xe_nvm_fini, xe); +} diff --git a/drivers/gpu/drm/xe/xe_nvm.h b/drivers/gpu/drm/xe/xe_nvm.h new file mode 100644 index 000000000..b14722103 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_nvm.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2019-2025 Intel Corporation. All rights reserved. + */ + +#ifndef _XE_NVM_H_ +#define _XE_NVM_H_ + +struct xe_device; + +int xe_nvm_init(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c new file mode 100644 index 000000000..9c5384b95 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -0,0 +1,2884 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "abi/guc_actions_slpc_abi.h" +#include "instructions/xe_mi_commands.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_oa_regs.h" +#include "xe_assert.h" +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_guc_rc.h" +#include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_oa.h" +#include "xe_observation.h" +#include "xe_pm.h" +#include "xe_reg_whitelist.h" +#include "xe_sched_job.h" +#include "xe_sriov.h" +#include "xe_sync.h" +#include "xe_wa.h" + +#define DEFAULT_POLL_FREQUENCY_HZ 200 +#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ) +#define XE_OA_UNIT_INVALID U32_MAX + +enum xe_oam_unit_type { + XE_OAM_UNIT_SAG, + XE_OAM_UNIT_SCMI_0, + XE_OAM_UNIT_SCMI_1, +}; + +enum xe_oa_submit_deps { + XE_OA_SUBMIT_NO_DEPS, + XE_OA_SUBMIT_ADD_DEPS, +}; + +enum xe_oa_user_extn_from { + XE_OA_USER_EXTN_FROM_OPEN, + XE_OA_USER_EXTN_FROM_CONFIG, +}; + +struct xe_oa_reg { + struct xe_reg addr; + u32 value; +}; + +struct xe_oa_config { + struct xe_oa *oa; + + char uuid[UUID_STRING_LEN + 1]; + int id; + + const struct xe_oa_reg *regs; + u32 regs_len; + + struct attribute_group sysfs_metric; + struct attribute *attrs[2]; + struct kobj_attribute sysfs_metric_id; + + struct kref ref; + struct rcu_head rcu; +}; + +struct xe_oa_open_param { + struct xe_file *xef; + struct xe_oa_unit *oa_unit; + bool sample; + u32 metric_set; + enum xe_oa_format_name oa_format; + int period_exponent; + bool disabled; + int exec_queue_id; + int engine_instance; + struct xe_exec_queue *exec_q; + struct xe_hw_engine *hwe; + bool no_preempt; + struct drm_xe_sync __user *syncs_user; + int num_syncs; + struct xe_sync_entry *syncs; + size_t oa_buffer_size; + int wait_num_reports; +}; + +struct xe_oa_config_bo { + struct llist_node node; + + struct xe_oa_config *oa_config; + struct xe_bb *bb; +}; + +struct xe_oa_fence { + /* @base: dma fence base */ + struct dma_fence base; + /* @lock: lock for the fence */ + spinlock_t lock; + /* @work: work to signal @base */ + struct delayed_work work; + /* @cb: callback to schedule @work */ + struct dma_fence_cb cb; +}; + +#define DRM_FMT(x) DRM_XE_OA_FMT_TYPE_##x + +static const struct xe_oa_format oa_formats[] = { + [XE_OA_FORMAT_C4_B8] = { 7, 64, DRM_FMT(OAG) }, + [XE_OA_FORMAT_A12] = { 0, 64, DRM_FMT(OAG) }, + [XE_OA_FORMAT_A12_B8_C8] = { 2, 128, DRM_FMT(OAG) }, + [XE_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAG) }, + [XE_OAR_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAR) }, + [XE_OA_FORMAT_A24u40_A14u32_B8_C8] = { 5, 256, DRM_FMT(OAG) }, + [XE_OAC_FORMAT_A24u64_B8_C8] = { 1, 320, DRM_FMT(OAC), HDR_64_BIT }, + [XE_OAC_FORMAT_A22u32_R2u32_B8_C8] = { 2, 192, DRM_FMT(OAC), HDR_64_BIT }, + [XE_OAM_FORMAT_MPEC8u64_B8_C8] = { 1, 192, DRM_FMT(OAM_MPEC), HDR_64_BIT }, + [XE_OAM_FORMAT_MPEC8u32_B8_C8] = { 2, 128, DRM_FMT(OAM_MPEC), HDR_64_BIT }, + [XE_OA_FORMAT_PEC64u64] = { 1, 576, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, + [XE_OA_FORMAT_PEC64u64_B8_C8] = { 1, 640, DRM_FMT(PEC), HDR_64_BIT, 1, 1 }, + [XE_OA_FORMAT_PEC64u32] = { 1, 320, DRM_FMT(PEC), HDR_64_BIT }, + [XE_OA_FORMAT_PEC32u64_G1] = { 5, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, + [XE_OA_FORMAT_PEC32u32_G1] = { 5, 192, DRM_FMT(PEC), HDR_64_BIT }, + [XE_OA_FORMAT_PEC32u64_G2] = { 6, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, + [XE_OA_FORMAT_PEC32u32_G2] = { 6, 192, DRM_FMT(PEC), HDR_64_BIT }, + [XE_OA_FORMAT_PEC36u64_G1_32_G2_4] = { 3, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, + [XE_OA_FORMAT_PEC36u64_G1_4_G2_32] = { 4, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, +}; + +static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head) +{ + return tail >= head ? tail - head : + tail + stream->oa_buffer.circ_size - head; +} + +static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n) +{ + return ptr + n >= stream->oa_buffer.circ_size ? + ptr + n - stream->oa_buffer.circ_size : ptr + n; +} + +static void xe_oa_config_release(struct kref *ref) +{ + struct xe_oa_config *oa_config = + container_of(ref, typeof(*oa_config), ref); + + kfree(oa_config->regs); + + kfree_rcu(oa_config, rcu); +} + +static void xe_oa_config_put(struct xe_oa_config *oa_config) +{ + if (!oa_config) + return; + + kref_put(&oa_config->ref, xe_oa_config_release); +} + +static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config) +{ + return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL; +} + +static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set) +{ + struct xe_oa_config *oa_config; + + rcu_read_lock(); + oa_config = idr_find(&oa->metrics_idr, metrics_set); + if (oa_config) + oa_config = xe_oa_config_get(oa_config); + rcu_read_unlock(); + + return oa_config; +} + +static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo, struct dma_fence *last_fence) +{ + xe_oa_config_put(oa_bo->oa_config); + xe_bb_free(oa_bo->bb, last_fence); + kfree(oa_bo); +} + +static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream) +{ + return &stream->oa_unit->regs; +} + +static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) +{ + return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) & + OAG_OATAILPTR_MASK; +} + +#define oa_report_header_64bit(__s) \ + ((__s)->oa_buffer.format->header == HDR_64_BIT) + +static u64 oa_report_id(struct xe_oa_stream *stream, u32 report_offset) +{ + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + return oa_report_header_64bit(stream) ? + xe_map_rd(stream->oa->xe, map, report_offset, u64) : + xe_map_rd(stream->oa->xe, map, report_offset, u32); +} + +static void oa_report_id_clear(struct xe_oa_stream *stream, u32 report_offset) +{ + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + oa_report_header_64bit(stream) ? + xe_map_wr(stream->oa->xe, map, report_offset, u64, 0) : + xe_map_wr(stream->oa->xe, map, report_offset, u32, 0); +} + +static u64 oa_timestamp(struct xe_oa_stream *stream, u32 report_offset) +{ + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + return oa_report_header_64bit(stream) ? + xe_map_rd(stream->oa->xe, map, report_offset + 8, u64) : + xe_map_rd(stream->oa->xe, map, report_offset + 4, u32); +} + +static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 report_offset) +{ + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + oa_report_header_64bit(stream) ? + xe_map_wr(stream->oa->xe, map, report_offset + 8, u64, 0) : + xe_map_wr(stream->oa->xe, map, report_offset + 4, u32, 0); +} + +static bool mert_wa_14026633728(struct xe_oa_stream *s) +{ + return s->oa_unit->type == DRM_XE_OA_UNIT_TYPE_MERT && XE_DEVICE_WA(s->oa->xe, 14026633728); +} + +static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) +{ + u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); + u32 tail, hw_tail, partial_report_size, available; + int report_size = stream->oa_buffer.format->size; + unsigned long flags; + + spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); + + hw_tail = xe_oa_hw_tail_read(stream); + hw_tail -= gtt_offset; + + /* + * The tail pointer increases in 64 byte (cacheline size), not in report_size + * increments. Also report size may not be a power of 2. Compute potential + * partially landed report in OA buffer. + */ + partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail); + partial_report_size %= report_size; + + /* Subtract partial amount off the tail */ + hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size); + + tail = hw_tail; + + /* + * Walk the stream backward until we find a report with report id and timestamp + * not 0. We can't tell whether a report has fully landed in memory before the + * report id and timestamp of the following report have landed. + * + * This is assuming that the writes of the OA unit land in memory in the order + * they were written. If not : (╯°□°）╯︵ ┻━┻ + */ + while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { + if (oa_report_id(stream, tail) || oa_timestamp(stream, tail)) + break; + + tail = xe_oa_circ_diff(stream, tail, report_size); + } + + if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size) + drm_dbg(&stream->oa->xe->drm, + "unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n", + stream->oa_buffer.head, tail, hw_tail); + + stream->oa_buffer.tail = tail; + + available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head); + stream->pollin = available >= stream->wait_num_reports * report_size; + + spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); + + return stream->pollin; +} + +static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer) +{ + struct xe_oa_stream *stream = + container_of(hrtimer, typeof(*stream), poll_check_timer); + + if (xe_oa_buffer_check_unlocked(stream)) + wake_up(&stream->poll_wq); + + hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns)); + + return HRTIMER_RESTART; +} + +static unsigned long +xe_oa_copy_to_user(struct xe_oa_stream *stream, void __user *dst, u32 report_offset, u32 len) +{ + xe_assert(stream->oa->xe, len <= stream->oa_buffer.format->size); + + xe_map_memcpy_from(stream->oa->xe, stream->oa_buffer.bounce, + &stream->oa_buffer.bo->vmap, report_offset, len); + return copy_to_user(dst, stream->oa_buffer.bounce, len); +} + +static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, + size_t count, size_t *offset, u32 report_offset) +{ + int report_size = stream->oa_buffer.format->size; + int report_size_partial; + + if ((count - *offset) < report_size) + return -ENOSPC; + + buf += *offset; + + report_size_partial = stream->oa_buffer.circ_size - report_offset; + + if (report_size_partial < report_size) { + if (xe_oa_copy_to_user(stream, buf, report_offset, report_size_partial)) + return -EFAULT; + buf += report_size_partial; + + if (xe_oa_copy_to_user(stream, buf, 0, report_size - report_size_partial)) + return -EFAULT; + } else if (xe_oa_copy_to_user(stream, buf, report_offset, report_size)) { + return -EFAULT; + } + + *offset += report_size; + + return 0; +} + +static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, + size_t count, size_t *offset) +{ + int report_size = stream->oa_buffer.format->size; + u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); + size_t start_offset = *offset; + unsigned long flags; + u32 head, tail; + int ret = 0; + + spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); + head = stream->oa_buffer.head; + tail = stream->oa_buffer.tail; + spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); + + xe_assert(stream->oa->xe, + head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size); + + for (; xe_oa_circ_diff(stream, tail, head); + head = xe_oa_circ_incr(stream, head, report_size)) { + ret = xe_oa_append_report(stream, buf, count, offset, head); + if (ret) + break; + + if (!(stream->oa_buffer.circ_size % report_size)) { + /* Clear out report id and timestamp to detect unlanded reports */ + oa_report_id_clear(stream, head); + oa_timestamp_clear(stream, head); + } else { + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + u32 part = stream->oa_buffer.circ_size - head; + + /* Zero out the entire report */ + if (report_size <= part) { + xe_map_memset(stream->oa->xe, map, head, 0, report_size); + } else { + xe_map_memset(stream->oa->xe, map, head, 0, part); + xe_map_memset(stream->oa->xe, map, 0, 0, report_size - part); + } + } + } + + if (start_offset != *offset) { + struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr; + + spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); + xe_mmio_write32(&stream->gt->mmio, oaheadptr, + (head + gtt_offset) & OAG_OAHEADPTR_MASK); + stream->oa_buffer.head = head; + spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); + } + + return ret; +} + +static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) +{ + u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); + int size_exponent = __ffs(xe_bo_size(stream->oa_buffer.bo)); + u32 oa_buf = gtt_offset | OAG_OABUFFER_MEMORY_SELECT; + struct xe_mmio *mmio = &stream->gt->mmio; + unsigned long flags; + + /* + * If oa buffer size is more than 16MB (exponent greater than 24), the + * oa buffer size field is multiplied by 8 in xe_oa_enable_metric_set. + */ + oa_buf |= REG_FIELD_PREP(OABUFFER_SIZE_MASK, + size_exponent > 24 ? size_exponent - 20 : size_exponent - 17); + + spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); + + xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0); + xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr, + gtt_offset & OAG_OAHEADPTR_MASK); + stream->oa_buffer.head = 0; + /* + * PRM says: "This MMIO must be set before the OATAILPTR register and after the + * OAHEADPTR register. This is to enable proper functionality of the overflow bit". + */ + xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf); + xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr, + gtt_offset & OAG_OATAILPTR_MASK); + + /* Mark that we need updated tail pointer to read from */ + stream->oa_buffer.tail = 0; + + spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); + + /* Zero out the OA buffer since we rely on zero report id and timestamp fields */ + xe_map_memset(stream->oa->xe, &stream->oa_buffer.bo->vmap, 0, 0, + xe_bo_size(stream->oa_buffer.bo)); +} + +static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask) +{ + return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) | + REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) | + REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size); +} + +static u32 __oa_ccs_select(struct xe_oa_stream *stream) +{ + u32 val; + + if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE) + return 0; + + val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance); + xe_assert(stream->oa->xe, + REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance); + return val; +} + +static u32 __oactrl_used_bits(struct xe_oa_stream *stream) +{ + return stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ? + OAG_OACONTROL_USED_BITS : OAM_OACONTROL_USED_BITS; +} + +static void xe_oa_enable(struct xe_oa_stream *stream) +{ + const struct xe_oa_format *format = stream->oa_buffer.format; + const struct xe_oa_regs *regs; + u32 val; + + /* + * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA + * buffer must be correctly initialized + */ + xe_oa_init_oa_buffer(stream); + + regs = __oa_regs(stream); + val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) | + __oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE; + + if (GRAPHICS_VER(stream->oa->xe) >= 20 && + stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG) + val |= OAG_OACONTROL_OA_PES_DISAG_EN; + + xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val); +} + +static void xe_oa_disable(struct xe_oa_stream *stream) +{ + struct xe_mmio *mmio = &stream->gt->mmio; + + xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0); + if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl, + OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50000, NULL, false)) + drm_err(&stream->oa->xe->drm, + "wait for OA to be disabled timed out\n"); + + if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) { + /* <= XE_METEORLAKE except XE_PVC */ + xe_mmio_write32(mmio, OA_TLB_INV_CR, 1); + if (xe_mmio_wait32(mmio, OA_TLB_INV_CR, 1, 0, 50000, NULL, false)) + drm_err(&stream->oa->xe->drm, + "wait for OA tlb invalidate timed out\n"); + } +} + +static int xe_oa_wait_unlocked(struct xe_oa_stream *stream) +{ + /* We might wait indefinitely if periodic sampling is not enabled */ + if (!stream->periodic) + return -EINVAL; + + return wait_event_interruptible(stream->poll_wq, + xe_oa_buffer_check_unlocked(stream)); +} + +#define OASTATUS_RELEVANT_BITS (OASTATUS_MMIO_TRG_Q_FULL | OASTATUS_COUNTER_OVERFLOW | \ + OASTATUS_BUFFER_OVERFLOW | OASTATUS_REPORT_LOST) + +static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf, + size_t count, size_t *offset) +{ + /* Only clear our bits to avoid side-effects */ + stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status, + OASTATUS_RELEVANT_BITS, 0); + /* + * Signal to userspace that there is non-zero OA status to read via + * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl + */ + if (stream->oa_status & OASTATUS_RELEVANT_BITS) + return -EIO; + + return xe_oa_append_reports(stream, buf, count, offset); +} + +static ssize_t xe_oa_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct xe_oa_stream *stream = file->private_data; + size_t offset = 0; + int ret; + + if (!stream->sample) + return -EINVAL; + + if (!(file->f_flags & O_NONBLOCK)) { + do { + ret = xe_oa_wait_unlocked(stream); + if (ret) + return ret; + + mutex_lock(&stream->stream_lock); + ret = __xe_oa_read(stream, buf, count, &offset); + mutex_unlock(&stream->stream_lock); + } while (!offset && !ret); + } else { + xe_oa_buffer_check_unlocked(stream); + mutex_lock(&stream->stream_lock); + ret = __xe_oa_read(stream, buf, count, &offset); + mutex_unlock(&stream->stream_lock); + } + + /* + * Typically we clear pollin here in order to wait for the new hrtimer callback + * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC, + * which means that more OA data is available than could fit in the user provided + * buffer. In this case we want the next poll() call to not block. + * + * Also in case of -EIO, we have already waited for data before returning + * -EIO, so need to wait again + */ + if (ret != -ENOSPC && ret != -EIO) + stream->pollin = false; + + /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */ + return offset ?: (ret ?: -EAGAIN); +} + +static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream, + struct file *file, poll_table *wait) +{ + __poll_t events = 0; + + poll_wait(file, &stream->poll_wq, wait); + + /* + * We don't explicitly check whether there's something to read here since this + * path may be hot depending on what else userspace is polling, or on the timeout + * in use. We rely on hrtimer xe_oa_poll_check_timer_cb to notify us when there + * are samples to read + */ + if (stream->pollin) + events |= EPOLLIN; + + return events; +} + +static __poll_t xe_oa_poll(struct file *file, poll_table *wait) +{ + struct xe_oa_stream *stream = file->private_data; + __poll_t ret; + + mutex_lock(&stream->stream_lock); + ret = xe_oa_poll_locked(stream, file, wait); + mutex_unlock(&stream->stream_lock); + + return ret; +} + +static void xe_oa_lock_vma(struct xe_exec_queue *q) +{ + if (q->vm) { + down_read(&q->vm->lock); + xe_vm_lock(q->vm, false); + } +} + +static void xe_oa_unlock_vma(struct xe_exec_queue *q) +{ + if (q->vm) { + xe_vm_unlock(q->vm); + up_read(&q->vm->lock); + } +} + +static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps, + struct xe_bb *bb) +{ + struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q; + struct xe_sched_job *job; + struct dma_fence *fence; + int err = 0; + + xe_oa_lock_vma(q); + + job = xe_bb_create_job(q, bb); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto exit; + } + job->ggtt = true; + + if (deps == XE_OA_SUBMIT_ADD_DEPS) { + for (int i = 0; i < stream->num_syncs && !err; i++) + err = xe_sync_entry_add_deps(&stream->syncs[i], job); + if (err) { + drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err); + goto err_put_job; + } + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + xe_oa_unlock_vma(q); + + return fence; +err_put_job: + xe_sched_job_put(job); +exit: + xe_oa_unlock_vma(q); + return ERR_PTR(err); +} + +static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs) +{ + u32 i; + +#define MI_LOAD_REGISTER_IMM_MAX_REGS (126) + + for (i = 0; i < n_regs; i++) { + if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) { + u32 n_lri = min_t(u32, n_regs - i, + MI_LOAD_REGISTER_IMM_MAX_REGS); + + bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri); + } + bb->cs[bb->len++] = reg_data[i].addr.addr; + bb->cs[bb->len++] = reg_data[i].value; + } +} + +static int num_lri_dwords(int num_regs) +{ + int count = 0; + + if (num_regs > 0) { + count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS); + count += num_regs * 2; + } + + return count; +} + +static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) +{ + xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); + kfree(stream->oa_buffer.bounce); +} + +static void xe_oa_free_configs(struct xe_oa_stream *stream) +{ + struct xe_oa_config_bo *oa_bo, *tmp; + + xe_oa_config_put(stream->oa_config); + llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) + free_oa_config_bo(oa_bo, stream->last_fence); + dma_fence_put(stream->last_fence); +} + +static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri, u32 count) +{ + struct dma_fence *fence; + struct xe_bb *bb; + int err; + + bb = xe_bb_new(stream->gt, 2 * count + 1, false); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + goto exit; + } + + write_cs_mi_lri(bb, reg_lri, count); + + fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto free_bb; + } + xe_bb_free(bb, fence); + dma_fence_put(fence); + + return 0; +free_bb: + xe_bb_free(bb, NULL); +exit: + return err; +} + +static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable) +{ + const struct xe_oa_format *format = stream->oa_buffer.format; + u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) | + (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0); + + struct xe_oa_reg reg_lri[] = { + { + OACTXCONTROL(stream->hwe->mmio_base), + enable ? OA_COUNTER_RESUME : 0, + }, + { + OAR_OACONTROL, + oacontrol, + }, + { + RING_CONTEXT_CONTROL(stream->hwe->mmio_base), + enable ? + REG_MASKED_FIELD_ENABLE(CTX_CTRL_OAC_CONTEXT_ENABLE) : + REG_MASKED_FIELD_DISABLE(CTX_CTRL_OAC_CONTEXT_ENABLE) + }, + }; + + return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); +} + +static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable) +{ + const struct xe_oa_format *format = stream->oa_buffer.format; + u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) | + (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0); + struct xe_oa_reg reg_lri[] = { + { + OACTXCONTROL(stream->hwe->mmio_base), + enable ? OA_COUNTER_RESUME : 0, + }, + { + OAC_OACONTROL, + oacontrol + }, + { + RING_CONTEXT_CONTROL(stream->hwe->mmio_base), + enable ? + REG_MASKED_FIELD_ENABLE(CTX_CTRL_OAC_CONTEXT_ENABLE | CTX_CTRL_RUN_ALONE) : + REG_MASKED_FIELD_DISABLE(CTX_CTRL_OAC_CONTEXT_ENABLE | CTX_CTRL_RUN_ALONE), + }, + }; + + /* Set ccs select to enable programming of OAC_OACONTROL */ + xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl, + __oa_ccs_select(stream)); + + return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); +} + +static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable) +{ + switch (stream->hwe->class) { + case XE_ENGINE_CLASS_RENDER: + return xe_oa_configure_oar_context(stream, enable); + case XE_ENGINE_CLASS_COMPUTE: + return xe_oa_configure_oac_context(stream, enable); + default: + /* Video engines do not support MI_REPORT_PERF_COUNT */ + return 0; + } +} + +#define HAS_OA_BPC_REPORTING(xe) (GRAPHICS_VERx100(xe) >= 1255) + +static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable) +{ + if (enable && stream && stream->sample) + return REG_MASKED_FIELD_DISABLE(OAG_OA_DEBUG_DISABLE_MMIO_TRG); + else + return REG_MASKED_FIELD_ENABLE(OAG_OA_DEBUG_DISABLE_MMIO_TRG); +} + +static void xe_oa_disable_metric_set(struct xe_oa_stream *stream) +{ + struct xe_mmio *mmio = &stream->gt->mmio; + u32 sqcnt1; + + /* Enable thread stall DOP gating and EU DOP gating. */ + if (XE_GT_WA(stream->gt, 1508761755)) { + xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, + REG_MASKED_FIELD_DISABLE(STALL_DOP_GATING_DISABLE)); + xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, + REG_MASKED_FIELD_DISABLE(DISABLE_DOP_GATING)); + } + + xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, + oag_configure_mmio_trigger(stream, false)); + + /* disable the context save/restore or OAR counters */ + if (stream->exec_q) + xe_oa_configure_oa_context(stream, false); + + /* Make sure we disable noa to save power. */ + if (GT_VER(stream->gt) < 35) + xe_mmio_rmw32(mmio, RPM_CONFIG1, GT_NOA_ENABLE, 0); + + sqcnt1 = SQCNT1_PMON_ENABLE | + (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); + + /* Reset PMON Enable to save power. */ + xe_mmio_rmw32(mmio, XELPMP_SQCNT1, sqcnt1, 0); + + if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || + stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && + GRAPHICS_VER(stream->oa->xe) >= 30) + xe_mmio_rmw32(mmio, OAM_COMPRESSION_T3_CONTROL, OAM_LAT_MEASURE_ENABLE, 0); +} + +static void xe_oa_stream_destroy(struct xe_oa_stream *stream) +{ + struct xe_oa_unit *u = stream->oa_unit; + struct xe_gt *gt = stream->hwe->gt; + + if (WARN_ON(stream != u->exclusive_stream)) + return; + + WRITE_ONCE(u->exclusive_stream, NULL); + + mutex_destroy(&stream->stream_lock); + + if (stream->sample) + xe_reg_dewhitelist_oa_regs(stream->gt); + + xe_oa_disable_metric_set(stream); + xe_exec_queue_put(stream->k_exec_q); + + xe_oa_free_oa_buffer(stream); + + xe_force_wake_put(gt_to_fw(gt), stream->fw_ref); + xe_pm_runtime_put(stream->oa->xe); + + xe_oa_free_configs(stream); + xe_file_put(stream->xef); +} + +static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size) +{ + u32 vram = mert_wa_14026633728(stream) ? + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(stream->oa->xe)) : + XE_BO_FLAG_SYSTEM; + struct xe_bo *bo; + + bo = xe_bo_create_pin_map_novm(stream->oa->xe, stream->gt->tile, + size, ttm_bo_type_kernel, + vram | XE_BO_FLAG_GGTT, false); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + stream->oa_buffer.bo = bo; + + stream->oa_buffer.bounce = kmalloc(stream->oa_buffer.format->size, GFP_KERNEL); + if (!stream->oa_buffer.bounce) { + xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); + return -ENOMEM; + } + + return 0; +} + +static struct xe_oa_config_bo * +__xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) +{ + struct xe_oa_config_bo *oa_bo; + size_t config_length; + struct xe_bb *bb; + + oa_bo = kzalloc_obj(*oa_bo); + if (!oa_bo) + return ERR_PTR(-ENOMEM); + + config_length = num_lri_dwords(oa_config->regs_len); + config_length = ALIGN(sizeof(u32) * config_length, XE_PAGE_SIZE) / sizeof(u32); + + bb = xe_bb_new(stream->gt, config_length, false); + if (IS_ERR(bb)) + goto err_free; + + write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len); + + oa_bo->bb = bb; + oa_bo->oa_config = xe_oa_config_get(oa_config); + llist_add(&oa_bo->node, &stream->oa_config_bos); + + return oa_bo; +err_free: + kfree(oa_bo); + return ERR_CAST(bb); +} + +static struct xe_oa_config_bo * +xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) +{ + struct xe_oa_config_bo *oa_bo; + + /* Look for the buffer in the already allocated BOs attached to the stream */ + llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { + if (oa_bo->oa_config == oa_config && + memcmp(oa_bo->oa_config->uuid, oa_config->uuid, + sizeof(oa_config->uuid)) == 0) + goto out; + } + + oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config); +out: + return oa_bo; +} + +static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence) +{ + dma_fence_put(stream->last_fence); + stream->last_fence = dma_fence_get(fence); +} + +static void xe_oa_fence_work_fn(struct work_struct *w) +{ + struct xe_oa_fence *ofence = container_of(w, typeof(*ofence), work.work); + + /* Signal fence to indicate new OA configuration is active */ + dma_fence_signal(&ofence->base); + dma_fence_put(&ofence->base); +} + +static void xe_oa_config_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + /* Additional empirical delay needed for NOA programming after registers are written */ +#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500 + + struct xe_oa_fence *ofence = container_of(cb, typeof(*ofence), cb); + + INIT_DELAYED_WORK(&ofence->work, xe_oa_fence_work_fn); + queue_delayed_work(system_dfl_wq, &ofence->work, + usecs_to_jiffies(NOA_PROGRAM_ADDITIONAL_DELAY_US)); + dma_fence_put(fence); +} + +static const char *xe_oa_get_driver_name(struct dma_fence *fence) +{ + return "xe_oa"; +} + +static const char *xe_oa_get_timeline_name(struct dma_fence *fence) +{ + return "unbound"; +} + +static const struct dma_fence_ops xe_oa_fence_ops = { + .get_driver_name = xe_oa_get_driver_name, + .get_timeline_name = xe_oa_get_timeline_name, +}; + +static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config) +{ +#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500 + struct xe_oa_config_bo *oa_bo; + struct xe_oa_fence *ofence; + int i, err, num_signal = 0; + struct dma_fence *fence; + + ofence = kzalloc_obj(*ofence); + if (!ofence) { + err = -ENOMEM; + goto exit; + } + + oa_bo = xe_oa_alloc_config_buffer(stream, config); + if (IS_ERR(oa_bo)) { + err = PTR_ERR(oa_bo); + goto exit; + } + + /* Emit OA configuration batch */ + fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto exit; + } + + /* Point of no return: initialize and set fence to signal */ + spin_lock_init(&ofence->lock); + dma_fence_init(&ofence->base, &xe_oa_fence_ops, &ofence->lock, 0, 0); + + for (i = 0; i < stream->num_syncs; i++) { + if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL) + num_signal++; + xe_sync_entry_signal(&stream->syncs[i], &ofence->base); + } + + /* Additional dma_fence_get in case we dma_fence_wait */ + if (!num_signal) + dma_fence_get(&ofence->base); + + /* Update last fence too before adding callback */ + xe_oa_update_last_fence(stream, fence); + + /* Add job fence callback to schedule work to signal ofence->base */ + err = dma_fence_add_callback(fence, &ofence->cb, xe_oa_config_cb); + xe_gt_assert(stream->gt, !err || err == -ENOENT); + if (err == -ENOENT) + xe_oa_config_cb(fence, &ofence->cb); + + /* If nothing needs to be signaled we wait synchronously */ + if (!num_signal) { + dma_fence_wait(&ofence->base, false); + dma_fence_put(&ofence->base); + } + + /* Done with syncs */ + for (i = 0; i < stream->num_syncs; i++) + xe_sync_entry_cleanup(&stream->syncs[i]); + kfree(stream->syncs); + + return 0; +exit: + kfree(ofence); + return err; +} + +static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream) +{ + /* If user didn't require OA reports, ask HW not to emit ctx switch reports */ + if (stream->sample) + return REG_MASKED_FIELD_DISABLE(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS); + else + return REG_MASKED_FIELD_ENABLE(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS); +} + +static u32 oag_buf_size_select(const struct xe_oa_stream *stream) +{ + if (xe_bo_size(stream->oa_buffer.bo) > SZ_16M) + return REG_MASKED_FIELD_ENABLE(OAG_OA_DEBUG_BUF_SIZE_SELECT); + else + return REG_MASKED_FIELD_DISABLE(OAG_OA_DEBUG_BUF_SIZE_SELECT); +} + +static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) +{ + struct xe_mmio *mmio = &stream->gt->mmio; + u32 oa_debug, sqcnt1; + int ret; + + /* + * EU NOA signals behave incorrectly if EU clock gating is enabled. + * Disable thread stall DOP gating and EU DOP gating. + */ + if (XE_GT_WA(stream->gt, 1508761755)) { + xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, + REG_MASKED_FIELD_ENABLE(STALL_DOP_GATING_DISABLE)); + xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, + REG_MASKED_FIELD_ENABLE(DISABLE_DOP_GATING)); + } + + /* Disable clk ratio reports */ + oa_debug = OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | + OAG_OA_DEBUG_INCLUDE_CLK_RATIO; + + if (GRAPHICS_VER(stream->oa->xe) >= 20) + oa_debug |= + /* The three bits below are needed to get PEC counters running */ + OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL | + OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL | + OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL; + + xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, + REG_MASKED_FIELD_ENABLE(oa_debug) | + oag_report_ctx_switches(stream) | + oag_buf_size_select(stream) | + oag_configure_mmio_trigger(stream, true)); + + xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, + OAG_OAGLBCTXCTRL_COUNTER_RESUME | + (stream->periodic ? + OAG_OAGLBCTXCTRL_TIMER_ENABLE | + REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK, + stream->period_exponent) : 0)); + + /* + * Initialize Super Queue Internal Cnt Register + * Set PMON Enable in order to collect valid metrics + * Enable bytes per clock reporting + */ + sqcnt1 = SQCNT1_PMON_ENABLE | + (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); + xe_mmio_rmw32(mmio, XELPMP_SQCNT1, 0, sqcnt1); + + if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || + stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && + GRAPHICS_VER(stream->oa->xe) >= 30) + xe_mmio_rmw32(mmio, OAM_COMPRESSION_T3_CONTROL, 0, OAM_LAT_MEASURE_ENABLE); + + /* Configure OAR/OAC */ + if (stream->exec_q) { + ret = xe_oa_configure_oa_context(stream, true); + if (ret) + return ret; + } + + return xe_oa_emit_oa_config(stream, stream->oa_config); +} + +static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name) +{ + u32 counter_size = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE, fmt); + u32 counter_sel = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SEL, fmt); + u32 bc_report = FIELD_GET(DRM_XE_OA_FORMAT_MASK_BC_REPORT, fmt); + u32 type = FIELD_GET(DRM_XE_OA_FORMAT_MASK_FMT_TYPE, fmt); + int idx; + + for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) { + const struct xe_oa_format *f = &oa->oa_formats[idx]; + + if (counter_size == f->counter_size && bc_report == f->bc_report && + type == f->type && counter_sel == f->counter_select) { + *name = idx; + return 0; + } + } + + return -EINVAL; +} + +static struct xe_oa_unit *xe_oa_lookup_oa_unit(struct xe_oa *oa, u32 oa_unit_id) +{ + struct xe_gt *gt; + int gt_id, i; + + for_each_gt(gt, oa->xe, gt_id) { + for (i = 0; i < gt->oa.num_oa_units; i++) { + struct xe_oa_unit *u = >->oa.oa_unit[i]; + + if (u->oa_unit_id == oa_unit_id) + return u; + } + } + + return NULL; +} + +static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->oa_unit = xe_oa_lookup_oa_unit(oa, value); + if (!param->oa_unit) { + drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value); + return -EINVAL; + } + return 0; +} + +static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->sample = value; + return 0; +} + +static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->metric_set = value; + return 0; +} + +static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + int ret = decode_oa_format(oa, value, ¶m->oa_format); + + if (ret) { + drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value); + return ret; + } + return 0; +} + +static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ +#define OA_EXPONENT_MAX 31 + + if (value > OA_EXPONENT_MAX) { + drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX); + return -EINVAL; + } + param->period_exponent = value; + return 0; +} + +static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->disabled = value; + return 0; +} + +static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->exec_queue_id = value; + return 0; +} + +static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->engine_instance = value; + return 0; +} + +static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->no_preempt = value; + return 0; +} + +static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + if (XE_IOCTL_DBG(oa->xe, value > DRM_XE_MAX_SYNCS)) + return -EINVAL; + + param->num_syncs = value; + return 0; +} + +static int xe_oa_set_prop_syncs_user(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + param->syncs_user = u64_to_user_ptr(value); + return 0; +} + +static int xe_oa_set_prop_oa_buffer_size(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + if (!is_power_of_2(value) || value < SZ_128K || value > SZ_128M) { + drm_dbg(&oa->xe->drm, "OA buffer size invalid %llu\n", value); + return -EINVAL; + } + param->oa_buffer_size = value; + return 0; +} + +static int xe_oa_set_prop_wait_num_reports(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + if (!value) { + drm_dbg(&oa->xe->drm, "wait_num_reports %llu\n", value); + return -EINVAL; + } + param->wait_num_reports = value; + return 0; +} + +static int xe_oa_set_prop_ret_inval(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param) +{ + return -EINVAL; +} + +typedef int (*xe_oa_set_property_fn)(struct xe_oa *oa, u64 value, + struct xe_oa_open_param *param); +static const xe_oa_set_property_fn xe_oa_set_property_funcs_open[] = { + [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_oa_unit_id, + [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_sample_oa, + [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set, + [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_oa_format, + [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_oa_exponent, + [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_disabled, + [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_exec_queue_id, + [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_engine_instance, + [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_no_preempt, + [DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs, + [DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user, + [DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_oa_buffer_size, + [DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_wait_num_reports, +}; + +static const xe_oa_set_property_fn xe_oa_set_property_funcs_config[] = { + [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set, + [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs, + [DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user, + [DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_ret_inval, + [DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_ret_inval, +}; + +static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_from from, + u64 extension, struct xe_oa_open_param *param) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(oa->xe, err)) + return -EFAULT; + + BUILD_BUG_ON(ARRAY_SIZE(xe_oa_set_property_funcs_open) != + ARRAY_SIZE(xe_oa_set_property_funcs_config)); + + if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) || + XE_IOCTL_DBG(oa->xe, !ext.property) || XE_IOCTL_DBG(oa->xe, ext.pad)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open)); + + if (from == XE_OA_USER_EXTN_FROM_CONFIG) + return xe_oa_set_property_funcs_config[idx](oa, ext.value, param); + else + return xe_oa_set_property_funcs_open[idx](oa, ext.value, param); +} + +typedef int (*xe_oa_user_extension_fn)(struct xe_oa *oa, enum xe_oa_user_extn_from from, + u64 extension, struct xe_oa_open_param *param); +static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = { + [DRM_XE_OA_EXTENSION_SET_PROPERTY] = xe_oa_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 16 +static int xe_oa_user_extensions(struct xe_oa *oa, enum xe_oa_user_extn_from from, u64 extension, + int ext_number, struct xe_oa_open_param *param) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(oa->xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(oa->xe, ext.pad) || + XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs)); + err = xe_oa_user_extension_funcs[idx](oa, from, extension, param); + if (XE_IOCTL_DBG(oa->xe, err)) + return err; + + if (ext.next_extension) + return xe_oa_user_extensions(oa, from, ext.next_extension, ++ext_number, param); + + return 0; +} + +static int xe_oa_parse_syncs(struct xe_oa *oa, + struct xe_oa_stream *stream, + struct xe_oa_open_param *param) +{ + int ret, num_syncs, num_ufence = 0; + + if (param->num_syncs && !param->syncs_user) { + drm_dbg(&oa->xe->drm, "num_syncs specified without sync array\n"); + ret = -EINVAL; + goto exit; + } + + if (param->num_syncs) { + param->syncs = kzalloc_objs(*param->syncs, param->num_syncs); + if (!param->syncs) { + ret = -ENOMEM; + goto exit; + } + } + + for (num_syncs = 0; num_syncs < param->num_syncs; num_syncs++) { + ret = xe_sync_entry_parse(oa->xe, param->xef, ¶m->syncs[num_syncs], + ¶m->syncs_user[num_syncs], + stream->ufence_syncobj, + ++stream->ufence_timeline_value, 0); + if (ret) + goto err_syncs; + + if (xe_sync_is_ufence(¶m->syncs[num_syncs])) + num_ufence++; + } + + if (XE_IOCTL_DBG(oa->xe, num_ufence > 1)) { + ret = -EINVAL; + goto err_syncs; + } + + return 0; + +err_syncs: + while (num_syncs--) + xe_sync_entry_cleanup(¶m->syncs[num_syncs]); + kfree(param->syncs); +exit: + return ret; +} + +static void xe_oa_stream_enable(struct xe_oa_stream *stream) +{ + stream->pollin = false; + + xe_oa_enable(stream); + + if (stream->sample) + hrtimer_start(&stream->poll_check_timer, + ns_to_ktime(stream->poll_period_ns), + HRTIMER_MODE_REL_PINNED); +} + +static void xe_oa_stream_disable(struct xe_oa_stream *stream) +{ + xe_oa_disable(stream); + + if (stream->sample) + hrtimer_cancel(&stream->poll_check_timer); + + /* Update stream->oa_buffer.tail to allow any final reports to be read */ + if (xe_oa_buffer_check_unlocked(stream)) + wake_up(&stream->poll_wq); +} + +static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) +{ + struct xe_exec_queue *q = stream->exec_q; + int ret1, ret2; + + /* Best effort recovery: try to revert both to original, irrespective of error */ + ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us); + ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us); + if (ret1 || ret2) + goto err; + return 0; +err: + drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2); + return ret1 ?: ret2; +} + +static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream) +{ + struct xe_exec_queue *q = stream->exec_q; + int ret; + + /* Setting values to 0 will disable timeslice and preempt_timeout */ + ret = q->ops->set_timeslice(q, 0); + if (ret) + goto err; + + ret = q->ops->set_preempt_timeout(q, 0); + if (ret) + goto err; + + return 0; +err: + xe_oa_enable_preempt_timeslice(stream); + drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret); + return ret; +} + +static int xe_oa_enable_locked(struct xe_oa_stream *stream) +{ + if (stream->enabled) + return 0; + + if (stream->no_preempt) { + int ret = xe_oa_disable_preempt_timeslice(stream); + + if (ret) + return ret; + } + + xe_oa_stream_enable(stream); + + stream->enabled = true; + return 0; +} + +static int xe_oa_disable_locked(struct xe_oa_stream *stream) +{ + int ret = 0; + + if (!stream->enabled) + return 0; + + xe_oa_stream_disable(stream); + + if (stream->no_preempt) + ret = xe_oa_enable_preempt_timeslice(stream); + + stream->enabled = false; + return ret; +} + +static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) +{ + struct xe_oa_open_param param = {}; + long ret = stream->oa_config->id; + struct xe_oa_config *config; + int err; + + err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, ¶m); + if (err) + return err; + + config = xe_oa_get_oa_config(stream->oa, param.metric_set); + if (!config) + return -ENODEV; + + param.xef = stream->xef; + err = xe_oa_parse_syncs(stream->oa, stream, ¶m); + if (err) + goto err_config_put; + + stream->num_syncs = param.num_syncs; + stream->syncs = param.syncs; + + err = xe_oa_emit_oa_config(stream, config); + if (!err) { + config = xchg(&stream->oa_config, config); + drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", + stream->oa_config->uuid); + } else { + while (param.num_syncs--) + xe_sync_entry_cleanup(¶m.syncs[param.num_syncs]); + kfree(param.syncs); + } + +err_config_put: + xe_oa_config_put(config); + + return err ?: ret; +} + +static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg) +{ + struct drm_xe_oa_stream_status status = {}; + void __user *uaddr = (void __user *)arg; + + /* Map from register to uapi bits */ + if (stream->oa_status & OASTATUS_REPORT_LOST) + status.oa_status |= DRM_XE_OASTATUS_REPORT_LOST; + if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW) + status.oa_status |= DRM_XE_OASTATUS_BUFFER_OVERFLOW; + if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW) + status.oa_status |= DRM_XE_OASTATUS_COUNTER_OVERFLOW; + if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL) + status.oa_status |= DRM_XE_OASTATUS_MMIO_TRG_Q_FULL; + + if (copy_to_user(uaddr, &status, sizeof(status))) + return -EFAULT; + + return 0; +} + +static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg) +{ + struct drm_xe_oa_stream_info info = { .oa_buf_size = xe_bo_size(stream->oa_buffer.bo), }; + void __user *uaddr = (void __user *)arg; + + if (copy_to_user(uaddr, &info, sizeof(info))) + return -EFAULT; + + return 0; +} + +static long xe_oa_ioctl_locked(struct xe_oa_stream *stream, + unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case DRM_XE_OBSERVATION_IOCTL_ENABLE: + return xe_oa_enable_locked(stream); + case DRM_XE_OBSERVATION_IOCTL_DISABLE: + return xe_oa_disable_locked(stream); + case DRM_XE_OBSERVATION_IOCTL_CONFIG: + return xe_oa_config_locked(stream, arg); + case DRM_XE_OBSERVATION_IOCTL_STATUS: + return xe_oa_status_locked(stream, arg); + case DRM_XE_OBSERVATION_IOCTL_INFO: + return xe_oa_info_locked(stream, arg); + } + + return -EINVAL; +} + +static long xe_oa_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct xe_oa_stream *stream = file->private_data; + long ret; + + mutex_lock(&stream->stream_lock); + ret = xe_oa_ioctl_locked(stream, cmd, arg); + mutex_unlock(&stream->stream_lock); + + return ret; +} + +static void xe_oa_destroy_locked(struct xe_oa_stream *stream) +{ + if (stream->enabled) + xe_oa_disable_locked(stream); + + xe_oa_stream_destroy(stream); + + if (stream->exec_q) + xe_exec_queue_put(stream->exec_q); + + drm_syncobj_put(stream->ufence_syncobj); + kfree(stream); +} + +static int xe_oa_release(struct inode *inode, struct file *file) +{ + struct xe_oa_stream *stream = file->private_data; + struct xe_gt *gt = stream->gt; + + xe_pm_runtime_get(gt_to_xe(gt)); + mutex_lock(>->oa.gt_lock); + xe_oa_destroy_locked(stream); + mutex_unlock(>->oa.gt_lock); + xe_pm_runtime_put(gt_to_xe(gt)); + + /* Release the reference the OA stream kept on the driver */ + drm_dev_put(>_to_xe(gt)->drm); + + return 0; +} + +static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct xe_oa_stream *stream = file->private_data; + int ret = xe_observation_paranoid_check(); + struct xe_bo *bo = stream->oa_buffer.bo; + + if (ret) { + drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); + return ret; + } + + /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ + if (vma->vm_end - vma->vm_start != xe_bo_size(bo)) { + drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); + return -EINVAL; + } + + /* + * Only support VM_READ, enforce MAP_PRIVATE by checking for + * VM_MAYSHARE, don't copy the vma on fork + */ + if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) { + drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n"); + return -EINVAL; + } + vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY, + VM_MAYWRITE | VM_MAYEXEC); + + return drm_gem_mmap_obj(&bo->ttm.base, xe_bo_size(bo), vma); +} + +static const struct file_operations xe_oa_fops = { + .owner = THIS_MODULE, + .release = xe_oa_release, + .poll = xe_oa_poll, + .read = xe_oa_read, + .unlocked_ioctl = xe_oa_ioctl, + .mmap = xe_oa_mmap, +}; + +static int xe_oa_stream_init(struct xe_oa_stream *stream, + struct xe_oa_open_param *param) +{ + struct xe_gt *gt = param->hwe->gt; + int ret; + + stream->exec_q = param->exec_q; + stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS; + stream->oa_unit = param->oa_unit; + stream->hwe = param->hwe; + stream->gt = stream->hwe->gt; + stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format]; + + stream->sample = param->sample; + stream->periodic = param->period_exponent >= 0; + stream->period_exponent = param->period_exponent; + stream->no_preempt = param->no_preempt; + stream->wait_num_reports = param->wait_num_reports; + + stream->xef = xe_file_get(param->xef); + stream->num_syncs = param->num_syncs; + stream->syncs = param->syncs; + + /* + * For Xe2+, when overrun mode is enabled, there are no partial reports at the end + * of buffer, making the OA buffer effectively a non-power-of-2 size circular + * buffer whose size, circ_size, is a multiple of the report size + */ + if (GRAPHICS_VER(stream->oa->xe) >= 20 && + stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample) + stream->oa_buffer.circ_size = + param->oa_buffer_size - + param->oa_buffer_size % stream->oa_buffer.format->size; + else + stream->oa_buffer.circ_size = param->oa_buffer_size; + + stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set); + if (!stream->oa_config) { + drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set); + ret = -EINVAL; + goto exit; + } + + /* Take runtime pm ref and forcewake to disable RC6 */ + xe_pm_runtime_get(stream->oa->xe); + stream->fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(stream->fw_ref, XE_FORCEWAKE_ALL)) { + ret = -ETIMEDOUT; + goto err_fw_put; + } + + ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size); + if (ret) + goto err_fw_put; + + stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL, + BIT(stream->hwe->logical_instance), 1, + stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0); + if (IS_ERR(stream->k_exec_q)) { + ret = PTR_ERR(stream->k_exec_q); + drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d", + stream->gt->info.id, stream->hwe->name, ret); + goto err_free_oa_buf; + } + + ret = xe_oa_enable_metric_set(stream); + if (ret) { + drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n"); + goto err_put_k_exec_q; + } + + drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n", + stream->oa_config->uuid); + + WRITE_ONCE(stream->oa_unit->exclusive_stream, stream); + + hrtimer_setup(&stream->poll_check_timer, xe_oa_poll_check_timer_cb, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + init_waitqueue_head(&stream->poll_wq); + + spin_lock_init(&stream->oa_buffer.ptr_lock); + mutex_init(&stream->stream_lock); + + return 0; + +err_put_k_exec_q: + xe_oa_disable_metric_set(stream); + xe_exec_queue_put(stream->k_exec_q); +err_free_oa_buf: + xe_oa_free_oa_buffer(stream); +err_fw_put: + xe_force_wake_put(gt_to_fw(gt), stream->fw_ref); + xe_pm_runtime_put(stream->oa->xe); + xe_oa_free_configs(stream); +exit: + xe_file_put(stream->xef); + return ret; +} + +static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa, + struct xe_oa_open_param *param) +{ + struct xe_oa_stream *stream; + struct drm_syncobj *ufence_syncobj; + int stream_fd; + int ret; + + /* We currently only allow exclusive access */ + if (param->oa_unit->exclusive_stream) { + drm_dbg(&oa->xe->drm, "OA unit already in use\n"); + ret = -EBUSY; + goto exit; + } + + ret = drm_syncobj_create(&ufence_syncobj, DRM_SYNCOBJ_CREATE_SIGNALED, + NULL); + if (ret) + goto exit; + + stream = kzalloc_obj(*stream); + if (!stream) { + ret = -ENOMEM; + goto err_syncobj; + } + stream->ufence_syncobj = ufence_syncobj; + stream->oa = oa; + + ret = xe_oa_parse_syncs(oa, stream, param); + if (ret) + goto err_free; + + ret = xe_oa_stream_init(stream, param); + if (ret) { + while (param->num_syncs--) + xe_sync_entry_cleanup(¶m->syncs[param->num_syncs]); + kfree(param->syncs); + goto err_free; + } + + if (!param->disabled) { + ret = xe_oa_enable_locked(stream); + if (ret) + goto err_destroy; + } + + stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0); + if (stream_fd < 0) { + ret = stream_fd; + goto err_disable; + } + + if (stream->sample) + xe_reg_whitelist_oa_regs(stream->gt); + + /* Hold a reference on the drm device till stream_fd is released */ + drm_dev_get(&stream->oa->xe->drm); + + return stream_fd; +err_disable: + if (!param->disabled) + xe_oa_disable_locked(stream); +err_destroy: + xe_oa_stream_destroy(stream); +err_free: + kfree(stream); +err_syncobj: + drm_syncobj_put(ufence_syncobj); +exit: + return ret; +} + +/** + * xe_oa_timestamp_frequency - Return OA timestamp frequency + * @gt: @xe_gt + * + * OA timestamp frequency = CS timestamp frequency in most platforms. On some + * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such + * cases, return the adjusted CS timestamp frequency to the user. + */ +u32 xe_oa_timestamp_frequency(struct xe_gt *gt) +{ + u32 reg, shift; + + if (XE_GT_WA(gt, 18013179988) || XE_GT_WA(gt, 14015568240)) { + xe_pm_runtime_get(gt_to_xe(gt)); + reg = xe_mmio_read32(>->mmio, RPM_CONFIG0); + xe_pm_runtime_put(gt_to_xe(gt)); + + shift = REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg); + return gt->info.reference_clock << (3 - shift); + } else { + return gt->info.reference_clock; + } +} + +static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent) +{ + u64 nom = (2ULL << exponent) * NSEC_PER_SEC; + u32 den = xe_oa_timestamp_frequency(gt); + + return div_u64(nom + den - 1, den); +} + +static bool oa_unit_supports_oa_format(struct xe_oa *oa, struct xe_oa_open_param *param) +{ + const struct xe_oa_format *f = &oa->oa_formats[param->oa_format]; + + switch (param->oa_unit->type) { + case DRM_XE_OA_UNIT_TYPE_OAG: + return f->type == DRM_XE_OA_FMT_TYPE_OAG || f->type == DRM_XE_OA_FMT_TYPE_OAR || + f->type == DRM_XE_OA_FMT_TYPE_OAC || f->type == DRM_XE_OA_FMT_TYPE_PEC; + case DRM_XE_OA_UNIT_TYPE_MERT: + if (XE_DEVICE_WA(oa->xe, 14026746987)) + return param->oa_format == XE_OAM_FORMAT_MPEC8u32_B8_C8; + fallthrough; + case DRM_XE_OA_UNIT_TYPE_OAM: + case DRM_XE_OA_UNIT_TYPE_OAM_SAG: + return f->type == DRM_XE_OA_FMT_TYPE_OAM || f->type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; + default: + return false; + } +} + +/** + * xe_oa_unit_id - Return OA unit ID for a hardware engine + * @hwe: @xe_hw_engine + * + * Return OA unit ID for a hardware engine when available + */ +u16 xe_oa_unit_id(struct xe_hw_engine *hwe) +{ + return hwe->oa_unit && hwe->oa_unit->num_engines ? + hwe->oa_unit->oa_unit_id : U16_MAX; +} + +/* A hwe must be assigned to stream/oa_unit for batch submissions */ +static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int ret = 0; + + /* When we have an exec_q, get hwe from the exec_q */ + if (param->exec_q) { + param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class, + param->engine_instance, true); + if (!param->hwe || param->hwe->oa_unit != param->oa_unit) + goto err; + goto out; + } + + /* Else just get the first hwe attached to the oa unit */ + for_each_hw_engine(hwe, param->oa_unit->gt, id) { + if (hwe->oa_unit == param->oa_unit) { + param->hwe = hwe; + goto out; + } + } + + /* If we still didn't find a hwe, just get one with a valid oa_unit from the same gt */ + for_each_hw_engine(hwe, param->oa_unit->gt, id) { + if (!hwe->oa_unit) + continue; + + param->hwe = hwe; + goto out; + } +err: + drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n", + param->exec_q ? param->exec_q->class : -1, + param->engine_instance, param->oa_unit->oa_unit_id); + ret = -EINVAL; +out: + return ret; +} + +/** + * xe_oa_stream_open_ioctl - Opens an OA stream + * @dev: @drm_device + * @data: pointer to struct @drm_xe_oa_config + * @file: @drm_file + * + * The functions opens an OA stream. An OA stream, opened with specified + * properties, enables OA counter samples to be collected, either + * periodically (time based sampling), or on request (using OA queries) + */ +int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_oa *oa = &xe->oa; + struct xe_file *xef = to_xe_file(file); + struct xe_oa_open_param param = {}; + const struct xe_oa_format *f; + bool privileged_op = true; + int ret; + + if (!oa->xe) { + drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); + return -ENODEV; + } + + param.xef = xef; + param.period_exponent = -1; + ret = xe_oa_user_extensions(oa, XE_OA_USER_EXTN_FROM_OPEN, data, 0, ¶m); + if (ret) + return ret; + + /* If not provided, OA unit defaults to OA unit 0 as per uapi */ + if (!param.oa_unit) + param.oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0]; + + if (param.exec_queue_id > 0) { + /* An exec_queue is only needed for OAR/OAC functionality on OAG */ + if (XE_IOCTL_DBG(oa->xe, param.oa_unit->type != DRM_XE_OA_UNIT_TYPE_OAG)) + return -EINVAL; + + param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id); + if (XE_IOCTL_DBG(oa->xe, !param.exec_q)) + return -ENOENT; + + if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) { + ret = -EOPNOTSUPP; + goto err_exec_q; + } + } + + /* + * Query based sampling (using MI_REPORT_PERF_COUNT) with OAR/OAC, + * without global stream access, can be an unprivileged operation + */ + if (param.exec_q && !param.sample) + privileged_op = false; + + if (param.no_preempt) { + if (!param.exec_q) { + drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n"); + ret = -EINVAL; + goto err_exec_q; + } + privileged_op = true; + } + + if (privileged_op) { + ret = xe_observation_paranoid_check(); + if (ret) { + drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); + goto err_exec_q; + } + } + + if (!param.exec_q && !param.sample) { + drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n"); + ret = -EINVAL; + goto err_exec_q; + } + + ret = xe_oa_assign_hwe(oa, ¶m); + if (ret) + goto err_exec_q; + + f = &oa->oa_formats[param.oa_format]; + if (!param.oa_format || !f->size || !oa_unit_supports_oa_format(oa, ¶m)) { + drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", + param.oa_format, f->type, f->size, param.hwe->class); + ret = -EINVAL; + goto err_exec_q; + } + + if (param.period_exponent >= 0) { + u64 oa_period, oa_freq_hz; + + /* Requesting samples from OAG buffer is a privileged operation */ + if (!param.sample) { + drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n"); + ret = -EINVAL; + goto err_exec_q; + } + oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent); + oa_freq_hz = div64_u64(NSEC_PER_SEC, oa_period); + drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz); + } + + if (!param.oa_buffer_size) + param.oa_buffer_size = DEFAULT_XE_OA_BUFFER_SIZE; + + if (!param.wait_num_reports) + param.wait_num_reports = 1; + if (param.wait_num_reports > param.oa_buffer_size / f->size) { + drm_dbg(&oa->xe->drm, "wait_num_reports %d\n", param.wait_num_reports); + ret = -EINVAL; + goto err_exec_q; + } + + mutex_lock(¶m.hwe->gt->oa.gt_lock); + ret = xe_oa_stream_open_ioctl_locked(oa, ¶m); + mutex_unlock(¶m.hwe->gt->oa.gt_lock); + if (ret < 0) + goto err_exec_q; + + return ret; + +err_exec_q: + if (param.exec_q) + xe_exec_queue_put(param.exec_q); + return ret; +} + +static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr) +{ + static const struct xe_reg flex_eu_regs[] = { + EU_PERF_CNTL0, + EU_PERF_CNTL1, + EU_PERF_CNTL2, + EU_PERF_CNTL3, + EU_PERF_CNTL4, + EU_PERF_CNTL5, + EU_PERF_CNTL6, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) { + if (flex_eu_regs[i].addr == addr) + return true; + } + return false; +} + +static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table) +{ + while (table->start && table->end) { + if (addr >= table->start && addr <= table->end) + return true; + + table++; + } + + return false; +} + +static const struct xe_mmio_range xehp_oa_b_counters[] = { + { .start = 0xdc48, .end = 0xdc48 }, /* OAA_ENABLE_REG */ + { .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */ + {} +}; + +static const struct xe_mmio_range gen12_oa_b_counters[] = { + { .start = 0x2b2c, .end = 0x2b2c }, /* OAG_OA_PESS */ + { .start = 0xd900, .end = 0xd91c }, /* OAG_OASTARTTRIG[1-8] */ + { .start = 0xd920, .end = 0xd93c }, /* OAG_OAREPORTTRIG1[1-8] */ + { .start = 0xd940, .end = 0xd97c }, /* OAG_CEC[0-7][0-1] */ + { .start = 0xdc00, .end = 0xdc3c }, /* OAG_SCEC[0-7][0-1] */ + { .start = 0xdc40, .end = 0xdc40 }, /* OAG_SPCTR_CNF */ + { .start = 0xdc44, .end = 0xdc44 }, /* OAA_DBG_REG */ + {} +}; + +static const struct xe_mmio_range mtl_oam_b_counters[] = { + { .start = 0x393000, .end = 0x39301c }, /* OAM_STARTTRIG1[1-8] */ + { .start = 0x393020, .end = 0x39303c }, /* OAM_REPORTTRIG1[1-8] */ + { .start = 0x393040, .end = 0x39307c }, /* OAM_CEC[0-7][0-1] */ + { .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */ + {} +}; + +static const struct xe_mmio_range xe2_oa_b_counters[] = { + { .start = 0x393200, .end = 0x39323C }, /* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */ + { .start = 0x394200, .end = 0x39423C }, /* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */ + { .start = 0x394A00, .end = 0x394A3C }, /* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */ + {}, +}; + +static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr) +{ + return xe_oa_reg_in_range_table(addr, xehp_oa_b_counters) || + xe_oa_reg_in_range_table(addr, gen12_oa_b_counters) || + xe_oa_reg_in_range_table(addr, mtl_oam_b_counters) || + (GRAPHICS_VER(oa->xe) >= 20 && + xe_oa_reg_in_range_table(addr, xe2_oa_b_counters)); +} + +static const struct xe_mmio_range mtl_oa_mux_regs[] = { + { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ + { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ + { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ + { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ + { .start = 0x38d100, .end = 0x38d114}, /* VISACTL */ + {} +}; + +static const struct xe_mmio_range gen12_oa_mux_regs[] = { + { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ + { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ + { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ + { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ + { .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */ + {} +}; + +static const struct xe_mmio_range xe2_oa_mux_regs[] = { + { .start = 0x5194, .end = 0x5194 }, /* SYS_MEM_LAT_MEASURE_MERTF_GRP_3D */ + { .start = 0x8704, .end = 0x8704 }, /* LMEM_LAT_MEASURE_MCFG_GRP */ + { .start = 0xB01C, .end = 0xB01C }, /* LNCF_MISC_CONFIG_REGISTER0 */ + { .start = 0xB1BC, .end = 0xB1BC }, /* L3_BANK_LAT_MEASURE_LBCF_GFX */ + { .start = 0xD0E0, .end = 0xD0F4 }, /* VISACTL */ + { .start = 0xE18C, .end = 0xE18C }, /* SAMPLER_MODE */ + { .start = 0xE590, .end = 0xE590 }, /* TDL_LSC_LAT_MEASURE_TDL_GFX */ + { .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */ + { .start = 0x145194, .end = 0x145194 }, /* SYS_MEM_LAT_MEASURE */ + { .start = 0x145340, .end = 0x14537C }, /* MERTSS_PES_0 - MERTSS_PES_7 */ + {}, +}; + +static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) +{ + if (GRAPHICS_VER(oa->xe) >= 20) + return xe_oa_reg_in_range_table(addr, xe2_oa_mux_regs); + else if (GRAPHICS_VERx100(oa->xe) >= 1270) + return xe_oa_reg_in_range_table(addr, mtl_oa_mux_regs); + else + return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs); +} + +static bool xe_oa_is_valid_config_reg(struct xe_oa *oa, u32 addr, u32 val) +{ + if (XE_DEVICE_WA(oa->xe, 14026779378) && + addr == SYS_MEM_LAT_MEASURE.addr && val & SYS_MEM_LAT_MEASURE_EN) + return false; + + return xe_oa_is_valid_flex_addr(oa, addr) || + xe_oa_is_valid_b_counter_addr(oa, addr) || + xe_oa_is_valid_mux_addr(oa, addr); +} + +static struct xe_oa_reg * +xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr, u32 val), + u32 __user *regs, u32 n_regs) +{ + struct xe_oa_reg *oa_regs; + int err; + u32 i; + + oa_regs = kmalloc_objs(*oa_regs, n_regs); + if (!oa_regs) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < n_regs; i++) { + u32 addr, value; + + err = get_user(addr, regs); + if (err) + goto addr_err; + + err = get_user(value, regs + 1); + if (err) + goto addr_err; + + if (!is_valid(oa, addr, value)) { + drm_dbg(&oa->xe->drm, "Invalid oa_reg addr/value: %#x %#x\n", addr, value); + err = -EINVAL; + goto addr_err; + } + + oa_regs[i].addr = XE_REG(addr); + oa_regs[i].value = value; + + regs += 2; + } + + return oa_regs; + +addr_err: + kfree(oa_regs); + return ERR_PTR(err); +} +ALLOW_ERROR_INJECTION(xe_oa_alloc_regs, ERRNO); + +static ssize_t show_dynamic_id(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct xe_oa_config *oa_config = + container_of(attr, typeof(*oa_config), sysfs_metric_id); + + return sysfs_emit(buf, "%d\n", oa_config->id); +} + +static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa, + struct xe_oa_config *oa_config) +{ + sysfs_attr_init(&oa_config->sysfs_metric_id.attr); + oa_config->sysfs_metric_id.attr.name = "id"; + oa_config->sysfs_metric_id.attr.mode = 0444; + oa_config->sysfs_metric_id.show = show_dynamic_id; + oa_config->sysfs_metric_id.store = NULL; + + oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; + oa_config->attrs[1] = NULL; + + oa_config->sysfs_metric.name = oa_config->uuid; + oa_config->sysfs_metric.attrs = oa_config->attrs; + + return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric); +} + +/** + * xe_oa_add_config_ioctl - Adds one OA config + * @dev: @drm_device + * @data: pointer to struct @drm_xe_oa_config + * @file: @drm_file + * + * The functions adds an OA config to the set of OA configs maintained in + * the kernel. The config determines which OA metrics are collected for an + * OA stream. + */ +int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_oa *oa = &xe->oa; + struct drm_xe_oa_config param; + struct drm_xe_oa_config *arg = ¶m; + struct xe_oa_config *oa_config, *tmp; + struct xe_oa_reg *regs; + int err, id; + + if (!oa->xe) { + drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); + return -ENODEV; + } + + err = xe_observation_paranoid_check(); + if (err) { + drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n"); + return err; + } + + err = copy_from_user(¶m, u64_to_user_ptr(data), sizeof(param)); + if (XE_IOCTL_DBG(oa->xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(oa->xe, arg->extensions) || + XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) || + XE_IOCTL_DBG(oa->xe, !arg->n_regs)) + return -EINVAL; + + oa_config = kzalloc_obj(*oa_config); + if (!oa_config) + return -ENOMEM; + + oa_config->oa = oa; + kref_init(&oa_config->ref); + + if (!uuid_is_valid(arg->uuid)) { + drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n"); + err = -EINVAL; + goto reg_err; + } + + /* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */ + memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); + + oa_config->regs_len = arg->n_regs; + regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg, + u64_to_user_ptr(arg->regs_ptr), + arg->n_regs); + if (IS_ERR(regs)) { + drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n"); + err = PTR_ERR(regs); + goto reg_err; + } + oa_config->regs = regs; + + err = mutex_lock_interruptible(&oa->metrics_lock); + if (err) + goto reg_err; + + /* We shouldn't have too many configs, so this iteration shouldn't be too costly */ + idr_for_each_entry(&oa->metrics_idr, tmp, id) { + if (!strcmp(tmp->uuid, oa_config->uuid)) { + drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n"); + err = -EADDRINUSE; + goto sysfs_err; + } + } + + err = create_dynamic_oa_sysfs_entry(oa, oa_config); + if (err) { + drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); + goto sysfs_err; + } + + oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL); + if (oa_config->id < 0) { + drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); + err = oa_config->id; + goto sysfs_err; + } + + id = oa_config->id; + + drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, id); + + mutex_unlock(&oa->metrics_lock); + + return id; + +sysfs_err: + mutex_unlock(&oa->metrics_lock); +reg_err: + xe_oa_config_put(oa_config); + drm_dbg(&oa->xe->drm, "Failed to add new OA config\n"); + return err; +} + +/** + * xe_oa_remove_config_ioctl - Removes one OA config + * @dev: @drm_device + * @data: pointer to struct @drm_xe_observation_param + * @file: @drm_file + */ +int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_oa *oa = &xe->oa; + struct xe_oa_config *oa_config; + u64 arg, *ptr = u64_to_user_ptr(data); + int ret; + + if (!oa->xe) { + drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); + return -ENODEV; + } + + ret = xe_observation_paranoid_check(); + if (ret) { + drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n"); + return ret; + } + + ret = get_user(arg, ptr); + if (XE_IOCTL_DBG(oa->xe, ret)) + return ret; + + ret = mutex_lock_interruptible(&oa->metrics_lock); + if (ret) + return ret; + + oa_config = idr_find(&oa->metrics_idr, arg); + if (!oa_config) { + drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n"); + ret = -ENOENT; + goto err_unlock; + } + + WARN_ON(arg != oa_config->id); + + sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric); + idr_remove(&oa->metrics_idr, arg); + + mutex_unlock(&oa->metrics_lock); + + drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id); + + xe_oa_config_put(oa_config); + + return 0; + +err_unlock: + mutex_unlock(&oa->metrics_lock); + return ret; +} + +static void xe_oa_unregister(void *arg) +{ + struct xe_oa *oa = arg; + + if (!oa->metrics_kobj) + return; + + kobject_put(oa->metrics_kobj); + oa->metrics_kobj = NULL; +} + +/** + * xe_oa_register - Xe OA registration + * @xe: @xe_device + * + * Exposes the metrics sysfs directory upon completion of module initialization + */ +int xe_oa_register(struct xe_device *xe) +{ + struct xe_oa *oa = &xe->oa; + + if (!oa->xe) + return 0; + + oa->metrics_kobj = kobject_create_and_add("metrics", + &xe->drm.primary->kdev->kobj); + if (!oa->metrics_kobj) + return -ENOMEM; + + return devm_add_action_or_reset(xe->drm.dev, xe_oa_unregister, oa); +} + +static u32 num_oa_units_per_gt(struct xe_gt *gt) +{ + if (xe_gt_is_main_type(gt) || GRAPHICS_VER(gt_to_xe(gt)) < 20) + /* + * Mert OA unit belongs to the SoC, not a gt, so should be accessed using + * xe_root_tile_mmio(). However, for all known platforms this is the same as + * accessing via xe_root_mmio_gt()->mmio. + */ + return xe_device_has_mert(gt_to_xe(gt)) ? 2 : 1; + else if (!IS_DGFX(gt_to_xe(gt))) + return XE_OAM_UNIT_SCMI_0 + 1; /* SAG + SCMI_0 */ + else + return XE_OAM_UNIT_SCMI_1 + 1; /* SAG + SCMI_0 + SCMI_1 */ +} + +static u32 __hwe_oam_unit(struct xe_hw_engine *hwe) +{ + if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) < 1270) + return XE_OA_UNIT_INVALID; + + xe_gt_WARN_ON(hwe->gt, xe_gt_is_main_type(hwe->gt)); + + if (GRAPHICS_VER(gt_to_xe(hwe->gt)) < 20) + return 0; + /* + * XE_OAM_UNIT_SAG has only GSCCS attached to it, but only on some platforms. Also + * GSCCS cannot be used to submit batches to program the OAM unit. Therefore we don't + * assign an OA unit to GSCCS. This means that XE_OAM_UNIT_SAG is exposed as an OA + * unit without attached engines. Fused off engines can also result in oa_unit's with + * num_engines == 0. OA streams can be opened on all OA units. + */ + else if (hwe->engine_id == XE_HW_ENGINE_GSCCS0) + return XE_OA_UNIT_INVALID; + else if (!IS_DGFX(gt_to_xe(hwe->gt))) + return XE_OAM_UNIT_SCMI_0; + else if (hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE) + return (hwe->instance / 2 & 0x1) + 1; + else if (hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE) + return (hwe->instance & 0x1) + 1; + + return XE_OA_UNIT_INVALID; +} + +static u32 __hwe_oa_unit(struct xe_hw_engine *hwe) +{ + switch (hwe->class) { + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + return 0; + + case XE_ENGINE_CLASS_VIDEO_DECODE: + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + case XE_ENGINE_CLASS_OTHER: + return __hwe_oam_unit(hwe); + + default: + return XE_OA_UNIT_INVALID; + } +} + +static struct xe_oa_regs __oam_regs(u32 base) +{ + return (struct xe_oa_regs) { + .base = base, + .oa_head_ptr = OAM_HEAD_POINTER(base), + .oa_tail_ptr = OAM_TAIL_POINTER(base), + .oa_buffer = OAM_BUFFER(base), + .oa_ctx_ctrl = OAM_CONTEXT_CONTROL(base), + .oa_ctrl = OAM_CONTROL(base), + .oa_debug = OAM_DEBUG(base), + .oa_status = OAM_STATUS(base), + .oa_mmio_trg = OAM_MMIO_TRG(base), + .oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK, + }; +} + +static struct xe_oa_regs __oag_regs(void) +{ + return (struct xe_oa_regs) { + .base = 0, + .oa_head_ptr = OAG_OAHEADPTR, + .oa_tail_ptr = OAG_OATAILPTR, + .oa_buffer = OAG_OABUFFER, + .oa_ctx_ctrl = OAG_OAGLBCTXCTRL, + .oa_ctrl = OAG_OACONTROL, + .oa_debug = OAG_OA_DEBUG, + .oa_status = OAG_OASTATUS, + .oa_mmio_trg = OAG_MMIOTRIGGER, + .oa_ctrl_counter_select_mask = OAG_OACONTROL_OA_COUNTER_SEL_MASK, + }; +} + +static struct xe_oa_regs __oamert_regs(void) +{ + return (struct xe_oa_regs) { + .base = 0, + .oa_head_ptr = OAMERT_HEAD_POINTER, + .oa_tail_ptr = OAMERT_TAIL_POINTER, + .oa_buffer = OAMERT_BUFFER, + .oa_ctx_ctrl = OAMERT_CONTEXT_CONTROL, + .oa_ctrl = OAMERT_CONTROL, + .oa_debug = OAMERT_DEBUG, + .oa_status = OAMERT_STATUS, + .oa_mmio_trg = OAMERT_MMIO_TRG, + .oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK, + }; +} + +static void __xe_oa_init_oa_units(struct xe_gt *gt) +{ + const u32 oam_base_addr[] = { + [XE_OAM_UNIT_SAG] = XE_OAM_SAG_BASE, + [XE_OAM_UNIT_SCMI_0] = XE_OAM_SCMI_0_BASE, + [XE_OAM_UNIT_SCMI_1] = XE_OAM_SCMI_1_BASE, + }; + int i, num_units = gt->oa.num_oa_units; + + for (i = 0; i < num_units; i++) { + struct xe_oa_unit *u = >->oa.oa_unit[i]; + + if (xe_gt_is_main_type(gt)) { + if (!i) { + u->regs = __oag_regs(); + u->type = DRM_XE_OA_UNIT_TYPE_OAG; + } else { + xe_gt_assert(gt, xe_device_has_mert(gt_to_xe(gt))); + xe_gt_assert(gt, gt == xe_root_mmio_gt(gt_to_xe(gt))); + u->regs = __oamert_regs(); + u->type = DRM_XE_OA_UNIT_TYPE_MERT; + } + } else { + xe_gt_assert(gt, GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270); + u->regs = __oam_regs(oam_base_addr[i]); + u->type = i == XE_OAM_UNIT_SAG && GRAPHICS_VER(gt_to_xe(gt)) >= 20 ? + DRM_XE_OA_UNIT_TYPE_OAM_SAG : DRM_XE_OA_UNIT_TYPE_OAM; + } + + u->gt = gt; + + xe_mmio_write32(>->mmio, u->regs.oa_ctrl, 0); + + /* Ensure MMIO trigger remains disabled till there is a stream */ + xe_mmio_write32(>->mmio, u->regs.oa_debug, + oag_configure_mmio_trigger(NULL, false)); + + /* Set oa_unit_ids now to ensure ids remain contiguous */ + u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++; + } +} + +static int xe_oa_init_gt(struct xe_gt *gt) +{ + u32 num_oa_units = num_oa_units_per_gt(gt); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_oa_unit *u; + + u = drmm_kcalloc(>_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL); + if (!u) + return -ENOMEM; + + for_each_hw_engine(hwe, gt, id) { + u32 index = __hwe_oa_unit(hwe); + + hwe->oa_unit = NULL; + if (index < num_oa_units) { + u[index].num_engines++; + hwe->oa_unit = &u[index]; + } + } + + gt->oa.num_oa_units = num_oa_units; + gt->oa.oa_unit = u; + + __xe_oa_init_oa_units(gt); + + return drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock); +} + +static void xe_oa_print_gt_oa_units(struct xe_gt *gt) +{ + enum xe_hw_engine_id hwe_id; + struct xe_hw_engine *hwe; + struct xe_oa_unit *u; + char buf[256]; + int i, n; + + for (i = 0; i < gt->oa.num_oa_units; i++) { + u = >->oa.oa_unit[i]; + buf[0] = '\0'; + n = 0; + + for_each_hw_engine(hwe, gt, hwe_id) + if (xe_oa_unit_id(hwe) == u->oa_unit_id) + n += scnprintf(buf + n, sizeof(buf) - n, "%s ", hwe->name); + + xe_gt_dbg(gt, "oa_unit %d, type %d, Engines: %s\n", u->oa_unit_id, u->type, buf); + } +} + +static void xe_oa_print_oa_units(struct xe_oa *oa) +{ + struct xe_gt *gt; + int gt_id; + + for_each_gt(gt, oa->xe, gt_id) + xe_oa_print_gt_oa_units(gt); +} + +static int xe_oa_init_oa_units(struct xe_oa *oa) +{ + struct xe_gt *gt; + int i, ret; + + /* Needed for OAM implementation here */ + BUILD_BUG_ON(XE_OAM_UNIT_SAG != 0); + BUILD_BUG_ON(XE_OAM_UNIT_SCMI_0 != 1); + BUILD_BUG_ON(XE_OAM_UNIT_SCMI_1 != 2); + + for_each_gt(gt, oa->xe, i) { + ret = xe_oa_init_gt(gt); + if (ret) + return ret; + } + + xe_oa_print_oa_units(oa); + + return 0; +} + +static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format) +{ + __set_bit(format, oa->format_mask); +} + +static void xe_oa_init_supported_formats(struct xe_oa *oa) +{ + if (GRAPHICS_VER(oa->xe) >= 20) { + /* Xe2+ */ + oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8); + oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8); + oa_format_add(oa, XE_OA_FORMAT_PEC64u64); + oa_format_add(oa, XE_OA_FORMAT_PEC64u64_B8_C8); + oa_format_add(oa, XE_OA_FORMAT_PEC64u32); + oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G1); + oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G1); + oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G2); + oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G2); + oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_32_G2_4); + oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_4_G2_32); + } else if (GRAPHICS_VERx100(oa->xe) >= 1270) { + /* XE_METEORLAKE */ + oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8); + oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8); + oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8); + oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8); + oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8); + oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8); + } else if (GRAPHICS_VERx100(oa->xe) >= 1255) { + /* XE_DG2, XE_PVC */ + oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8); + oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8); + oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8); + oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8); + } else { + /* Gen12+ */ + xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12); + oa_format_add(oa, XE_OA_FORMAT_A12); + oa_format_add(oa, XE_OA_FORMAT_A12_B8_C8); + oa_format_add(oa, XE_OA_FORMAT_A32u40_A4u32_B8_C8); + oa_format_add(oa, XE_OA_FORMAT_C4_B8); + } +} + +static int destroy_config(int id, void *p, void *data) +{ + xe_oa_config_put(p); + + return 0; +} + +static void xe_oa_fini(void *arg) +{ + struct xe_device *xe = arg; + struct xe_oa *oa = &xe->oa; + + if (!oa->xe) + return; + + idr_for_each(&oa->metrics_idr, destroy_config, oa); + idr_destroy(&oa->metrics_idr); + + oa->xe = NULL; +} + +/** + * xe_oa_init - OA initialization during device probe + * @xe: @xe_device + * + * Return: 0 on success or a negative error code on failure + */ +int xe_oa_init(struct xe_device *xe) +{ + struct xe_oa *oa = &xe->oa; + int ret; + + /* Support OA only with GuC submission and Gen12+ */ + if (!xe_device_uc_enabled(xe) || GRAPHICS_VER(xe) < 12) + return 0; + + if (IS_SRIOV_VF(xe)) + return 0; + + oa->xe = xe; + oa->oa_formats = oa_formats; + + ret = drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock); + if (ret) + goto exit; + + idr_init_base(&oa->metrics_idr, 1); + + ret = xe_oa_init_oa_units(oa); + if (ret) { + drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret)); + goto exit; + } + + xe_oa_init_supported_formats(oa); + + return devm_add_action_or_reset(xe->drm.dev, xe_oa_fini, xe); + +exit: + oa->xe = NULL; + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_oa.h b/drivers/gpu/drm/xe/xe_oa.h new file mode 100644 index 000000000..e510826f9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_oa.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_OA_H_ +#define _XE_OA_H_ + +#include "xe_oa_types.h" + +struct drm_device; +struct drm_file; +struct xe_device; +struct xe_gt; +struct xe_hw_engine; + +int xe_oa_init(struct xe_device *xe); +int xe_oa_register(struct xe_device *xe); +int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file); +int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file); +int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file); +u32 xe_oa_timestamp_frequency(struct xe_gt *gt); +u16 xe_oa_unit_id(struct xe_hw_engine *hwe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h new file mode 100644 index 000000000..e876e9be9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_oa_types.h @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_OA_TYPES_H_ +#define _XE_OA_TYPES_H_ + +#include +#include +#include +#include + +#include +#include "regs/xe_reg_defs.h" +#include "xe_hw_engine_types.h" + +struct drm_syncobj; + +#define DEFAULT_XE_OA_BUFFER_SIZE SZ_16M + +enum xe_oa_report_header { + HDR_32_BIT = 0, + HDR_64_BIT, +}; + +enum xe_oa_format_name { + XE_OA_FORMAT_C4_B8, + + /* Gen8+ */ + XE_OA_FORMAT_A12, + XE_OA_FORMAT_A12_B8_C8, + XE_OA_FORMAT_A32u40_A4u32_B8_C8, + + /* DG2 */ + XE_OAR_FORMAT_A32u40_A4u32_B8_C8, + XE_OA_FORMAT_A24u40_A14u32_B8_C8, + + /* DG2/MTL OAC */ + XE_OAC_FORMAT_A24u64_B8_C8, + XE_OAC_FORMAT_A22u32_R2u32_B8_C8, + + /* MTL OAM */ + XE_OAM_FORMAT_MPEC8u64_B8_C8, + XE_OAM_FORMAT_MPEC8u32_B8_C8, + + /* Xe2+ */ + XE_OA_FORMAT_PEC64u64, + XE_OA_FORMAT_PEC64u64_B8_C8, + XE_OA_FORMAT_PEC64u32, + XE_OA_FORMAT_PEC32u64_G1, + XE_OA_FORMAT_PEC32u32_G1, + XE_OA_FORMAT_PEC32u64_G2, + XE_OA_FORMAT_PEC32u32_G2, + XE_OA_FORMAT_PEC36u64_G1_32_G2_4, + XE_OA_FORMAT_PEC36u64_G1_4_G2_32, + + __XE_OA_FORMAT_MAX, +}; + +/** + * struct xe_oa_format - Format fields for supported OA formats. OA format + * properties are specified in PRM/Bspec 52198 and 60942 + */ +struct xe_oa_format { + /** @counter_select: counter select value (see Bspec 52198/60942) */ + u32 counter_select; + /** @size: record size as written by HW (multiple of 64 byte cachelines) */ + int size; + /** @type: of enum drm_xe_oa_format_type */ + int type; + /** @header: 32 or 64 bit report headers */ + enum xe_oa_report_header header; + /** @counter_size: counter size value (see Bspec 60942) */ + u16 counter_size; + /** @bc_report: BC report value (see Bspec 60942) */ + u16 bc_report; +}; + +/** struct xe_oa_regs - Registers for each OA unit */ +struct xe_oa_regs { + u32 base; + struct xe_reg oa_head_ptr; + struct xe_reg oa_tail_ptr; + struct xe_reg oa_buffer; + struct xe_reg oa_ctx_ctrl; + struct xe_reg oa_ctrl; + struct xe_reg oa_debug; + struct xe_reg oa_status; + struct xe_reg oa_mmio_trg; + u32 oa_ctrl_counter_select_mask; +}; + +/** + * struct xe_oa_unit - Hardware OA unit + */ +struct xe_oa_unit { + /** @oa_unit_id: identifier for the OA unit */ + u16 oa_unit_id; + + /** @gt: gt associated with the OA unit */ + struct xe_gt *gt; + + /** @type: Type of OA unit - OAM, OAG etc. */ + enum drm_xe_oa_unit_type type; + + /** @regs: OA registers for programming the OA unit */ + struct xe_oa_regs regs; + + /** @num_engines: number of engines attached to this OA unit */ + u32 num_engines; + + /** @exclusive_stream: The stream currently using the OA unit */ + struct xe_oa_stream *exclusive_stream; +}; + +/** + * struct xe_oa_gt - OA per-gt information + */ +struct xe_oa_gt { + /** @gt_lock: lock protecting create/destroy OA streams */ + struct mutex gt_lock; + + /** @num_oa_units: number of oa units for each gt */ + u32 num_oa_units; + + /** @oa_unit: array of oa_units */ + struct xe_oa_unit *oa_unit; + + /** @whitelist_count: number of open streams for which oa registers are whitelisted */ + u32 whitelist_count; +}; + +/** + * struct xe_oa - OA device level information + */ +struct xe_oa { + /** @xe: back pointer to xe device */ + struct xe_device *xe; + + /** @metrics_kobj: kobj for metrics sysfs */ + struct kobject *metrics_kobj; + + /** @metrics_lock: lock protecting add/remove configs */ + struct mutex metrics_lock; + + /** @metrics_idr: List of dynamic configurations (struct xe_oa_config) */ + struct idr metrics_idr; + + /** @oa_formats: tracks all OA formats across platforms */ + const struct xe_oa_format *oa_formats; + + /** @format_mask: tracks valid OA formats for a platform */ + unsigned long format_mask[BITS_TO_LONGS(__XE_OA_FORMAT_MAX)]; + + /** @oa_unit_ids: tracks oa unit ids assigned across gt's */ + u16 oa_unit_ids; +}; + +/** + * struct xe_oa_buffer - State of the stream OA buffer + */ +struct xe_oa_buffer { + /** @format: data format */ + const struct xe_oa_format *format; + + /** @bo: xe_bo backing the OA buffer */ + struct xe_bo *bo; + + /** @bounce: bounce buffer used with xe_map layer */ + void *bounce; + + /** @ptr_lock: Lock protecting reads/writes to head/tail pointers */ + spinlock_t ptr_lock; + + /** @head: Cached head to read from */ + u32 head; + + /** @tail: The last verified cached tail where HW has completed writing */ + u32 tail; + + /** @circ_size: The effective circular buffer size, for Xe2+ */ + u32 circ_size; +}; + +/** + * struct xe_oa_stream - state for a single open stream FD + */ +struct xe_oa_stream { + /** @oa: xe_oa backpointer */ + struct xe_oa *oa; + + /** @gt: gt associated with the oa stream */ + struct xe_gt *gt; + + /** @oa_unit: oa unit for this stream */ + struct xe_oa_unit *oa_unit; + + /** @hwe: hardware engine associated with this oa stream */ + struct xe_hw_engine *hwe; + + /** @stream_lock: Lock serializing stream operations */ + struct mutex stream_lock; + + /** @sample: true if DRM_XE_OA_PROP_SAMPLE_OA is provided */ + bool sample; + + /** @exec_q: Exec queue corresponding to DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID */ + struct xe_exec_queue *exec_q; + + /** @k_exec_q: kernel exec_q used for OA programming batch submissions */ + struct xe_exec_queue *k_exec_q; + + /** @enabled: Whether the stream is currently enabled */ + bool enabled; + + /** @oa_config: OA configuration used by the stream */ + struct xe_oa_config *oa_config; + + /** @oa_config_bos: List of struct @xe_oa_config_bo's */ + struct llist_head oa_config_bos; + + /** @poll_check_timer: Timer to periodically check for data in the OA buffer */ + struct hrtimer poll_check_timer; + + /** @poll_wq: Wait queue for waiting for OA data to be available */ + wait_queue_head_t poll_wq; + + /** @pollin: Whether there is data available to read */ + bool pollin; + + /** @wait_num_reports: Number of reports to wait for before signalling pollin */ + int wait_num_reports; + + /** @periodic: Whether periodic sampling is currently enabled */ + bool periodic; + + /** @period_exponent: OA unit sampling frequency is derived from this */ + int period_exponent; + + /** @oa_buffer: OA buffer for the stream */ + struct xe_oa_buffer oa_buffer; + + /** @poll_period_ns: hrtimer period for checking OA buffer for available data */ + u64 poll_period_ns; + + /** @oa_status: temporary storage for oa_status register value */ + u32 oa_status; + + /** @no_preempt: Whether preemption and timeslicing is disabled for stream exec_q */ + u32 no_preempt; + + /** @xef: xe_file with which the stream was opened */ + struct xe_file *xef; + + /** @ufence_syncobj: User fence syncobj */ + struct drm_syncobj *ufence_syncobj; + + /** @ufence_timeline_value: User fence timeline value */ + u64 ufence_timeline_value; + + /** @last_fence: fence to use in stream destroy when needed */ + struct dma_fence *last_fence; + + /** @num_syncs: size of @syncs array */ + u32 num_syncs; + + /** @syncs: syncs to wait on and to signal */ + struct xe_sync_entry *syncs; + + /** @fw_ref: Forcewake reference */ + unsigned int fw_ref; +}; +#endif diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c new file mode 100644 index 000000000..39e05b913 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_observation.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include +#include + +#include + +#include "xe_eu_stall.h" +#include "xe_oa.h" +#include "xe_observation.h" + +static u32 xe_observation_paranoid = true; +static struct ctl_table_header *sysctl_header; + +/** + * xe_observation_paranoid_check - Gate access to xe observation streams. + * + * When the xe-specific observation_paranoid sysctl is enabled (the + * default), defer to perf_allow_cpu() so that access is governed by the + * same policy as system-wide perf CPU events: kernel.perf_event_paranoid + * plus the security_perf_event_open() LSM hook. When the sysctl has been + * cleared by a privileged user, observation is open to all callers. + * + * Return: 0 if access is permitted, a negative errno otherwise. + */ +int xe_observation_paranoid_check(void) +{ + if (!xe_observation_paranoid) + return 0; + + return perf_allow_cpu(); +} + +static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg, + struct drm_file *file) +{ + switch (arg->observation_op) { + case DRM_XE_OBSERVATION_OP_STREAM_OPEN: + return xe_oa_stream_open_ioctl(dev, arg->param, file); + case DRM_XE_OBSERVATION_OP_ADD_CONFIG: + return xe_oa_add_config_ioctl(dev, arg->param, file); + case DRM_XE_OBSERVATION_OP_REMOVE_CONFIG: + return xe_oa_remove_config_ioctl(dev, arg->param, file); + default: + return -EINVAL; + } +} + +static int xe_eu_stall_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg, + struct drm_file *file) +{ + switch (arg->observation_op) { + case DRM_XE_OBSERVATION_OP_STREAM_OPEN: + return xe_eu_stall_stream_open(dev, arg->param, file); + default: + return -EINVAL; + } +} + +/** + * xe_observation_ioctl - The top level observation layer ioctl + * @dev: @drm_device + * @data: pointer to struct @drm_xe_observation_param + * @file: @drm_file + * + * The function is called for different observation streams types and + * allows execution of different operations supported by those stream + * types. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_xe_observation_param *arg = data; + + if (arg->extensions) + return -EINVAL; + + switch (arg->observation_type) { + case DRM_XE_OBSERVATION_TYPE_OA: + return xe_oa_ioctl(dev, arg, file); + case DRM_XE_OBSERVATION_TYPE_EU_STALL: + return xe_eu_stall_ioctl(dev, arg, file); + default: + return -EINVAL; + } +} + +static const struct ctl_table observation_ctl_table[] = { + { + .procname = "observation_paranoid", + .data = &xe_observation_paranoid, + .maxlen = sizeof(xe_observation_paranoid), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +}; + +/** + * xe_observation_sysctl_register - Register the observation_paranoid sysctl + * + * When dev.xe.observation_paranoid is set (the default), access to + * observation streams follows the system-wide perf_allow_cpu() policy: + * kernel.perf_event_paranoid plus the security_perf_event_open() LSM + * hook. A privileged user can clear the sysctl to bypass that gate and + * allow unprivileged access to observation data. + * + * Return: always returns 0 + */ +int xe_observation_sysctl_register(void) +{ + sysctl_header = register_sysctl("dev/xe", observation_ctl_table); + return 0; +} + +/** + * xe_observation_sysctl_unregister - Unregister xe_observation_paranoid sysctl + */ +void xe_observation_sysctl_unregister(void) +{ + unregister_sysctl_table(sysctl_header); +} diff --git a/drivers/gpu/drm/xe/xe_observation.h b/drivers/gpu/drm/xe/xe_observation.h new file mode 100644 index 000000000..73a03e03c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_observation.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_OBSERVATION_H_ +#define _XE_OBSERVATION_H_ + +#include + +struct drm_device; +struct drm_file; + +int xe_observation_paranoid_check(void); +int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int xe_observation_sysctl_register(void); +void xe_observation_sysctl_unregister(void); + +#endif diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c new file mode 100644 index 000000000..da1ed99cd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_page_reclaim.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include + +#include "xe_page_reclaim.h" + +#include "xe_gt_stats.h" +#include "xe_guc_tlb_inval.h" +#include "xe_macros.h" +#include "xe_pat.h" +#include "xe_sa.h" +#include "xe_tlb_inval_types.h" + +/** + * xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA + * @tile: Tile owning the VMA + * @vma: VMA under consideration + * + * PPC flushing may be handled by HW for specific PAT encodings. + * Skip PPC flushing/Page Reclaim for scenarios below due to redundant + * flushes. + * - pat_index is transient display (1) + * + * For cases of NULL VMA, there should be no corresponding PRL entry + * so skip over. + * + * Return: true when page reclamation is unnecessary, false otherwise. + */ +bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma) +{ + u8 l3_policy; + + if (xe_vma_is_null(vma)) + return true; + + l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index); + + /* + * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC + * Transient display flushes is taken care by HW, l3_policy = 1. + * + * HW will sequence these transient flushes at various sync points so + * any event of page reclamation will hit these sync points before + * page reclamation could execute. + */ + return (l3_policy == XE_L3_POLICY_XD); +} + +/** + * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO + * @tlb_inval: TLB invalidation frontend associated with the request + * @prl: page reclaim list data that bo will copy from + * @fence: tlb invalidation fence that page reclaim action is paired to + * + * Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU + * copy into the BO and queues the buffer for release when @fence signals. + * + * Return: struct drm_suballoc pointer on success or ERR_PTR on failure. + */ +struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, + struct xe_page_reclaim_list *prl, + struct xe_tlb_inval_fence *fence) +{ + struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval); + struct xe_tile *tile = gt_to_tile(gt); + /* (+1) for NULL page_reclaim_entry to indicate end of list */ + int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) * + sizeof(struct xe_guc_page_reclaim_entry); + struct drm_suballoc *prl_sa; + + /* Maximum size of PRL is 1 4K-page */ + prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool, + prl_size, GFP_ATOMIC); + if (IS_ERR(prl_sa)) + return prl_sa; + + memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries, + prl_size); + xe_sa_bo_flush_write(prl_sa); + /* Queue up sa_bo_free on tlb invalidation fence signal */ + xe_sa_bo_free(prl_sa, &fence->base); + + return prl_sa; +} + +/** + * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid + * @prl: Page reclaim list to reset + * + * Clears the entries pointer and marks the list as invalid so + * future use knows PRL is unusable. It is expected that the entries + * have already been released. + */ +void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl) +{ + xe_page_reclaim_entries_put(prl->entries); + prl->entries = NULL; + prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST; +} + +/** + * xe_page_reclaim_list_init() - Initialize a page reclaim list + * @prl: Page reclaim list to initialize + * + * NULLs both values in list to prepare on initalization. + */ +void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl) +{ + prl->entries = NULL; + prl->num_entries = 0; +} + +/** + * xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries + * @prl: Page reclaim list to allocate entries for + * + * Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL. + */ +int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl) +{ + struct page *page; + + if (XE_WARN_ON(prl->entries)) + return 0; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (page) { + prl->entries = page_address(page); + prl->num_entries = 0; + } + + return page ? 0 : -ENOMEM; +} + +/** + * xe_guc_page_reclaim_done_handler() - Page reclaim done handler + * @guc: guc + * @msg: message indicating page reclamation done + * @len: length of message + * + * Page reclamation is an extension of TLB invalidation. Both + * operations share the same seqno and fence. When either + * action completes, we need to signal the corresponding + * fence. Since the handling logic is currently identical, this + * function delegates to the TLB invalidation handler. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + return xe_guc_tlb_inval_done_handler(guc, msg, len); +} diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h new file mode 100644 index 000000000..0412611f3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_page_reclaim.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PAGE_RECLAIM_H_ +#define _XE_PAGE_RECLAIM_H_ + +#include +#include +#include +#include +#include +#include + +#define XE_PAGE_RECLAIM_MAX_ENTRIES 512 +#define XE_PAGE_RECLAIM_LIST_MAX_SIZE SZ_4K + +struct xe_tlb_inval; +struct xe_tlb_inval_fence; +struct xe_tile; +struct xe_gt; +struct xe_guc; +struct xe_vma; + +struct xe_guc_page_reclaim_entry { + u64 qw; +/* valid reclaim entry bit */ +#define XE_PAGE_RECLAIM_VALID BIT_ULL(0) +/* + * offset order of page size to be reclaimed + * page_size = 1 << (XE_PTE_SHIFT + reclamation_size) + */ +#define XE_PAGE_RECLAIM_SIZE GENMASK_ULL(6, 1) +#define XE_PAGE_RECLAIM_RSVD_0 GENMASK_ULL(11, 7) +/* lower 20 bits of the physical address */ +#define XE_PAGE_RECLAIM_ADDR_LO GENMASK_ULL(31, 12) +/* upper 20 bits of the physical address */ +#define XE_PAGE_RECLAIM_ADDR_HI GENMASK_ULL(51, 32) +#define XE_PAGE_RECLAIM_RSVD_1 GENMASK_ULL(63, 52) +} __packed; + +struct xe_page_reclaim_list { + /** @entries: array of page reclaim entries, page allocated */ + struct xe_guc_page_reclaim_entry *entries; + /** @num_entries: number of entries */ + int num_entries; +#define XE_PAGE_RECLAIM_INVALID_LIST -1 +}; + +/** + * xe_page_reclaim_list_is_new() - Check if PRL is new allocation + * @prl: Pointer to page reclaim list + * + * PRL indicates it hasn't been allocated through both values being NULL + */ +static inline bool xe_page_reclaim_list_is_new(struct xe_page_reclaim_list *prl) +{ + return !prl->entries && prl->num_entries == 0; +} + +/** + * xe_page_reclaim_list_valid() - Check if the page reclaim list is valid + * @prl: Pointer to page reclaim list + * + * PRL uses the XE_PAGE_RECLAIM_INVALID_LIST to indicate that a PRL + * is unusable. + */ +static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl) +{ + return !xe_page_reclaim_list_is_new(prl) && + prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST; +} + +bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma); +struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, + struct xe_page_reclaim_list *prl, + struct xe_tlb_inval_fence *fence); +void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl); + +/** + * xe_page_reclaim_list_abort() - Invalidate a PRL and log an abort reason + * @gt: GT owning the page reclaim request + * @prl: Page reclaim list to invalidate + * @fmt: format string for the log message with args + * + * Abort page reclaim process by invalidating PRL and doing any relevant logging. + */ +#define xe_page_reclaim_list_abort(gt, prl, fmt, ...) \ + do { \ + struct xe_gt *__gt = (gt); \ + struct xe_page_reclaim_list *__prl = (prl); \ + \ + xe_page_reclaim_list_invalidate(__prl); \ + xe_gt_stats_incr(__gt, XE_GT_STATS_ID_PRL_ABORTED_COUNT, 1); \ + vm_dbg(>_to_xe(__gt)->drm, "PRL aborted: " fmt, ##__VA_ARGS__); \ + } while (0) + +void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl); +int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl); +/** + * xe_page_reclaim_entries_get() - Increment the reference count of page reclaim entries. + * @entries: Pointer to the array of page reclaim entries. + * + * This function increments the reference count of the backing page. + */ +static inline void xe_page_reclaim_entries_get(struct xe_guc_page_reclaim_entry *entries) +{ + if (entries) + get_page(virt_to_page(entries)); +} + +/** + * xe_page_reclaim_entries_put() - Decrement the reference count of page reclaim entries. + * @entries: Pointer to the array of page reclaim entries. + * + * This function decrements the reference count of the backing page + * and frees it if the count reaches zero. + */ +static inline void xe_page_reclaim_entries_put(struct xe_guc_page_reclaim_entry *entries) +{ + if (entries) + put_page(virt_to_page(entries)); +} + +int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len); + +#endif /* _XE_PAGE_RECLAIM_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c new file mode 100644 index 000000000..dbf8f71d3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_gt_stats.h" +#include "xe_hw_engine.h" +#include "xe_pagefault.h" +#include "xe_pagefault_types.h" +#include "xe_svm.h" +#include "xe_trace_bo.h" +#include "xe_vm.h" + +/** + * DOC: Xe page faults + * + * Xe page faults are handled in two layers. The producer layer interacts with + * hardware or firmware to receive and parse faults into struct xe_pagefault, + * then forwards them to the consumer. The consumer layer services the faults + * (e.g., memory migration, page table updates) and acknowledges the result back + * to the producer, which then forwards the results to the hardware or firmware. + * The consumer uses a page fault queue sized to absorb all potential faults and + * a multi-threaded worker to process them. Multiple producers are supported, + * with a single shared consumer. + * + * xe_pagefault.c implements the consumer layer. + */ + +static int xe_pagefault_entry_size(void) +{ + /* + * Power of two alignment is not a hardware requirement, rather a + * software restriction which makes the math for page fault queue + * management simplier. + */ + return roundup_pow_of_two(sizeof(struct xe_pagefault)); +} + +static int xe_pagefault_begin(struct drm_exec *exec, struct xe_vma *vma, + struct xe_vram_region *vram, bool need_vram_move) +{ + struct xe_bo *bo = xe_vma_bo(vma); + struct xe_vm *vm = xe_vma_vm(vma); + int err; + + err = xe_vm_lock_vma(exec, vma); + if (err) + return err; + + if (!bo) + return 0; + + /* + * Skip validate/migrate for DONTNEED/purged BOs - repopulating + * their pages would prevent the shrinker from reclaiming them. + * For non-scratch VMs there is no safe fallback so fail the fault. + * For scratch VMs let xe_vma_rebind() run normally; it will install + * scratch PTEs so the GPU gets safe zero reads instead of faulting. + */ + if (unlikely(xe_bo_madv_is_dontneed(bo) || xe_bo_is_purged(bo))) { + if (!xe_vm_has_scratch(vm)) + return -EACCES; + return 0; + } + + return need_vram_move ? xe_bo_migrate(bo, vram->placement, NULL, exec) : + xe_bo_validate(bo, vm, true, exec); +} + +static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma, + bool atomic) +{ + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct dma_fence *fence; + int err, needs_vram; + + lockdep_assert_held_write(&vm->lock); + + needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); + if (needs_vram < 0 || (needs_vram && xe_vma_is_userptr(vma))) + return needs_vram < 0 ? needs_vram : -EACCES; + + xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1); + xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB, + xe_vma_size(vma) / SZ_1K); + + trace_xe_vma_pagefault(vma); + + /* Check if VMA is valid, opportunistic check only */ + if (xe_vm_has_valid_gpu_mapping(tile, vma->tile_present, + vma->tile_invalidated) && !atomic) + return 0; + +retry_userptr: + if (xe_vma_is_userptr(vma) && + xe_vma_userptr_check_repin(to_userptr_vma(vma))) { + struct xe_userptr_vma *uvma = to_userptr_vma(vma); + + err = xe_vma_userptr_pin_pages(uvma); + if (err) + return err; + } + + /* Lock VM and BOs dma-resv */ + xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {}); + drm_exec_until_all_locked(&exec) { + err = xe_pagefault_begin(&exec, vma, tile->mem.vram, + needs_vram == 1); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + if (err) + goto unlock_dma_resv; + + /* Bind VMA only to the GT that has faulted */ + trace_xe_vma_pf_bind(vma); + xe_vm_set_validation_exec(vm, &exec); + fence = xe_vma_rebind(vm, vma, BIT(tile->id)); + xe_vm_set_validation_exec(vm, NULL); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + xe_validation_retry_on_oom(&ctx, &err); + goto unlock_dma_resv; + } + } + + dma_fence_wait(fence, false); + dma_fence_put(fence); + +unlock_dma_resv: + xe_validation_ctx_fini(&ctx); + if (err == -EAGAIN) + goto retry_userptr; + + return err; +} + +static bool +xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type) +{ + return (access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK) == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC; +} + +static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid) +{ + struct xe_vm *vm; + + down_read(&xe->usm.lock); + vm = xa_load(&xe->usm.asid_to_vm, asid); + if (vm && xe_vm_in_fault_mode(vm)) + xe_vm_get(vm); + else + vm = ERR_PTR(-EINVAL); + up_read(&xe->usm.lock); + + return vm; +} + +static int xe_pagefault_service(struct xe_pagefault *pf) +{ + struct xe_gt *gt = pf->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_vm *vm; + struct xe_vma *vma = NULL; + int err; + bool atomic; + + /* Producer flagged this fault to be nacked */ + if (pf->consumer.fault_type_level == XE_PAGEFAULT_TYPE_LEVEL_NACK) + return -EFAULT; + + vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid); + if (IS_ERR(vm)) + return PTR_ERR(vm); + + /* + * TODO: Change to read lock? Using write lock for simplicity. + */ + down_write(&vm->lock); + + if (xe_vm_is_closed(vm)) { + err = -ENOENT; + goto unlock_vm; + } + + vma = xe_vm_find_vma_by_addr(vm, pf->consumer.page_addr); + if (!vma) { + err = -EINVAL; + goto unlock_vm; + } + + if (xe_vma_read_only(vma) && + pf->consumer.access_type != XE_PAGEFAULT_ACCESS_TYPE_READ) { + err = -EPERM; + goto unlock_vm; + } + + atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type); + + if (xe_vma_is_cpu_addr_mirror(vma)) + err = xe_svm_handle_pagefault(vm, vma, gt, + pf->consumer.page_addr, atomic); + else + err = xe_pagefault_handle_vma(gt, vma, atomic); + +unlock_vm: + if (!err) + vm->usm.last_fault_vma = vma; + up_write(&vm->lock); + xe_vm_put(vm); + + return err; +} + +static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, + struct xe_pagefault *pf) +{ + bool found_fault = false; + + spin_lock_irq(&pf_queue->lock); + if (pf_queue->tail != pf_queue->head) { + memcpy(pf, pf_queue->data + pf_queue->tail, sizeof(*pf)); + pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) % + pf_queue->size; + found_fault = true; + } + spin_unlock_irq(&pf_queue->lock); + + return found_fault; +} + +static void xe_pagefault_print(struct xe_pagefault *pf) +{ + xe_gt_info(pf->gt, "\n\tASID: %d\n" + "\tFaulted Address: 0x%08x%08x\n" + "\tFaultType: %lu\n" + "\tAccessType: %lu\n" + "\tFaultLevel: %lu\n" + "\tEngineClass: %d %s\n" + "\tEngineInstance: %d\n", + pf->consumer.asid, + upper_32_bits(pf->consumer.page_addr), + lower_32_bits(pf->consumer.page_addr), + FIELD_GET(XE_PAGEFAULT_TYPE_MASK, + pf->consumer.fault_type_level), + FIELD_GET(XE_PAGEFAULT_ACCESS_TYPE_MASK, + pf->consumer.access_type), + FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, + pf->consumer.fault_type_level), + pf->consumer.engine_class, + xe_hw_engine_class_to_str(pf->consumer.engine_class), + pf->consumer.engine_instance); +} + +static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf) +{ + struct xe_vm *vm; + + /* + * Pagefault may be asociated to VM that is not in fault mode. + * Perform asid_to_vm behavior, except if VM is not in fault + * mode, return VM anyways. + */ + down_read(&xe->usm.lock); + vm = xa_load(&xe->usm.asid_to_vm, pf->consumer.asid); + if (vm) + xe_vm_get(vm); + else + vm = ERR_PTR(-EINVAL); + up_read(&xe->usm.lock); + + if (IS_ERR(vm)) + return; + + xe_vm_add_fault_entry_pf(vm, pf); + + xe_vm_put(vm); +} + +static void xe_pagefault_queue_work(struct work_struct *w) +{ + struct xe_pagefault_queue *pf_queue = + container_of(w, typeof(*pf_queue), worker); + struct xe_pagefault pf; + unsigned long threshold; + +#define USM_QUEUE_MAX_RUNTIME_MS 20 + threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); + + while (xe_pagefault_queue_pop(pf_queue, &pf)) { + int err; + + if (!pf.gt) /* Fault squashed during reset */ + continue; + + err = xe_pagefault_service(&pf); + if (err) { + xe_pagefault_save_to_vm(gt_to_xe(pf.gt), &pf); + if (!(pf.consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) { + xe_pagefault_print(&pf); + xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n", + ERR_PTR(err)); + } else { + xe_gt_stats_incr(pf.gt, XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1); + xe_gt_dbg(pf.gt, "Prefetch Fault response: Unsuccessful %pe\n", + ERR_PTR(err)); + } + } + + pf.producer.ops->ack_fault(&pf, err); + + if (time_after(jiffies, threshold)) { + queue_work(gt_to_xe(pf.gt)->usm.pf_wq, w); + break; + } + } +#undef USM_QUEUE_MAX_RUNTIME_MS +} + +static int xe_pagefault_queue_init(struct xe_device *xe, + struct xe_pagefault_queue *pf_queue) +{ + struct xe_gt *gt; + int total_num_eus = 0; + u8 id; + + for_each_gt(gt, xe, id) { + xe_dss_mask_t all_dss; + int num_dss, num_eus; + + num_dss = bitmap_weighted_or(all_dss, gt->fuse_topo.g_dss_mask, + gt->fuse_topo.c_dss_mask, XE_MAX_DSS_FUSE_BITS); + + num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss, + XE_MAX_EU_FUSE_BITS) * num_dss; + + total_num_eus += num_eus; + } + + xe_assert(xe, total_num_eus); + + /* + * user can issue separate page faults per EU and per CS + * + * XXX: Multiplier required as compute UMD are getting PF queue errors + * without it. Follow on why this multiplier is required. + */ +#define PF_MULTIPLIER 8 + pf_queue->size = (total_num_eus + XE_NUM_HW_ENGINES) * + xe_pagefault_entry_size() * PF_MULTIPLIER; + pf_queue->size = roundup_pow_of_two(pf_queue->size); +#undef PF_MULTIPLIER + + drm_dbg(&xe->drm, "xe_pagefault_entry_size=%d, total_num_eus=%d, pf_queue->size=%u", + xe_pagefault_entry_size(), total_num_eus, pf_queue->size); + + spin_lock_init(&pf_queue->lock); + INIT_WORK(&pf_queue->worker, xe_pagefault_queue_work); + + pf_queue->data = drmm_kzalloc(&xe->drm, pf_queue->size, GFP_KERNEL); + if (!pf_queue->data) + return -ENOMEM; + + return 0; +} + +static void xe_pagefault_fini(void *arg) +{ + struct xe_device *xe = arg; + + destroy_workqueue(xe->usm.pf_wq); +} + +/** + * xe_pagefault_init() - Page fault init + * @xe: xe device instance + * + * Initialize Xe page fault state. Must be done after reading fuses. + * + * Return: 0 on Success, errno on failure + */ +int xe_pagefault_init(struct xe_device *xe) +{ + int err, i; + + if (!xe->info.has_usm) + return 0; + + xe->usm.pf_wq = alloc_workqueue("xe_page_fault_work_queue", + WQ_UNBOUND | WQ_HIGHPRI, + XE_PAGEFAULT_QUEUE_COUNT); + if (!xe->usm.pf_wq) + return -ENOMEM; + + for (i = 0; i < XE_PAGEFAULT_QUEUE_COUNT; ++i) { + err = xe_pagefault_queue_init(xe, xe->usm.pf_queue + i); + if (err) + goto err_out; + } + + return devm_add_action_or_reset(xe->drm.dev, xe_pagefault_fini, xe); + +err_out: + destroy_workqueue(xe->usm.pf_wq); + return err; +} + +static void xe_pagefault_queue_reset(struct xe_device *xe, struct xe_gt *gt, + struct xe_pagefault_queue *pf_queue) +{ + u32 i; + + /* Driver load failure guard / USM not enabled guard */ + if (!pf_queue->data) + return; + + /* Squash all pending faults on the GT */ + + spin_lock_irq(&pf_queue->lock); + for (i = pf_queue->tail; i != pf_queue->head; + i = (i + xe_pagefault_entry_size()) % pf_queue->size) { + struct xe_pagefault *pf = pf_queue->data + i; + + if (pf->gt == gt) + pf->gt = NULL; + } + spin_unlock_irq(&pf_queue->lock); +} + +/** + * xe_pagefault_reset() - Page fault reset for a GT + * @xe: xe device instance + * @gt: GT being reset + * + * Reset the Xe page fault state for a GT; that is, squash any pending faults on + * the GT. + */ +void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt) +{ + int i; + + for (i = 0; i < XE_PAGEFAULT_QUEUE_COUNT; ++i) + xe_pagefault_queue_reset(xe, gt, xe->usm.pf_queue + i); +} + +static bool xe_pagefault_queue_full(struct xe_pagefault_queue *pf_queue) +{ + lockdep_assert_held(&pf_queue->lock); + + return CIRC_SPACE(pf_queue->head, pf_queue->tail, pf_queue->size) <= + xe_pagefault_entry_size(); +} + +/** + * xe_pagefault_handler() - Page fault handler + * @xe: xe device instance + * @pf: Page fault + * + * Sink the page fault to a queue (i.e., a memory buffer) and queue a worker to + * service it. Safe to be called from IRQ or process context. Reclaim safe. + * + * Return: 0 on success, errno on failure + */ +int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf) +{ + struct xe_pagefault_queue *pf_queue = xe->usm.pf_queue + + (pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT); + unsigned long flags; + bool full; + + spin_lock_irqsave(&pf_queue->lock, flags); + full = xe_pagefault_queue_full(pf_queue); + if (!full) { + memcpy(pf_queue->data + pf_queue->head, pf, sizeof(*pf)); + pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) % + pf_queue->size; + queue_work(xe->usm.pf_wq, &pf_queue->worker); + } else { + drm_warn(&xe->drm, + "PageFault Queue (%d) full, shouldn't be possible\n", + pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT); + } + spin_unlock_irqrestore(&pf_queue->lock, flags); + + return full ? -ENOSPC : 0; +} diff --git a/drivers/gpu/drm/xe/xe_pagefault.h b/drivers/gpu/drm/xe/xe_pagefault.h new file mode 100644 index 000000000..bd0cdf9ed --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pagefault.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PAGEFAULT_H_ +#define _XE_PAGEFAULT_H_ + +struct xe_device; +struct xe_gt; +struct xe_pagefault; + +int xe_pagefault_init(struct xe_device *xe); + +void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt); + +int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h new file mode 100644 index 000000000..c4ee625b9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PAGEFAULT_TYPES_H_ +#define _XE_PAGEFAULT_TYPES_H_ + +#include + +struct xe_gt; +struct xe_pagefault; + +/** enum xe_pagefault_access_type - Xe page fault access type */ +enum xe_pagefault_access_type { + /** @XE_PAGEFAULT_ACCESS_TYPE_READ: Read access type */ + XE_PAGEFAULT_ACCESS_TYPE_READ = 0, + /** @XE_PAGEFAULT_ACCESS_TYPE_WRITE: Write access type */ + XE_PAGEFAULT_ACCESS_TYPE_WRITE = 1, + /** @XE_PAGEFAULT_ACCESS_TYPE_ATOMIC: Atomic access type */ + XE_PAGEFAULT_ACCESS_TYPE_ATOMIC = 2, +}; + +/** enum xe_pagefault_type - Xe page fault type */ +enum xe_pagefault_type { + /** @XE_PAGEFAULT_TYPE_NOT_PRESENT: Not present */ + XE_PAGEFAULT_TYPE_NOT_PRESENT = 0, + /** @XE_PAGEFAULT_TYPE_WRITE_ACCESS_VIOLATION: Write access violation */ + XE_PAGEFAULT_TYPE_WRITE_ACCESS_VIOLATION = 1, + /** @XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION: Atomic access violation */ + XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION = 2, +}; + +/** struct xe_pagefault_ops - Xe pagefault ops (producer) */ +struct xe_pagefault_ops { + /** + * @ack_fault: Ack fault + * @pf: Page fault + * @err: Error state of fault + * + * Page fault producer receives acknowledgment from the consumer and + * sends the result to the HW/FW interface. + */ + void (*ack_fault)(struct xe_pagefault *pf, int err); +}; + +/** + * struct xe_pagefault - Xe page fault + * + * Generic page fault structure for communication between producer and consumer. + * Carefully sized to be 64 bytes. Upon a device page fault, the producer + * populates this structure, and the consumer copies it into the page-fault + * queue for deferred handling. + */ +struct xe_pagefault { + /** + * @gt: GT of fault + */ + struct xe_gt *gt; + /** + * @consumer: State for the software handling the fault. Populated by + * the producer and may be modified by the consumer to communicate + * information back to the producer upon fault acknowledgment. + */ + struct { + /** @consumer.page_addr: address of page fault */ + u64 page_addr; + /** @consumer.asid: address space ID */ + u32 asid; + /** + * @consumer.access_type: access type and prefetch flag packed + * into a u8. + */ + u8 access_type; +#define XE_PAGEFAULT_ACCESS_TYPE_MASK GENMASK(1, 0) +#define XE_PAGEFAULT_ACCESS_PREFETCH BIT(7) + /** + * @consumer.fault_type_level: fault type and level, u8 rather + * than enum to keep size compact + */ + u8 fault_type_level; +#define XE_PAGEFAULT_TYPE_LEVEL_NACK 0xff /* Producer indicates nack fault */ +#define XE_PAGEFAULT_LEVEL_MASK GENMASK(3, 0) +#define XE_PAGEFAULT_TYPE_MASK GENMASK(7, 4) + /** @consumer.engine_class: engine class */ + u8 engine_class; + /** @consumer.engine_instance: engine instance */ + u8 engine_instance; + /** @consumer.reserved: reserved bits for future expansion */ + u64 reserved; + } consumer; + /** + * @producer: State for the producer (i.e., HW/FW interface). Populated + * by the producer and should not be modified—or even inspected—by the + * consumer, except for calling operations. + */ + struct { + /** @producer.private: private pointer */ + void *private; + /** @producer.ops: operations */ + const struct xe_pagefault_ops *ops; +#define XE_PAGEFAULT_PRODUCER_MSG_LEN_DW 4 + /** + * @producer.msg: page fault message, used by producer in fault + * acknowledgment to formulate response to HW/FW interface. + * Included in the page-fault message because the producer + * typically receives the fault in a context where memory cannot + * be allocated (e.g., atomic context or the reclaim path). + */ + u32 msg[XE_PAGEFAULT_PRODUCER_MSG_LEN_DW]; + } producer; +}; + +/** + * struct xe_pagefault_queue - Xe pagefault queue (consumer) + * + * Used to capture all device page faults for deferred processing. Size this + * queue to absorb the device’s worst-case number of outstanding faults. + */ +struct xe_pagefault_queue { + /** + * @data: Data in queue containing struct xe_pagefault, protected by + * @lock + */ + void *data; + /** @size: Size of queue in bytes */ + u32 size; + /** @head: Head pointer in bytes, moved by producer, protected by @lock */ + u32 head; + /** @tail: Tail pointer in bytes, moved by consumer, protected by @lock */ + u32 tail; + /** @lock: protects page fault queue */ + spinlock_t lock; + /** @worker: to process page faults */ + struct work_struct worker; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c new file mode 100644 index 000000000..a5fe1beec --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -0,0 +1,797 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_pat.h" + +#include + +#include + +#include "regs/xe_gt_regs.h" +#include "regs/xe_reg_defs.h" +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_wa.h" + +#define _PAT_ATS 0x47fc +#define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ + 0x4800, 0x4804, \ + 0x4848, 0x484c) +#define _PAT_PTA 0x4820 +#define _PAT_TR_PTA 0x48cc + +#define XE2_NO_PROMOTE REG_BIT(10) +#define XE2_COMP_EN REG_BIT(9) +#define XE2_L3_CLOS REG_GENMASK(7, 6) +#define XE2_L3_POLICY REG_GENMASK(5, 4) +#define XE2_L4_POLICY REG_GENMASK(3, 2) +#define XE2_COH_MODE REG_GENMASK(1, 0) + +#define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) +#define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) +#define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) +#define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) +#define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) +#define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) +#define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) +#define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) + +#define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) +#define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) + +#define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) +#define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) +#define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) +#define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) +#define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) + +#define PAT_LABEL_LEN 20 + +static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; + +static void xe_pat_index_label(char *label, size_t len, int index) +{ + snprintf(label, len, "PAT[%2d] ", index); +} + +static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat) +{ + u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); + + drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index, + XELP_MEM_TYPE_STR_MAP[mem_type], pat); +} + +static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat) +{ + drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, + REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), + REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); +} + +static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat) +{ + drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, + REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), + REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); +} + +struct xe_pat_ops { + void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], + int n_entries); + void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], + int n_entries); + int (*dump)(struct xe_gt *gt, struct drm_printer *p); + void (*entry_dump)(struct drm_printer *p, const char *label, u32 pat, bool rsvd); +}; + +static const struct xe_pat_table_entry xelp_pat_table[] = { + [0] = { XELP_PAT_WB, XE_COH_1WAY }, + [1] = { XELP_PAT_WC, XE_COH_NONE }, + [2] = { XELP_PAT_WT, XE_COH_NONE }, + [3] = { XELP_PAT_UC, XE_COH_NONE }, +}; + +static const struct xe_pat_table_entry xehpc_pat_table[] = { + [0] = { XELP_PAT_UC, XE_COH_NONE }, + [1] = { XELP_PAT_WC, XE_COH_NONE }, + [2] = { XELP_PAT_WT, XE_COH_NONE }, + [3] = { XELP_PAT_WB, XE_COH_1WAY }, + [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, + [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_1WAY }, + [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, + [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_1WAY }, +}; + +static const struct xe_pat_table_entry xelpg_pat_table[] = { + [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, + [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, + [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, + [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_1WAY }, + [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_2WAY }, +}; + +/* + * The Xe2 table is getting large/complicated so it's easier to review if + * provided in a form that exactly matches the bspec's formatting. The meaning + * of the fields here are: + * - no_promote: 0=promotable, 1=no promote + * - comp_en: 0=disable, 1=enable + * - l3clos: L3 class of service (0-3) + * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), + * 2=XA ("WB - Transient App" for Xe3p), 3=UC + * - l4_policy: 0=WB, 1=WT, 3=UC + * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent + * + * Reserved entries should be programmed with the maximum caching, minimum + * coherency (which matches an all-0's encoding), so we can just omit them + * in the table. + * + * Note: There is an implicit assumption in the driver that compression and + * coh_1way+ are mutually exclusive for platforms prior to Xe3. Starting + * with Xe3, compression can be combined with coherency. If using compression + * with coherency, userptr and imported dma-buf from external device will + * have uncleared ccs state. See also xe_bo_needs_ccs_pages(). + */ +#define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ + { \ + .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ + (comp_en ? XE2_COMP_EN : 0) | \ + REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ + REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ + REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ + REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ + .coh_mode = __coh_mode ? __coh_mode : XE_COH_NONE, \ + .valid = 1 \ + } + +static const struct xe_pat_table_entry xe2_pat_table[] = { + [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), + [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), + [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), + [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), + [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), + [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), + [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), + [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), + [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), + [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), + [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), + [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), + [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), + [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), + [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), + [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), + /* 16..19 are reserved; leave set to all 0's */ + [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), + [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), + [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), + [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), + [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), + [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), + [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), + [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), + [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), + [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), + [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), + [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), +}; + +static const struct xe_pat_table_entry xe3_lpg_pat_table[] = { + [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), + [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), + [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), + [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), + [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), + [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), + [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), + [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), + [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), + [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), + [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), + [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), + [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), + [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), + [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), + [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), + [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), + /* 17..19 are reserved; leave set to all 0's */ + [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), + [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), + [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), + [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), + [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), + [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), + [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), + [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), + [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), + [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), + [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), + [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), +}; +/* Special PAT values programmed outside the main table */ +static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); +static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); + +/* + * Xe3p_XPC PAT table uses the same layout as Xe2/Xe3, except that there's no + * option for compression. Also note that the "L3" and "L4" register fields + * actually control L2 and L3 cache respectively on this platform. + */ +#define XE3P_XPC_PAT(no_promote, l3clos, l3_policy, l4_policy, __coh_mode) \ + XE2_PAT(no_promote, 0, l3clos, l3_policy, l4_policy, __coh_mode) + +static const struct xe_pat_table_entry xe3p_xpc_pat_ats = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ); +static const struct xe_pat_table_entry xe3p_xpc_pat_pta = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ); + +static const struct xe_pat_table_entry xe3p_xpc_pat_table[] = { + [ 0] = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ), + [ 1] = XE3P_XPC_PAT( 0, 0, 0, 0, 2 ), + [ 2] = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ), + [ 3] = XE3P_XPC_PAT( 0, 0, 3, 3, 0 ), + [ 4] = XE3P_XPC_PAT( 0, 0, 3, 3, 2 ), + [ 5] = XE3P_XPC_PAT( 0, 0, 3, 0, 0 ), + [ 6] = XE3P_XPC_PAT( 0, 0, 3, 0, 2 ), + [ 7] = XE3P_XPC_PAT( 0, 0, 3, 0, 3 ), + [ 8] = XE3P_XPC_PAT( 0, 0, 0, 3, 0 ), + [ 9] = XE3P_XPC_PAT( 0, 0, 0, 3, 2 ), + [10] = XE3P_XPC_PAT( 0, 0, 0, 3, 3 ), + /* 11..22 are reserved; leave set to all 0's */ + [23] = XE3P_XPC_PAT( 0, 1, 0, 0, 0 ), + [24] = XE3P_XPC_PAT( 0, 1, 0, 0, 2 ), + [25] = XE3P_XPC_PAT( 0, 1, 0, 0, 3 ), + [26] = XE3P_XPC_PAT( 0, 2, 0, 0, 0 ), + [27] = XE3P_XPC_PAT( 0, 2, 0, 0, 2 ), + [28] = XE3P_XPC_PAT( 0, 2, 0, 0, 3 ), + [29] = XE3P_XPC_PAT( 0, 3, 0, 0, 0 ), + [30] = XE3P_XPC_PAT( 0, 3, 0, 0, 2 ), + [31] = XE3P_XPC_PAT( 0, 3, 0, 0, 3 ), +}; + +static const struct xe_pat_table_entry xe3p_primary_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 3); +static const struct xe_pat_table_entry xe3p_media_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 2); +static const struct xe_pat_table_entry xe3p_pat_tr_pta = XE2_PAT(0, 0, 0, 0, 0, 0); + +static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = { + [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), + [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), + [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), + [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), + [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), + [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), + [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), + [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), + [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), + [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), + [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), + [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), + [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), + [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), + [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), + [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), + [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), + /* 17 is reserved; leave set to all 0's */ + [18] = XE2_PAT( 1, 0, 0, 2, 3, 0 ), + [19] = XE2_PAT( 1, 0, 0, 2, 3, 2 ), + [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), + [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), + [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), + [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), + [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), + [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), + [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), + [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), + [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), + [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), + [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), + [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), +}; + +u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) +{ + WARN_ON(pat_index >= xe->pat.n_entries); + return xe->pat.table[pat_index].coh_mode; +} + +bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) +{ + WARN_ON(pat_index >= xe->pat.n_entries); + return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); +} + +u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) +{ + WARN_ON(pat_index >= xe->pat.n_entries); + + return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); +} + +static const struct xe_pat_table_entry *gt_pta_entry(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (xe_gt_is_main_type(gt)) + return xe->pat.pat_primary_pta; + + if (xe_gt_is_media_type(gt)) + return xe->pat.pat_media_pta; + + xe_assert(xe, false); + return NULL; +} + +static const struct xe_pat_table_entry *gt_tr_pta_entry(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (xe_gt_is_main_type(gt)) + return xe->pat.pat_primary_tr_pta; + + if (xe_gt_is_media_type(gt)) + return xe->pat.pat_media_tr_pta; + + xe_assert(xe, false); + return NULL; +} + +static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], + int n_entries) +{ + struct xe_device *xe = gt_to_xe(gt); + const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); + const struct xe_pat_table_entry *tr_pta_entry = gt_tr_pta_entry(gt); + + for (int i = 0; i < n_entries; i++) { + struct xe_reg reg = XE_REG(_PAT_INDEX(i)); + + xe_mmio_write32(>->mmio, reg, table[i].value); + } + + if (xe->pat.pat_ats) + xe_mmio_write32(>->mmio, XE_REG(_PAT_ATS), xe->pat.pat_ats->value); + + if (pta_entry) + xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), pta_entry->value); + + if (tr_pta_entry) + xe_mmio_write32(>->mmio, XE_REG(_PAT_TR_PTA), tr_pta_entry->value); +} + +static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], + int n_entries) +{ + struct xe_device *xe = gt_to_xe(gt); + const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); + const struct xe_pat_table_entry *tr_pta_entry = gt_tr_pta_entry(gt); + + for (int i = 0; i < n_entries; i++) { + struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); + + xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); + } + + if (xe->pat.pat_ats) + xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value); + + if (pta_entry) + xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), pta_entry->value); + + if (tr_pta_entry) + xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_TR_PTA), tr_pta_entry->value); +} + +static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + int i; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + drm_printf(p, "PAT table:\n"); + + for (i = 0; i < xe->pat.n_entries; i++) { + u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); + + xelp_pat_entry_dump(p, i, pat); + } + + return 0; +} + +static const struct xe_pat_ops xelp_pat_ops = { + .program_graphics = program_pat, + .dump = xelp_dump, +}; + +static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + int i; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + drm_printf(p, "PAT table:\n"); + + for (i = 0; i < xe->pat.n_entries; i++) { + u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); + + xelp_pat_entry_dump(p, i, pat); + } + + return 0; +} + +static const struct xe_pat_ops xehp_pat_ops = { + .program_graphics = program_pat_mcr, + .dump = xehp_dump, +}; + +static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + int i; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + drm_printf(p, "PAT table:\n"); + + for (i = 0; i < xe->pat.n_entries; i++) { + u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); + + xehpc_pat_entry_dump(p, i, pat); + } + + return 0; +} + +static const struct xe_pat_ops xehpc_pat_ops = { + .program_graphics = program_pat_mcr, + .dump = xehpc_dump, +}; + +static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + int i; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + drm_printf(p, "PAT table:\n"); + + for (i = 0; i < xe->pat.n_entries; i++) { + u32 pat; + + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); + + xelpg_pat_entry_dump(p, i, pat); + } + + return 0; +} + +/* + * SAMedia register offsets are adjusted by the write methods and they target + * registers that are not MCR, while for normal GT they are MCR + */ +static const struct xe_pat_ops xelpg_pat_ops = { + .program_graphics = program_pat, + .program_media = program_pat_mcr, + .dump = xelpg_dump, +}; + +static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) +{ + drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label, + !!(pat & XE2_NO_PROMOTE), + !!(pat & XE2_COMP_EN), + REG_FIELD_GET(XE2_L3_CLOS, pat), + REG_FIELD_GET(XE2_L3_POLICY, pat), + REG_FIELD_GET(XE2_L4_POLICY, pat), + REG_FIELD_GET(XE2_COH_MODE, pat), + pat, rsvd ? " *" : ""); +} + +static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) +{ + drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label, + !!(pat & XE2_NO_PROMOTE), + REG_FIELD_GET(XE2_L3_CLOS, pat), + REG_FIELD_GET(XE2_L3_POLICY, pat), + REG_FIELD_GET(XE2_L4_POLICY, pat), + REG_FIELD_GET(XE2_COH_MODE, pat), + pat, rsvd ? " *" : ""); +} + +static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 pat; + int i; + char label[PAT_LABEL_LEN]; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + drm_printf(p, "PAT table: (* = reserved entry)\n"); + + for (i = 0; i < xe->pat.n_entries; i++) { + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); + + xe_pat_index_label(label, sizeof(label), i); + xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); + } + + /* + * Also print PTA_MODE, which describes how the hardware accesses + * PPGTT entries. + */ + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_PTA)); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); + + drm_printf(p, "Page Table Access:\n"); + xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); + + if (gt_tr_pta_entry(gt)) { + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_TR_PTA)); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_TR_PTA)); + + drm_printf(p, "TRTT Page Table Access:\n"); + xe->pat.ops->entry_dump(p, "TR_PTA_MODE", pat, false); + } + + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_ATS)); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_ATS)); + + drm_printf(p, "PCIe ATS/PASID:\n"); + xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); + + return 0; +} + +static const struct xe_pat_ops xe2_pat_ops = { + .program_graphics = program_pat_mcr, + .program_media = program_pat, + .dump = xe2_dump, + .entry_dump = xe2_pat_entry_dump, +}; + +static const struct xe_pat_ops xe3p_xpc_pat_ops = { + .program_graphics = program_pat_mcr, + .program_media = program_pat, + .dump = xe2_dump, + .entry_dump = xe3p_xpc_pat_entry_dump, +}; + +void xe_pat_init_early(struct xe_device *xe) +{ + xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; + xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = XE_PAT_INVALID_IDX; + if (GRAPHICS_VERx100(xe) == 3511) { + xe->pat.ops = &xe3p_xpc_pat_ops; + xe->pat.table = xe3p_xpc_pat_table; + xe->pat.pat_ats = &xe3p_xpc_pat_ats; + xe->pat.pat_primary_pta = &xe3p_xpc_pat_pta; + xe->pat.pat_media_pta = &xe3p_xpc_pat_pta; + xe->pat.n_entries = ARRAY_SIZE(xe3p_xpc_pat_table); + xe->pat.idx[XE_CACHE_NONE] = 3; + xe->pat.idx[XE_CACHE_WT] = 3; /* N/A (no display); use UC */ + xe->pat.idx[XE_CACHE_WB] = 2; + } else if (GRAPHICS_VER(xe) == 35) { + xe->pat.ops = &xe2_pat_ops; + xe->pat.table = xe3p_lpg_pat_table; + xe->pat.pat_ats = &xe2_pat_ats; + if (!IS_DGFX(xe)) { + xe->pat.pat_primary_pta = &xe3p_primary_pat_pta; + xe->pat.pat_media_pta = &xe3p_media_pat_pta; + xe->pat.pat_primary_tr_pta = &xe3p_pat_tr_pta; + xe->pat.pat_media_tr_pta = &xe3p_pat_tr_pta; + } + xe->pat.n_entries = ARRAY_SIZE(xe3p_lpg_pat_table); + xe->pat.idx[XE_CACHE_NONE] = 3; + xe->pat.idx[XE_CACHE_WT] = 15; + xe->pat.idx[XE_CACHE_WB] = 2; + xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; + xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; + } else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) { + xe->pat.ops = &xe2_pat_ops; + if (GRAPHICS_VER(xe) == 30) { + xe->pat.table = xe3_lpg_pat_table; + xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; + } else { + xe->pat.table = xe2_pat_table; + } + xe->pat.pat_ats = &xe2_pat_ats; + if (IS_DGFX(xe)) { + xe->pat.pat_primary_pta = &xe2_pat_pta; + xe->pat.pat_media_pta = &xe2_pat_pta; + } + + /* Wa_16023588340. XXX: Should use XE_WA */ + if (GRAPHICS_VERx100(xe) == 2001) + xe->pat.n_entries = 28; /* Disable CLOS3 */ + else + xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); + + xe->pat.idx[XE_CACHE_NONE] = 3; + xe->pat.idx[XE_CACHE_WT] = 15; + xe->pat.idx[XE_CACHE_WB] = 2; + xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ + } else if (xe->info.platform == XE_METEORLAKE) { + xe->pat.ops = &xelpg_pat_ops; + xe->pat.table = xelpg_pat_table; + xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); + xe->pat.idx[XE_CACHE_NONE] = 2; + xe->pat.idx[XE_CACHE_WT] = 1; + xe->pat.idx[XE_CACHE_WB] = 3; + } else if (xe->info.platform == XE_PVC) { + xe->pat.ops = &xehpc_pat_ops; + xe->pat.table = xehpc_pat_table; + xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); + xe->pat.idx[XE_CACHE_NONE] = 0; + xe->pat.idx[XE_CACHE_WT] = 2; + xe->pat.idx[XE_CACHE_WB] = 3; + } else if (xe->info.platform == XE_DG2) { + /* + * Table is the same as previous platforms, but programming + * method has changed. + */ + xe->pat.ops = &xehp_pat_ops; + xe->pat.table = xelp_pat_table; + xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); + xe->pat.idx[XE_CACHE_NONE] = 3; + xe->pat.idx[XE_CACHE_WT] = 2; + xe->pat.idx[XE_CACHE_WB] = 0; + } else if (GRAPHICS_VERx100(xe) <= 1210) { + WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); + xe->pat.ops = &xelp_pat_ops; + xe->pat.table = xelp_pat_table; + xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); + xe->pat.idx[XE_CACHE_NONE] = 3; + xe->pat.idx[XE_CACHE_WT] = 2; + xe->pat.idx[XE_CACHE_WB] = 0; + } else { + /* + * Going forward we expect to need new PAT settings for most + * new platforms; failure to provide a new table can easily + * lead to subtle, hard-to-debug problems. If none of the + * conditions above match the platform we're running on we'll + * raise an error rather than trying to silently inherit the + * most recent platform's behavior. + */ + drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", + GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); + } + + xe_assert(xe, xe->pat.ops->dump); + xe_assert(xe, xe->pat.ops->program_graphics); + xe_assert(xe, MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); + xe_assert(xe, GRAPHICS_VER(xe) < 20 || xe->pat.ops->entry_dump); +} + +void xe_pat_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (IS_SRIOV_VF(xe)) + return; + + if (xe_gt_is_media_type(gt)) + xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); + else + xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); +} + +/** + * xe_pat_dump() - Dump GT PAT table into a drm printer. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (IS_SRIOV_VF(xe)) + return -EOPNOTSUPP; + + return xe->pat.ops->dump(gt, p); +} + +/** + * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); + const struct xe_pat_table_entry *tr_pta_entry = gt_tr_pta_entry(gt); + char label[PAT_LABEL_LEN]; + + if (!xe->pat.table || !xe->pat.n_entries) + return -EOPNOTSUPP; + + drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : ""); + for (u32 i = 0; i < xe->pat.n_entries; i++) { + u32 pat = xe->pat.table[i].value; + + if (GRAPHICS_VER(xe) >= 20) { + xe_pat_index_label(label, sizeof(label), i); + xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); + } else if (xe->info.platform == XE_METEORLAKE) { + xelpg_pat_entry_dump(p, i, pat); + } else if (xe->info.platform == XE_PVC) { + xehpc_pat_entry_dump(p, i, pat); + } else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) { + xelp_pat_entry_dump(p, i, pat); + } else { + return -EOPNOTSUPP; + } + } + + if (pta_entry) { + u32 pat = pta_entry->value; + + drm_printf(p, "Page Table Access:\n"); + xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); + } + + if (tr_pta_entry) { + u32 pat = tr_pta_entry->value; + + drm_printf(p, "TRTT Page Table Access:\n"); + xe->pat.ops->entry_dump(p, "TR_PTA_MODE", pat, false); + } + + if (xe->pat.pat_ats) { + u32 pat = xe->pat.pat_ats->value; + + drm_printf(p, "PCIe ATS/PASID:\n"); + xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); + } + + drm_printf(p, "Cache Level:\n"); + drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]); + drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]); + drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]); + if (GRAPHICS_VER(xe) >= 20) { + drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n", + xe->pat.idx[XE_CACHE_NONE_COMPRESSION]); + drm_printf(p, "IDX[XE_CACHE_WB_COMPRESSION] = %d\n", + xe->pat.idx[XE_CACHE_WB_COMPRESSION]); + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h new file mode 100644 index 000000000..7060f66e1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_PAT_H_ +#define _XE_PAT_H_ + +#include + +struct drm_printer; +struct xe_device; +struct xe_gt; + +#define XE_PAT_INVALID_IDX U16_MAX + +/** + * struct xe_pat_table_entry - The pat_index encoding and other meta information. + */ +struct xe_pat_table_entry { + /** + * @value: The platform specific value encoding the various memory + * attributes (this maps to some fixed pat_index). So things like + * caching, coherency, compression etc can be encoded here. + */ + u32 value; + + /** + * @coh_mode: The GPU coherency mode that @value maps to. + */ +#define XE_COH_NONE 1 +#define XE_COH_1WAY 2 +#define XE_COH_2WAY 3 + u16 coh_mode; + + /** + * @valid: Set to 1 if the entry is valid, 0 if it's reserved. + */ + u16 valid; +}; + +/** + * xe_pat_init_early - SW initialization, setting up data based on device + * @xe: xe device + */ +void xe_pat_init_early(struct xe_device *xe); + +/** + * xe_pat_init - Program HW PAT table + * @gt: GT structure + */ +void xe_pat_init(struct xe_gt *gt); + +int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p); +int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p); + +/** + * xe_pat_index_get_coh_mode - Extract the coherency mode for the given + * pat_index. + * @xe: xe device + * @pat_index: The pat_index to query + */ +u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index); + +/** + * xe_pat_index_get_comp_en - Extract the compression enable flag for + * the given pat_index. + * @xe: xe device + * @pat_index: The pat_index to query + * + * Return: true if compression is enabled for this pat_index, false otherwise. + */ +bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index); + +#define XE_L3_POLICY_WB 0 /* Write-back */ +#define XE_L3_POLICY_XD 1 /* WB - Transient Display */ +#define XE_L3_POLICY_UC 3 /* Uncached */ +/** + * xe_pat_index_get_l3_policy - Extract the L3 policy for the given pat_index. + * @xe: xe device + * @pat_index: The pat_index to query + */ +u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index); + +#define xe_cache_pat_idx(xe, cache_mode) ({ \ + const struct xe_device *__xedev = (xe); \ + enum xe_cache_level __mode = (cache_mode); \ + xe_assert(__xedev, __mode < __XE_CACHE_LEVEL_COUNT); \ + xe_assert(__xedev, __xedev->pat.idx[__mode] != XE_PAT_INVALID_IDX); \ + __xedev->pat.idx[__mode]; \ +}) + +#endif diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c new file mode 100644 index 000000000..5f2a0b198 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -0,0 +1,1478 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_pci.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "display/xe_display.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_drv.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_vf.h" +#include "xe_guc.h" +#include "xe_mmio.h" +#include "xe_module.h" +#include "xe_pci_error.h" +#include "xe_pci_rebar.h" +#include "xe_pci_sriov.h" +#include "xe_pci_types.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_sriov.h" +#include "xe_step.h" +#include "xe_survivability_mode.h" +#include "xe_tile.h" +#include "xe_tile_printk.h" + +enum toggle_d3cold { + D3COLD_DISABLE, + D3COLD_ENABLE, +}; + +__diag_push(); +__diag_ignore_all("-Woverride-init", "Allow field overrides in table"); + +#define PLATFORM(x) \ + .platform = XE_##x, \ + .platform_name = #x + +#define NOP(x) x + +static const struct xe_graphics_desc graphics_xelp = { + .hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0), + .num_geometry_xecore_fuse_regs = 1, +}; + +#define XE_HP_FEATURES \ + .has_range_tlb_inval = true + +static const struct xe_graphics_desc graphics_xehpg = { + .hw_engine_mask = + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | + BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | + BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), + .num_geometry_xecore_fuse_regs = 1, + .num_compute_xecore_fuse_regs = 1, + + XE_HP_FEATURES, +}; + +static const struct xe_graphics_desc graphics_xehpc = { + .hw_engine_mask = + BIT(XE_HW_ENGINE_BCS0) | BIT(XE_HW_ENGINE_BCS1) | + BIT(XE_HW_ENGINE_BCS2) | BIT(XE_HW_ENGINE_BCS3) | + BIT(XE_HW_ENGINE_BCS4) | BIT(XE_HW_ENGINE_BCS5) | + BIT(XE_HW_ENGINE_BCS6) | BIT(XE_HW_ENGINE_BCS7) | + BIT(XE_HW_ENGINE_BCS8) | + BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | + BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), + + XE_HP_FEATURES, + + .has_access_counter = 1, + .has_asid = 1, + .has_atomic_enable_pte_bit = 1, + .has_usm = 1, + .num_compute_xecore_fuse_regs = 2, +}; + +static const struct xe_graphics_desc graphics_xelpg = { + .hw_engine_mask = + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | + BIT(XE_HW_ENGINE_CCS0), + .num_geometry_xecore_fuse_regs = 1, + .num_compute_xecore_fuse_regs = 1, + + XE_HP_FEATURES, +}; + +#define XE2_GFX_FEATURES \ + .has_access_counter = 1, \ + .has_asid = 1, \ + .has_atomic_enable_pte_bit = 1, \ + .has_range_tlb_inval = 1, \ + .has_usm = 1, \ + .has_64bit_timestamp = 1, \ + .hw_engine_mask = \ + BIT(XE_HW_ENGINE_RCS0) | \ + BIT(XE_HW_ENGINE_BCS8) | BIT(XE_HW_ENGINE_BCS0) | \ + GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0) + +static const struct xe_graphics_desc graphics_xe2 = { + XE2_GFX_FEATURES, + .num_geometry_xecore_fuse_regs = 3, + .num_compute_xecore_fuse_regs = 3, +}; + +static const struct xe_graphics_desc graphics_xe3p_lpg = { + XE2_GFX_FEATURES, + .has_indirect_ring_state = 1, + .has_uncorrectable_error_reporting = 1, + .multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | BIT(XE_ENGINE_CLASS_COMPUTE), + .num_geometry_xecore_fuse_regs = 3, + .num_compute_xecore_fuse_regs = 3, +}; + +static const struct xe_graphics_desc graphics_xe3p_xpc = { + XE2_GFX_FEATURES, + .has_access_counter = 0, + .has_indirect_ring_state = 1, + .has_uncorrectable_error_reporting = 1, + .hw_engine_mask = + GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) | + GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0), + .multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | + BIT(XE_ENGINE_CLASS_COMPUTE), + .num_geometry_xecore_fuse_regs = 4, + .num_compute_xecore_fuse_regs = 4, +}; + +static const struct xe_media_desc media_xem = { + .hw_engine_mask = + GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | + GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0), +}; + +static const struct xe_media_desc media_xelpmp = { + .hw_engine_mask = + GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | + GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) | + BIT(XE_HW_ENGINE_GSCCS0) +}; + +static const struct xe_media_desc media_xe3p_hpm = { + .has_uncorrectable_error_reporting = 1, + .hw_engine_mask = + GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | + GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) | + BIT(XE_HW_ENGINE_GSCCS0) +}; + +/* Pre-GMDID Graphics IPs */ +static const struct xe_ip graphics_ip_xelp = { 1200, "Xe_LP", &graphics_xelp }; +static const struct xe_ip graphics_ip_xelpp = { 1210, "Xe_LP+", &graphics_xelp }; +static const struct xe_ip graphics_ip_xehpg = { 1255, "Xe_HPG", &graphics_xehpg }; +static const struct xe_ip graphics_ip_xehpc = { 1260, "Xe_HPC", &graphics_xehpc }; + +/* GMDID-based Graphics IPs */ +static const struct xe_ip graphics_ips[] = { + { 1270, "Xe_LPG", &graphics_xelpg }, + { 1271, "Xe_LPG", &graphics_xelpg }, + { 1274, "Xe_LPG+", &graphics_xelpg }, + { 2001, "Xe2_HPG", &graphics_xe2 }, + { 2002, "Xe2_HPG", &graphics_xe2 }, + { 2004, "Xe2_LPG", &graphics_xe2 }, + { 3000, "Xe3_LPG", &graphics_xe2 }, + { 3001, "Xe3_LPG", &graphics_xe2 }, + { 3003, "Xe3_LPG", &graphics_xe2 }, + { 3004, "Xe3_LPG", &graphics_xe2 }, + { 3005, "Xe3_LPG", &graphics_xe2 }, + { 3510, "Xe3p_LPG", &graphics_xe3p_lpg }, + { 3511, "Xe3p_XPC", &graphics_xe3p_xpc }, +}; + +/* Pre-GMDID Media IPs */ +static const struct xe_ip media_ip_xem = { 1200, "Xe_M", &media_xem }; +static const struct xe_ip media_ip_xehpm = { 1255, "Xe_HPM", &media_xem }; + +/* GMDID-based Media IPs */ +static const struct xe_ip media_ips[] = { + { 1300, "Xe_LPM+", &media_xelpmp }, + { 1301, "Xe2_HPM", &media_xelpmp }, + { 2000, "Xe2_LPM", &media_xelpmp }, + { 3000, "Xe3_LPM", &media_xelpmp }, + { 3002, "Xe3_LPM", &media_xelpmp }, + { 3500, "Xe3p_LPM", &media_xelpmp }, + { 3503, "Xe3p_HPM", &media_xe3p_hpm }, +}; + +#define MULTI_LRC_MASK \ + .multi_lrc_mask = BIT(XE_ENGINE_CLASS_VIDEO_DECODE) | \ + BIT(XE_ENGINE_CLASS_VIDEO_ENHANCE) + +static const struct xe_device_desc tgl_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(TIGERLAKE), + .dma_mask_size = 39, + .has_cached_pt = true, + .has_display = true, + .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, +}; + +static const struct xe_device_desc rkl_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ROCKETLAKE), + .dma_mask_size = 39, + .has_cached_pt = true, + .has_display = true, + .has_llc = true, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, +}; + +static const u16 adls_rpls_ids[] = { INTEL_RPLS_IDS(NOP), 0 }; + +static const struct xe_device_desc adl_s_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ALDERLAKE_S), + .dma_mask_size = 39, + .has_cached_pt = true, + .has_display = true, + .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .subplatforms = (const struct xe_subplatform_desc[]) { + { XE_SUBPLATFORM_ALDERLAKE_S_RPLS, "RPLS", adls_rpls_ids }, + {}, + }, + .va_bits = 48, + .vm_max_level = 3, +}; + +static const u16 adlp_rplu_ids[] = { INTEL_RPLU_IDS(NOP), 0 }; + +static const struct xe_device_desc adl_p_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ALDERLAKE_P), + .dma_mask_size = 39, + .has_cached_pt = true, + .has_display = true, + .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .subplatforms = (const struct xe_subplatform_desc[]) { + { XE_SUBPLATFORM_ALDERLAKE_P_RPLU, "RPLU", adlp_rplu_ids }, + {}, + }, + .va_bits = 48, + .vm_max_level = 3, +}; + +static const struct xe_device_desc adl_n_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ALDERLAKE_N), + .dma_mask_size = 39, + .has_cached_pt = true, + .has_display = true, + .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, +}; + +#define DGFX_FEATURES \ + .is_dgfx = 1 + +static const struct xe_device_desc dg1_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xelpp, + .pre_gmdid_media_ip = &media_ip_xem, + DGFX_FEATURES, + PLATFORM(DG1), + .dma_mask_size = 39, + .has_display = true, + .has_gsc_nvm = 1, + .has_heci_gscfi = 1, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, +}; + +static const u16 dg2_g10_ids[] = { INTEL_DG2_G10_IDS(NOP), INTEL_ATS_M150_IDS(NOP), 0 }; +static const u16 dg2_g11_ids[] = { INTEL_DG2_G11_IDS(NOP), INTEL_ATS_M75_IDS(NOP), 0 }; +static const u16 dg2_g12_ids[] = { INTEL_DG2_G12_IDS(NOP), 0 }; + +#define DG2_FEATURES \ + DGFX_FEATURES, \ + PLATFORM(DG2), \ + .has_flat_ccs = 1, \ + .has_gsc_nvm = 1, \ + .has_heci_gscfi = 1, \ + .subplatforms = (const struct xe_subplatform_desc[]) { \ + { XE_SUBPLATFORM_DG2_G10, "G10", dg2_g10_ids }, \ + { XE_SUBPLATFORM_DG2_G11, "G11", dg2_g11_ids }, \ + { XE_SUBPLATFORM_DG2_G12, "G12", dg2_g12_ids }, \ + { } \ + }, \ + .va_bits = 48, \ + .vm_max_level = 3, \ + .vram_flags = XE_VRAM_FLAGS_NEED64K + +static const struct xe_device_desc ats_m_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xehpg, + .pre_gmdid_media_ip = &media_ip_xehpm, + .dma_mask_size = 46, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + + DG2_FEATURES, + .has_display = false, + .has_sriov = true, +}; + +static const struct xe_device_desc dg2_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xehpg, + .pre_gmdid_media_ip = &media_ip_xehpm, + .dma_mask_size = 46, + .max_gt_per_tile = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + + DG2_FEATURES, + .has_display = true, + .has_fan_control = true, + .has_mbx_power_limits = false, +}; + +static const __maybe_unused struct xe_device_desc pvc_desc = { + .pre_gmdid_graphics_ip = &graphics_ip_xehpc, + DGFX_FEATURES, + PLATFORM(PVC), + .dma_mask_size = 52, + .has_display = false, + .has_drm_ras = true, + .has_gsc_nvm = 1, + .has_heci_gscfi = 1, + .max_gt_per_tile = 1, + .max_remote_tiles = 1, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 57, + .vm_max_level = 4, + .vram_flags = XE_VRAM_FLAGS_NEED64K, + .has_mbx_power_limits = false, +}; + +static const struct xe_device_desc mtl_desc = { + /* .graphics and .media determined via GMD_ID */ + .require_force_probe = true, + PLATFORM(METEORLAKE), + .dma_mask_size = 46, + .has_display = true, + .has_pxp = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .va_bits = 48, + .vm_max_level = 3, +}; + +static const struct xe_device_desc lnl_desc = { + PLATFORM(LUNARLAKE), + .dma_mask_size = 46, + .has_display = true, + .has_flat_ccs = 1, + .has_pxp = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .needs_scratch = true, + .va_bits = 48, + .vm_max_level = 4, +}; + +static const u16 bmg_g21_ids[] = { INTEL_BMG_G21_IDS(NOP), 0 }; + +static const struct xe_device_desc bmg_desc = { + DGFX_FEATURES, + PLATFORM(BATTLEMAGE), + .dma_mask_size = 46, + .has_display = true, + .has_fan_control = true, + .has_flat_ccs = 1, + .has_mbx_power_limits = true, + .has_mbx_thermal_info = true, + .has_gsc_nvm = 1, + .has_heci_cscfi = 1, + .has_i2c = true, + .has_late_bind = true, + .has_pre_prod_wa = 1, + .has_soc_remapper_telem = true, + .has_sriov = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .needs_scratch = true, + .subplatforms = (const struct xe_subplatform_desc[]) { + { XE_SUBPLATFORM_BATTLEMAGE_G21, "G21", bmg_g21_ids }, + { } + }, + .va_bits = 48, + .vm_max_level = 4, +}; + +static const struct xe_device_desc ptl_desc = { + PLATFORM(PANTHERLAKE), + .dma_mask_size = 46, + .has_display = true, + .has_flat_ccs = 1, + .has_sriov = true, + .has_pre_prod_wa = 1, + .has_pxp = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .needs_scratch = true, + .needs_shared_vf_gt_wq = true, + .va_bits = 48, + .vm_max_level = 4, +}; + +static const struct xe_device_desc nvls_desc = { + PLATFORM(NOVALAKE_S), + .dma_mask_size = 46, + .has_display = true, + .has_flat_ccs = 1, + .has_pre_prod_wa = 1, + .has_sriov = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .va_bits = 48, + .vm_max_level = 4, +}; + +static const struct xe_device_desc cri_desc = { + DGFX_FEATURES, + PLATFORM(CRESCENTISLAND), + .dma_mask_size = 52, + .has_display = false, + .has_drm_ras = true, + .has_flat_ccs = false, + .has_gsc_nvm = 1, + .has_i2c = true, + .has_mbx_power_limits = true, + .has_mbx_thermal_info = true, + .has_mert = true, + .has_pre_prod_wa = 1, + .has_soc_remapper_sysctrl = true, + .has_soc_remapper_telem = true, + .has_sriov = true, + .has_sysctrl = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 57, + .vm_max_level = 4, +}; + +static const struct xe_device_desc nvlp_desc = { + PLATFORM(NOVALAKE_P), + .dma_mask_size = 46, + .has_cached_pt = true, + .has_display = true, + .has_flat_ccs = 1, + .has_page_reclaim_hw_assist = true, + .has_pre_prod_wa = true, + .has_sriov = true, + .max_gt_per_tile = 2, + MULTI_LRC_MASK, + .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 4, +}; + +#undef PLATFORM +__diag_pop(); + +/* + * Make sure any device matches here are from most specific to most + * general. For example, since the Quanta match is based on the subsystem + * and subvendor IDs, we need it to come before the more general IVB + * PCI ID matches, otherwise we'll use the wrong info struct above. + */ +static const struct pci_device_id pciidlist[] = { + INTEL_TGL_IDS(INTEL_VGA_DEVICE, &tgl_desc), + INTEL_RKL_IDS(INTEL_VGA_DEVICE, &rkl_desc), + INTEL_ADLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), + INTEL_ADLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + INTEL_ADLN_IDS(INTEL_VGA_DEVICE, &adl_n_desc), + INTEL_RPLU_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + INTEL_RPLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + INTEL_RPLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), + INTEL_DG1_IDS(INTEL_VGA_DEVICE, &dg1_desc), + INTEL_ATS_M_IDS(INTEL_VGA_DEVICE, &ats_m_desc), + INTEL_ARL_IDS(INTEL_VGA_DEVICE, &mtl_desc), + INTEL_DG2_IDS(INTEL_VGA_DEVICE, &dg2_desc), + INTEL_MTL_IDS(INTEL_VGA_DEVICE, &mtl_desc), + INTEL_LNL_IDS(INTEL_VGA_DEVICE, &lnl_desc), + INTEL_BMG_IDS(INTEL_VGA_DEVICE, &bmg_desc), + INTEL_PTL_IDS(INTEL_VGA_DEVICE, &ptl_desc), + INTEL_WCL_IDS(INTEL_VGA_DEVICE, &ptl_desc), + INTEL_NVLS_IDS(INTEL_VGA_DEVICE, &nvls_desc), + INTEL_CRI_IDS(INTEL_PCI_DEVICE, &cri_desc), + INTEL_NVLP_IDS(INTEL_VGA_DEVICE, &nvlp_desc), + { } +}; +MODULE_DEVICE_TABLE(pci, pciidlist); + +/* is device_id present in comma separated list of ids */ +static bool device_id_in_list(u16 device_id, const char *devices, bool negative) +{ + char *s, *p, *tok; + bool ret; + + if (!devices || !*devices) + return false; + + /* match everything */ + if (negative && strcmp(devices, "!*") == 0) + return true; + if (!negative && strcmp(devices, "*") == 0) + return true; + + s = kstrdup(devices, GFP_KERNEL); + if (!s) + return false; + + for (p = s, ret = false; (tok = strsep(&p, ",")) != NULL; ) { + u16 val; + + if (negative && tok[0] == '!') + tok++; + else if ((negative && tok[0] != '!') || + (!negative && tok[0] == '!')) + continue; + + if (kstrtou16(tok, 16, &val) == 0 && val == device_id) { + ret = true; + break; + } + } + + kfree(s); + + return ret; +} + +static bool id_forced(u16 device_id) +{ + return device_id_in_list(device_id, xe_modparam.force_probe, false); +} + +static bool id_blocked(u16 device_id) +{ + return device_id_in_list(device_id, xe_modparam.force_probe, true); +} + +static const struct xe_subplatform_desc * +find_subplatform(const struct xe_device_desc *desc, u16 devid) +{ + const struct xe_subplatform_desc *sp; + const u16 *id; + + for (sp = desc->subplatforms; sp && sp->subplatform; sp++) + for (id = sp->pciidlist; *id; id++) + if (*id == devid) + return sp; + + return NULL; +} + +enum xe_gmdid_type { + GMDID_GRAPHICS, + GMDID_MEDIA +}; + +static int read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u32 *revid) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + struct xe_reg gmdid_reg = GMD_ID; + u32 val; + + if (IS_SRIOV_VF(xe)) { + /* + * To get the value of the GMDID register, VFs must obtain it + * from the GuC using MMIO communication. + * + * Note that at this point the GTs are not initialized and only + * tile-level access to MMIO registers is possible. To use our + * existing GuC communication functions we must create a dummy + * GT structure and perform at least basic xe_gt and xe_guc + * initialization. + */ + struct xe_gt *gt __free(kfree) = NULL; + int err; + + /* Don't try to read media ver if media GT is not allowed */ + if (type == GMDID_MEDIA && !xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { + *ver = *revid = 0; + return 0; + } + + gt = kzalloc(sizeof(*gt), GFP_KERNEL); + if (!gt) + return -ENOMEM; + + gt->tile = &xe->tiles[0]; + if (type == GMDID_MEDIA) { + gt->info.id = 1; + gt->info.type = XE_GT_TYPE_MEDIA; + } else { + gt->info.id = 0; + gt->info.type = XE_GT_TYPE_MAIN; + } + + xe_gt_mmio_init(gt); + xe_guc_comm_init_early(>->uc.guc); + + err = xe_gt_sriov_vf_bootstrap(gt); + if (err) + return err; + + val = xe_gt_sriov_vf_gmdid(gt); + } else { + /* + * GMD_ID is a GT register, but at this point in the driver + * init we haven't fully initialized the GT yet so we need to + * read the register with the tile's MMIO accessor. That means + * we need to apply the GSI offset manually since it won't get + * automatically added as it would if we were using a GT mmio + * accessor. + */ + if (type == GMDID_MEDIA) + gmdid_reg.addr += MEDIA_GT_GSI_OFFSET; + + val = xe_mmio_read32(mmio, gmdid_reg); + } + + *ver = REG_FIELD_GET(GMD_ID_ARCH_MASK, val) * 100 + REG_FIELD_GET(GMD_ID_RELEASE_MASK, val); + *revid = REG_FIELD_GET(GMD_ID_REVID, val); + + return 0; +} + +static const struct xe_ip *find_graphics_ip(unsigned int verx100) +{ + KUNIT_STATIC_STUB_REDIRECT(find_graphics_ip, verx100); + + for (int i = 0; i < ARRAY_SIZE(graphics_ips); i++) + if (graphics_ips[i].verx100 == verx100) + return &graphics_ips[i]; + return NULL; +} + +static const struct xe_ip *find_media_ip(unsigned int verx100) +{ + KUNIT_STATIC_STUB_REDIRECT(find_media_ip, verx100); + + for (int i = 0; i < ARRAY_SIZE(media_ips); i++) + if (media_ips[i].verx100 == verx100) + return &media_ips[i]; + return NULL; +} + +/* + * Read IP version from hardware and select graphics/media IP descriptors + * based on the result. + */ +static int handle_gmdid(struct xe_device *xe, + const struct xe_ip **graphics_ip, + const struct xe_ip **media_ip, + u32 *graphics_revid, + u32 *media_revid) +{ + u32 ver; + int ret; + + *graphics_ip = NULL; + *media_ip = NULL; + + ret = read_gmdid(xe, GMDID_GRAPHICS, &ver, graphics_revid); + if (ret) + return ret; + + *graphics_ip = find_graphics_ip(ver); + if (!*graphics_ip) { + drm_err(&xe->drm, "Hardware reports unknown graphics version %u.%02u\n", + ver / 100, ver % 100); + } + + ret = read_gmdid(xe, GMDID_MEDIA, &ver, media_revid); + if (ret) + return ret; + + /* Media may legitimately be fused off / not present */ + if (ver == 0) + return 0; + + *media_ip = find_media_ip(ver); + if (!*media_ip) { + drm_err(&xe->drm, "Hardware reports unknown media version %u.%02u\n", + ver / 100, ver % 100); + } + + return 0; +} + +struct xe_probed_info { + u16 devid; + u8 revid; + u8 tile_count; + struct xe_step_info step; + const struct xe_ip *graphics_ip; + const struct xe_ip *media_ip; +}; + +/* + * Probe from the hardware the info required by xe_info_init_early(). + */ +static int xe_probe_info_early(struct xe_device *xe, + const struct xe_device_desc *desc, + struct xe_probed_info *probed_info) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + probed_info->devid = pdev->device; + probed_info->revid = pdev->revision; + + xe_step_platform_get(desc->platform, probed_info->revid, &probed_info->step); + + return 0; +} + +/* + * Initialize device info content that only depends on static driver_data + * passed to the driver at probe time from PCI ID table. + */ +static int xe_info_init_early(struct xe_device *xe, + const struct xe_device_desc *desc, + const struct xe_subplatform_desc *subplatform_desc, + struct xe_probed_info *probed_info) +{ + int err; + + xe->info.devid = probed_info->devid; + xe->info.revid = probed_info->revid; + xe->info.step.platform = probed_info->step.platform; + + xe->info.platform_name = desc->platform_name; + xe->info.platform = desc->platform; + xe->info.subplatform = subplatform_desc ? + subplatform_desc->subplatform : XE_SUBPLATFORM_NONE; + + xe->info.dma_mask_size = desc->dma_mask_size; + xe->info.va_bits = desc->va_bits; + xe->info.vm_max_level = desc->vm_max_level; + xe->info.vram_flags = desc->vram_flags; + + xe->info.is_dgfx = desc->is_dgfx; + xe->info.has_cached_pt = desc->has_cached_pt; + xe->info.has_drm_ras = desc->has_drm_ras; + xe->info.has_fan_control = desc->has_fan_control; + /* runtime fusing may force flat_ccs to disabled later */ + xe->info.has_flat_ccs = desc->has_flat_ccs; + xe->info.has_mbx_power_limits = desc->has_mbx_power_limits; + xe->info.has_mbx_thermal_info = desc->has_mbx_thermal_info; + xe->info.has_gsc_nvm = desc->has_gsc_nvm; + xe->info.has_heci_gscfi = desc->has_heci_gscfi; + xe->info.has_heci_cscfi = desc->has_heci_cscfi; + xe->info.has_i2c = desc->has_i2c; + xe->info.has_late_bind = desc->has_late_bind; + xe->info.has_llc = desc->has_llc; + xe->info.has_mert = desc->has_mert; + xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; + xe->info.has_pre_prod_wa = desc->has_pre_prod_wa; + xe->info.has_pxp = desc->has_pxp; + xe->info.has_soc_remapper_sysctrl = desc->has_soc_remapper_sysctrl; + xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem; + xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && + desc->has_sriov; + xe->info.has_sysctrl = desc->has_sysctrl; + xe->info.skip_guc_pc = desc->skip_guc_pc; + xe->info.skip_pcode = desc->skip_pcode; + xe->info.needs_scratch = desc->needs_scratch; + xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq; + xe->info.multi_lrc_mask = desc->multi_lrc_mask; + + xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) && + xe_modparam.probe_display && + desc->has_display && + !xe_device_is_admin_only(xe); + + xe_assert(xe, desc->max_gt_per_tile > 0); + xe_assert(xe, desc->max_gt_per_tile <= XE_MAX_GT_PER_TILE); + xe->info.max_gt_per_tile = desc->max_gt_per_tile; + + err = xe_tile_init_early(xe_device_get_root_tile(xe), xe, 0); + if (err) + return err; + + return 0; +} + +static void xe_probe_tile_count(struct xe_device *xe, + const struct xe_device_desc *desc, + struct xe_probed_info *probed_info) +{ + struct xe_mmio *mmio; + u8 tile_count; + u32 mtcfg; + + probed_info->tile_count = 1 + desc->max_remote_tiles; + + /* + * Probe for tile count only for platforms that support multiple + * tiles. + */ + if (probed_info->tile_count == 1) + return; + + mmio = xe_root_tile_mmio(xe); + + /* + * Although the per-tile mmio regs are not yet initialized, this + * is fine as it's going to the root tile's mmio, that's + * guaranteed to be initialized earlier in xe_mmio_probe_early() + */ + mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR); + tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; + + if (tile_count < probed_info->tile_count) { + drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", + probed_info->tile_count, tile_count); + probed_info->tile_count = tile_count; + } +} + +static struct xe_gt *alloc_primary_gt(struct xe_tile *tile, + const struct xe_graphics_desc *graphics_desc, + const struct xe_media_desc *media_desc) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_gt *gt; + + if (!xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev))) { + xe_tile_info(tile, "Primary GT disabled via configfs\n"); + return NULL; + } + + gt = xe_gt_alloc(tile); + if (IS_ERR(gt)) + return gt; + + gt->info.type = XE_GT_TYPE_MAIN; + gt->info.id = tile->id * xe->info.max_gt_per_tile; + gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state; + gt->info.has_uncorrectable_error_reporting = + graphics_desc->has_uncorrectable_error_reporting; + gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask; + if (!xe_configfs_get_enable_multi_queue(to_pci_dev(xe->drm.dev))) { + xe_gt_info(gt, "Multi-queue disabled via configfs\n"); + gt->info.multi_queue_engine_class_mask = 0; + } + gt->info.engine_mask = graphics_desc->hw_engine_mask; + gt->info.num_geometry_xecore_fuse_regs = graphics_desc->num_geometry_xecore_fuse_regs; + gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs; + + /* + * Even if the service copy engines wind up being fused off, their + * presence in the IP descriptor indicates that the platform supports + * Xe2-style MEM_SET and MEM_COPY functionality. + */ + if (graphics_desc->hw_engine_mask & GENMASK(XE_HW_ENGINE_BCS8, + XE_HW_ENGINE_BCS1)) + gt->info.has_xe2_blt_instructions = true; + + /* + * Before media version 13, the media IP was part of the primary GT + * so we need to add the media engines to the primary GT's engine list. + */ + if (MEDIA_VER(xe) < 13 && media_desc) + gt->info.engine_mask |= media_desc->hw_engine_mask; + + return gt; +} + +static struct xe_gt *alloc_media_gt(struct xe_tile *tile, + const struct xe_media_desc *media_desc) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_gt *gt; + + if (!xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { + xe_tile_info(tile, "Media GT disabled via configfs\n"); + return NULL; + } + + if (MEDIA_VER(xe) < 13 || !media_desc) + return NULL; + + gt = xe_gt_alloc(tile); + if (IS_ERR(gt)) + return gt; + + gt->info.type = XE_GT_TYPE_MEDIA; + gt->info.id = tile->id * xe->info.max_gt_per_tile + 1; + gt->info.has_indirect_ring_state = media_desc->has_indirect_ring_state; + gt->info.has_uncorrectable_error_reporting = media_desc->has_uncorrectable_error_reporting; + gt->info.engine_mask = media_desc->hw_engine_mask; + + return gt; +} + +static int xe_probe_ips(struct xe_device *xe, + const struct xe_device_desc *desc, + struct xe_probed_info *probed_info) +{ + /* + * If this platform supports GMD_ID, we'll detect the proper IP + * descriptor to use from hardware registers. + * desc->pre_gmdid_graphics_ip will only ever be set at this point for + * platforms before GMD_ID. In that case the IP descriptions and + * versions are simply derived from that. + */ + if (desc->pre_gmdid_graphics_ip) { + probed_info->graphics_ip = desc->pre_gmdid_graphics_ip; + probed_info->media_ip = desc->pre_gmdid_media_ip; + xe_step_pre_gmdid_get(xe, &probed_info->step); + } else { + int err; + u32 graphics_revid, media_revid; + + xe_assert(xe, !desc->pre_gmdid_media_ip); + + err = handle_gmdid(xe, &probed_info->graphics_ip, &probed_info->media_ip, + &graphics_revid, &media_revid); + if (err) + return err; + + xe_step_gmdid_get(xe, graphics_revid, media_revid, &probed_info->step); + } + + /* + * If we couldn't detect the graphics IP, that's considered a fatal + * error and we should abort driver load. Failing to detect media + * IP is non-fatal; we'll just proceed without enabling media support. + */ + if (!probed_info->graphics_ip) + return -ENODEV; + + return 0; +} + +/* + * Probe from the hardware the info required by xe_info_init(). + */ +static int xe_probe_info(struct xe_device *xe, + const struct xe_device_desc *desc, + struct xe_probed_info *probed_info) +{ + int err; + + xe_probe_tile_count(xe, desc, probed_info); + + err = xe_probe_ips(xe, desc, probed_info); + if (err) + return err; + + return 0; +} + +/* + * Initialize device info content that does require knowledge about + * graphics / media IP version. + * Make sure that GT / tile structures allocated by the driver match the data + * present in device info. + */ +static int xe_info_init(struct xe_device *xe, + const struct xe_device_desc *desc, + struct xe_probed_info *probed_info) +{ + const struct xe_ip *graphics_ip; + const struct xe_ip *media_ip; + const struct xe_graphics_desc *graphics_desc; + const struct xe_media_desc *media_desc; + struct xe_tile *tile; + struct xe_gt *gt; + u8 id; + + graphics_ip = probed_info->graphics_ip; + media_ip = probed_info->media_ip; + + xe->info.tile_count = probed_info->tile_count; + xe->info.step.basedie = probed_info->step.basedie; + xe->info.step.graphics = probed_info->step.graphics; + xe->info.step.media = probed_info->step.media; + + xe->info.graphics_verx100 = graphics_ip->verx100; + xe->info.graphics_name = graphics_ip->name; + graphics_desc = graphics_ip->desc; + + if (media_ip) { + xe->info.media_verx100 = media_ip->verx100; + xe->info.media_name = media_ip->name; + media_desc = media_ip->desc; + } else { + xe->info.media_name = "none"; + media_desc = NULL; + } + + xe->info.has_access_counter = graphics_desc->has_access_counter; + xe->info.has_asid = graphics_desc->has_asid; + xe->info.has_atomic_enable_pte_bit = graphics_desc->has_atomic_enable_pte_bit; + if (xe->info.platform != XE_PVC) + xe->info.has_device_atomics_on_smem = 1; + + xe->info.has_range_tlb_inval = graphics_desc->has_range_tlb_inval; + xe->info.has_ctx_tlb_inval = graphics_desc->has_ctx_tlb_inval; + xe->info.has_usm = graphics_desc->has_usm; + xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; + xe->info.has_mem_copy_instr = GRAPHICS_VER(xe) >= 20; + + if (IS_SRIOV_VF(xe)) { + xe->info.has_sysctrl = 0; + xe->info.has_soc_remapper_sysctrl = 0; + xe->info.has_soc_remapper_telem = 0; + } + + for_each_remote_tile(tile, xe, id) { + int err; + + err = xe_tile_init_early(tile, xe, id); + if (err) + return err; + } + + /* Allocate any GT and VRAM structures necessary for the platform. */ + for_each_tile(tile, xe, id) { + int err; + + err = xe_tile_alloc_vram(tile); + if (err) + return err; + + tile->primary_gt = alloc_primary_gt(tile, graphics_desc, media_desc); + if (IS_ERR(tile->primary_gt)) + return PTR_ERR(tile->primary_gt); + + /* + * It's not currently possible to probe a device with the + * primary GT disabled. With some work, this may be future in + * the possible for igpu platforms (although probably not for + * dgpu's since access to the primary GT's BCS engines is + * required for VRAM management). + */ + if (!tile->primary_gt) { + drm_err(&xe->drm, "Cannot probe device with without a primary GT\n"); + return -ENODEV; + } + + tile->media_gt = alloc_media_gt(tile, media_desc); + if (IS_ERR(tile->media_gt)) + return PTR_ERR(tile->media_gt); + } + + /* + * Now that we have tiles and GTs defined, let's loop over valid GTs + * in order to define gt_count. + */ + for_each_gt(gt, xe, id) + xe->info.gt_count++; + + return 0; +} + +static void xe_pci_remove(struct pci_dev *pdev) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + + if (IS_SRIOV_PF(xe)) + xe_pci_sriov_configure(pdev, 0); + + if (xe_survivability_mode_is_boot_enabled(xe)) + return; + + xe_device_remove(xe); + + /* + * Preserve remove-time flush after moving destroy work to module + * lifetime. + */ + xe_destroy_wq_flush(); + xe_pm_fini(xe); +} + +/* + * Probe the PCI device, initialize various parts of the driver. + * + * Fault injection is used to test the error paths of some initialization + * functions called either directly from xe_pci_probe() or indirectly for + * example through xe_device_probe(). Those functions use the kernel fault + * injection capabilities infrastructure, see + * Documentation/fault-injection/fault-injection.rst for details. The macro + * ALLOW_ERROR_INJECTION() is used to conditionally skip function execution + * at runtime and use a provided return value. The first requirement for + * error injectable functions is proper handling of the error code by the + * caller for recovery, which is always the case here. The second + * requirement is that no state is changed before the first error return. + * It is not strictly fulfilled for all initialization functions using the + * ALLOW_ERROR_INJECTION() macro but this is acceptable because for those + * error cases at probe time, the error code is simply propagated up by the + * caller. Therefore there is no consequence on those specific callers when + * function error injection skips the whole function. + */ +static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct xe_probed_info probed_info = {}; + const struct xe_device_desc *desc = (const void *)ent->driver_data; + const struct xe_subplatform_desc *subplatform_desc; + struct xe_device *xe; + void *group; + int err; + + subplatform_desc = find_subplatform(desc, pdev->device); + + xe_configfs_check_device(pdev); + + if (desc->require_force_probe && !id_forced(pdev->device)) { + dev_info(&pdev->dev, + "Your graphics device %04x is not officially supported\n" + "by xe driver in this kernel version. To force Xe probe,\n" + "use xe.force_probe='%04x' and i915.force_probe='!%04x'\n" + "module parameters or CONFIG_DRM_XE_FORCE_PROBE='%04x' and\n" + "CONFIG_DRM_I915_FORCE_PROBE='!%04x' configuration options.\n", + pdev->device, pdev->device, pdev->device, + pdev->device, pdev->device); + return -ENODEV; + } + + if (id_blocked(pdev->device)) { + dev_info(&pdev->dev, "Probe blocked for device [%04x:%04x].\n", + pdev->vendor, pdev->device); + return -ENODEV; + } + + if (xe_display_driver_probe_defer(pdev)) + return -EPROBE_DEFER; + + /* Group all devres so xe_pci_error_slot_reset() can release them as a unit. */ + group = devres_open_group(&pdev->dev, NULL, GFP_KERNEL); + if (!group) + return -ENOMEM; + + err = pcim_enable_device(pdev); + if (err) + return err; + + xe = xe_device_create(pdev); + if (IS_ERR(xe)) + return PTR_ERR(xe); + + xe->devres_group = group; + + pci_set_drvdata(pdev, &xe->drm); + + xe_pm_assert_unbounded_bridge(xe); + + pci_set_master(pdev); + + err = xe_probe_info_early(xe, desc, &probed_info); + if (err) + return err; + + err = xe_info_init_early(xe, desc, subplatform_desc, &probed_info); + if (err) + return err; + + xe_pci_rebar_resize(xe); + + err = xe_device_probe_early(xe); + /* + * In Boot Survivability mode, no drm card is exposed and driver + * is loaded with bare minimum to allow for firmware to be + * flashed through mei. Return success, if survivability mode + * is enabled due to pcode failure or configfs being set + */ + if (xe_survivability_mode_is_boot_enabled(xe)) + return 0; + + if (err) + return err; + + err = xe_probe_info(xe, desc, &probed_info); + if (err) + return err; + + err = xe_info_init(xe, desc, &probed_info); + if (err) + return err; + + err = xe_display_probe(xe); + if (err) + return err; + + drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx:%s (%d.%02d) media:%s (%d.%02d) display:%s dma_m_s:%d tc:%d gscfi:%d cscfi:%d", + desc->platform_name, + subplatform_desc ? subplatform_desc->name : "", + xe->info.devid, xe->info.revid, + xe->info.is_dgfx, + xe->info.graphics_name, + xe->info.graphics_verx100 / 100, + xe->info.graphics_verx100 % 100, + xe->info.media_name, + xe->info.media_verx100 / 100, + xe->info.media_verx100 % 100, + str_yes_no(xe->info.probe_display), + xe->info.dma_mask_size, xe->info.tile_count, + xe->info.has_heci_gscfi, xe->info.has_heci_cscfi); + + drm_dbg(&xe->drm, "Stepping = (G:%s, M:%s, B:%s)\n", + xe_step_name(xe->info.step.graphics), + xe_step_name(xe->info.step.media), + xe_step_name(xe->info.step.basedie)); + + drm_dbg(&xe->drm, "SR-IOV support: %s (mode: %s)\n", + str_yes_no(xe_device_has_sriov(xe)), + xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); + + err = xe_pm_probe(xe); + if (err) + return err; + + err = xe_device_probe(xe); + if (err) + return err; + + err = xe_pm_init(xe); + if (err) + goto err_driver_cleanup; + + return 0; + +err_driver_cleanup: + xe_pci_remove(pdev); + return err; +} + +static void xe_pci_shutdown(struct pci_dev *pdev) +{ + xe_device_shutdown(pdev_to_xe_device(pdev)); +} + +#ifdef CONFIG_PM_SLEEP +static void d3cold_toggle(struct pci_dev *pdev, enum toggle_d3cold toggle) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold.capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + switch (toggle) { + case D3COLD_DISABLE: + pci_d3cold_disable(root_pdev); + break; + case D3COLD_ENABLE: + pci_d3cold_enable(root_pdev); + break; + } +} + +static int xe_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + int err; + + if (xe_survivability_mode_is_boot_enabled(xe)) + return -EBUSY; + + err = xe_pm_suspend(xe); + if (err) + return err; + + /* + * Enabling D3Cold is needed for S2Idle/S0ix. + * It is save to allow here since xe_pm_suspend has evicted + * the local memory and the direct complete optimization is disabled. + */ + d3cold_toggle(pdev, D3COLD_ENABLE); + + pci_save_state(pdev); + pci_disable_device(pdev); + pci_set_power_state(pdev, PCI_D3cold); + + return 0; +} + +static int xe_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int err; + + /* Give back the D3Cold decision to the runtime P M*/ + d3cold_toggle(pdev, D3COLD_DISABLE); + + err = pci_set_power_state(pdev, PCI_D0); + if (err) + return err; + + pci_restore_state(pdev); + + err = pci_enable_device(pdev); + if (err) + return err; + + pci_set_master(pdev); + + err = xe_pm_resume(pdev_to_xe_device(pdev)); + if (err) + return err; + + return 0; +} + +static int xe_pci_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + int err; + + /* + * We hold an additional reference to the runtime PM to keep PF in D0 + * during VFs lifetime, as our VFs do not implement the PM capability. + * This means we should never be runtime suspending as long as VFs are + * enabled. + */ + xe_assert(xe, !IS_SRIOV_VF(xe)); + xe_assert(xe, !pci_num_vf(pdev)); + + err = xe_pm_runtime_suspend(xe); + if (err) + return err; + + pci_save_state(pdev); + + if (xe->d3cold.allowed) { + d3cold_toggle(pdev, D3COLD_ENABLE); + pci_disable_device(pdev); + pci_ignore_hotplug(pdev); + pci_set_power_state(pdev, PCI_D3cold); + } else { + d3cold_toggle(pdev, D3COLD_DISABLE); + pci_set_power_state(pdev, PCI_D3hot); + } + + return 0; +} + +static int xe_pci_runtime_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + int err; + + err = pci_set_power_state(pdev, PCI_D0); + if (err) + return err; + + pci_restore_state(pdev); + + if (xe->d3cold.allowed) { + err = pci_enable_device(pdev); + if (err) + return err; + + pci_set_master(pdev); + } + + return xe_pm_runtime_resume(xe); +} + +static int xe_pci_runtime_idle(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + + xe_pm_d3cold_allowed_toggle(xe); + + return 0; +} + +static const struct dev_pm_ops xe_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(xe_pci_suspend, xe_pci_resume) + SET_RUNTIME_PM_OPS(xe_pci_runtime_suspend, xe_pci_runtime_resume, xe_pci_runtime_idle) +}; +#endif + +static struct pci_driver xe_pci_driver = { + .name = DRIVER_NAME, + .id_table = pciidlist, + .probe = xe_pci_probe, + .remove = xe_pci_remove, + .shutdown = xe_pci_shutdown, + .sriov_configure = xe_pci_sriov_configure, + .err_handler = &xe_pci_error_handlers, +#ifdef CONFIG_PM_SLEEP + .driver.pm = &xe_pm_ops, +#endif +}; + +/** + * xe_pci_to_pf_device() - Get PF &xe_device. + * @pdev: the VF &pci_dev device + * + * Return: pointer to PF &xe_device, NULL otherwise. + */ +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev) +{ + struct drm_device *drm; + + drm = pci_iov_get_pf_drvdata(pdev, &xe_pci_driver); + if (IS_ERR(drm)) + return NULL; + + return to_xe_device(drm); +} + +int xe_register_pci_driver(void) +{ + return pci_register_driver(&xe_pci_driver); +} + +void xe_unregister_pci_driver(void) +{ + pci_unregister_driver(&xe_pci_driver); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_pci.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_pci.h b/drivers/gpu/drm/xe/xe_pci.h new file mode 100644 index 000000000..11bcc5fe2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_PCI_H_ +#define _XE_PCI_H_ + +struct pci_dev; + +int xe_register_pci_driver(void); +void xe_unregister_pci_driver(void); +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pci_error.c b/drivers/gpu/drm/xe/xe_pci_error.c new file mode 100644 index 000000000..e41af2ac7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_error.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_pci.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_survivability_mode.h" + +static void prepare_device_for_reset(struct pci_dev *pdev) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_gt *gt; + u8 id; + + /* + * Wedge the device to prevent userspace access but do not send the uevent. + * xe_device_wedged_fini() releases runtime pm if wedged flag is set, so acquire a runtime + * pm reference to avoid underflow. + */ + if (!atomic_xchg(&xe->wedged.flag, 1)) + xe_pm_runtime_get_noresume(xe); + + xe_device_set_in_reset(xe); + + for_each_gt(gt, xe, id) + xe_gt_declare_wedged(gt); + + pci_disable_device(pdev); +} + +static pci_ers_result_t ras_action_to_pci_result(struct pci_dev *pdev, u8 action) +{ + switch (action) { + case XE_RAS_RECOVERY_ACTION_RECOVERED: + return PCI_ERS_RESULT_RECOVERED; + case XE_RAS_RECOVERY_ACTION_RESET: + prepare_device_for_reset(pdev); + return PCI_ERS_RESULT_NEED_RESET; + case XE_RAS_RECOVERY_ACTION_DISCONNECT: + return PCI_ERS_RESULT_DISCONNECT; + default: + return PCI_ERS_RESULT_DISCONNECT; + } +} + +static pci_ers_result_t xe_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + + xe_info(xe, "PCI error: detected state = %d\n", state); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + /* If the device is already wedged or in survivability mode, do not attempt recovery */ + if (xe_survivability_mode_is_boot_enabled(xe) || xe_device_wedged(xe)) + return PCI_ERS_RESULT_DISCONNECT; + + switch (state) { + case pci_channel_io_normal: + return PCI_ERS_RESULT_CAN_RECOVER; + case pci_channel_io_frozen: + prepare_device_for_reset(pdev); + return PCI_ERS_RESULT_NEED_RESET; + default: + xe_info(xe, "PCI error: unknown state %d\n", state); + return PCI_ERS_RESULT_DISCONNECT; + } +} + +static pci_ers_result_t xe_pci_error_mmio_enabled(struct pci_dev *pdev) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + enum xe_ras_recovery_action action; + + xe_info(xe, "PCI error: MMIO enabled\n"); + action = xe_ras_process_errors(xe); + + return ras_action_to_pci_result(pdev, action); +} + +static pci_ers_result_t xe_pci_error_slot_reset(struct pci_dev *pdev) +{ + const struct pci_device_id *ent = pci_match_id(pdev->driver->id_table, pdev); + struct xe_device *xe = pdev_to_xe_device(pdev); + + xe_info(xe, "PCI error: slot reset\n"); + + pci_restore_state(pdev); + + if (pci_enable_device(pdev)) { + xe_err(xe, "Cannot re-enable PCI device after reset\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + /* + * Secondary Bus Reset causes all VRAM state to be lost along with + * hardware state. As an initial step, re-probe the device to + * re-initialize the driver and hardware. + * TODO: optimize by re-initializing only the hardware state and re-creating + * kernel BOs. + */ + xe_device_clear_in_reset(xe); + pdev->driver->remove(pdev); + devres_release_group(&pdev->dev, xe->devres_group); + + if (pdev->driver->probe(pdev, ent)) + return PCI_ERS_RESULT_DISCONNECT; + + xe = pdev_to_xe_device(pdev); + + /* Wedge the device to prevent I/O operations till the resume callback */ + atomic_set(&xe->wedged.flag, 1); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void xe_pci_error_resume(struct pci_dev *pdev) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + + xe_info(xe, "PCI error: resume\n"); + + atomic_set(&xe->wedged.flag, 0); +} + +const struct pci_error_handlers xe_pci_error_handlers = { + .error_detected = xe_pci_error_detected, + .mmio_enabled = xe_pci_error_mmio_enabled, + .slot_reset = xe_pci_error_slot_reset, + .resume = xe_pci_error_resume, +}; diff --git a/drivers/gpu/drm/xe/xe_pci_error.h b/drivers/gpu/drm/xe/xe_pci_error.h new file mode 100644 index 000000000..725ad0214 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_error.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_PCI_ERROR_H_ +#define _XE_PCI_ERROR_H_ + +struct pci_error_handlers; + +extern const struct pci_error_handlers xe_pci_error_handlers; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pci_rebar.c b/drivers/gpu/drm/xe/xe_pci_rebar.c new file mode 100644 index 000000000..7e2c7079b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_rebar.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "regs/xe_bars.h" +#include "xe_device_types.h" +#include "xe_module.h" +#include "xe_pci_rebar.h" +#include "xe_printk.h" + +static void resize_bar(struct xe_device *xe, int resno, resource_size_t size) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int bar_size = pci_rebar_bytes_to_size(size); + int ret; + + ret = pci_resize_resource(pdev, resno, bar_size, 0); + if (ret) { + xe_info(xe, "Failed to resize BAR%d to %dMiB (%pe). Consider enabling 'Resizable BAR' support in your BIOS\n", + resno, 1 << bar_size, ERR_PTR(ret)); + return; + } + + xe_info(xe, "BAR%d resized to %dMiB\n", resno, 1 << bar_size); +} + +/* + * xe_pci_rebar_resize - Resize the LMEMBAR + * @xe: xe device instance + * + * If vram_bar_size module param is set, attempt to set to the requested size + * else set to maximum possible size. + */ +void xe_pci_rebar_resize(struct xe_device *xe) +{ + int force_vram_bar_size = xe_modparam.force_vram_bar_size; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct pci_bus *root = pdev->bus; + resource_size_t current_size; + resource_size_t rebar_size; + struct resource *root_res; + int max_size, i; + u32 pci_cmd; + + /* gather some relevant info */ + current_size = pci_resource_len(pdev, LMEM_BAR); + + if (force_vram_bar_size < 0) + return; + + /* set to a specific size? */ + if (force_vram_bar_size) { + rebar_size = pci_rebar_bytes_to_size(force_vram_bar_size * + (resource_size_t)SZ_1M); + + if (!pci_rebar_size_supported(pdev, LMEM_BAR, rebar_size)) { + xe_info(xe, "Requested size %lluMiB is not supported by rebar sizes: 0x%llx. Leaving default: %lluMiB\n", + (u64)pci_rebar_size_to_bytes(rebar_size) >> ilog2(SZ_1M), + pci_rebar_get_possible_sizes(pdev, LMEM_BAR), + (u64)current_size >> ilog2(SZ_1M)); + return; + } + + rebar_size = pci_rebar_size_to_bytes(rebar_size); + if (rebar_size == current_size) + return; + } else { + max_size = pci_rebar_get_max_size(pdev, LMEM_BAR); + if (max_size < 0) + return; + rebar_size = pci_rebar_size_to_bytes(max_size); + + /* only resize if larger than current */ + if (rebar_size <= current_size) + return; + } + + xe_info(xe, "Attempting to resize bar from %lluMiB -> %lluMiB\n", + (u64)current_size >> ilog2(SZ_1M), (u64)rebar_size >> ilog2(SZ_1M)); + + while (root->parent) + root = root->parent; + + pci_bus_for_each_resource(root, root_res, i) { + if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && + (u64)root_res->start > 0x100000000ul) + break; + } + + if (!root_res) { + xe_info(xe, "Can't resize VRAM BAR - platform support is missing. Consider enabling 'Resizable BAR' support in your BIOS\n"); + return; + } + + pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); + + resize_bar(xe, LMEM_BAR, rebar_size); + + pci_assign_unassigned_bus_resources(pdev->bus); + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); +} diff --git a/drivers/gpu/drm/xe/xe_pci_rebar.h b/drivers/gpu/drm/xe/xe_pci_rebar.h new file mode 100644 index 000000000..8677921ac --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_rebar.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PCI_REBAR_H_ +#define _XE_PCI_REBAR_H_ + +struct xe_device; + +void xe_pci_rebar_resize(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c new file mode 100644 index 000000000..35e6b53e1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include + +#include "regs/xe_bars.h" +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_printk.h" +#include "xe_guc_engine_activity.h" +#include "xe_pci_sriov.h" +#include "xe_pm.h" +#include "xe_sriov.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_provision.h" +#include "xe_sriov_pf_sysfs.h" +#include "xe_sriov_printk.h" + +static void pf_reset_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + unsigned int n; + + for (n = 1; n <= num_vfs; n++) + xe_sriov_pf_control_reset_vf(xe, n); +} + +static void pf_link_vfs(struct xe_device *xe, int num_vfs) +{ + struct pci_dev *pdev_pf = to_pci_dev(xe->drm.dev); + struct device_link *link; + struct pci_dev *pdev_vf; + unsigned int n; + + /* + * When both PF and VF devices are enabled on the host, during system + * resume they are resuming in parallel. + * + * But PF has to complete the provision of VF first to allow any VFs to + * successfully resume. + * + * Create a parent-child device link between PF and VF devices that will + * enforce correct resume order. + */ + for (n = 1; n <= num_vfs; n++) { + pdev_vf = xe_pci_sriov_get_vf_pdev(pdev_pf, n); + + /* unlikely, something weird is happening, abort */ + if (!pdev_vf) { + xe_sriov_err(xe, "Cannot find VF%u device, aborting link%s creation!\n", + n, str_plural(num_vfs)); + break; + } + + link = device_link_add(&pdev_vf->dev, &pdev_pf->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + /* unlikely and harmless, continue with other VFs */ + if (!link) + xe_sriov_notice(xe, "Failed linking VF%u\n", n); + + pci_dev_put(pdev_vf); + } +} + +static void pf_engine_activity_stats(struct xe_device *xe, unsigned int num_vfs, bool enable) +{ + struct xe_gt *gt; + unsigned int id; + int ret = 0; + + for_each_gt(gt, xe, id) { + ret = xe_guc_engine_activity_function_stats(>->uc.guc, num_vfs, enable); + if (ret) + xe_gt_sriov_info(gt, "Failed to %s engine activity function stats (%pe)\n", + str_enable_disable(enable), ERR_PTR(ret)); + } +} + +static int resize_vf_vram_bar(struct xe_device *xe, int num_vfs) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + u32 sizes; + + sizes = pci_iov_vf_bar_get_sizes(pdev, VF_LMEM_BAR, num_vfs); + if (!sizes) + return 0; + + return pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, __fls(sizes)); +} + +static int pf_prepare_vfs_enabling(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + /* make sure we are not locked-down by other components */ + return xe_sriov_pf_arm_guard(xe, &xe->sriov.pf.guard_vfs_enabling, false, NULL); +} + +static void pf_finish_vfs_enabling(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + /* allow other components to lockdown VFs enabling */ + xe_sriov_pf_disarm_guard(xe, &xe->sriov.pf.guard_vfs_enabling, false, NULL); +} + +static int pf_enable_vfs(struct xe_device *xe, int num_vfs) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int total_vfs = xe_sriov_pf_get_totalvfs(xe); + int err; + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, num_vfs > 0); + xe_assert(xe, num_vfs <= total_vfs); + xe_sriov_dbg(xe, "enabling %u VF%s\n", num_vfs, str_plural(num_vfs)); + + err = xe_sriov_pf_wait_ready(xe); + if (err) + goto out; + + err = pf_prepare_vfs_enabling(xe); + if (err) + goto out; + + /* + * We must hold additional reference to the runtime PM to keep PF in D0 + * during VFs lifetime, as our VFs do not implement the PM capability. + * + * With PF being in D0 state, all VFs will also behave as in D0 state. + * This will also keep GuC alive with all VFs' configurations. + * + * We will release this additional PM reference in pf_disable_vfs(). + */ + xe_pm_runtime_get_noresume(xe); + + err = xe_sriov_pf_provision_vfs(xe, num_vfs); + if (err < 0) + goto failed; + + if (IS_DGFX(xe)) { + err = resize_vf_vram_bar(xe, num_vfs); + if (err) + xe_sriov_info(xe, "Failed to set VF LMEM BAR size: %d\n", err); + } + + err = pci_enable_sriov(pdev, num_vfs); + if (err < 0) + goto failed; + + pf_link_vfs(xe, num_vfs); + + xe_sriov_info(xe, "Enabled %u of %u VF%s\n", + num_vfs, total_vfs, str_plural(total_vfs)); + + xe_sriov_pf_sysfs_link_vfs(xe, num_vfs); + + pf_engine_activity_stats(xe, num_vfs, true); + + return num_vfs; + +failed: + xe_sriov_pf_unprovision_vfs(xe, num_vfs); + xe_pm_runtime_put(xe); + pf_finish_vfs_enabling(xe); +out: + xe_sriov_notice(xe, "Failed to enable %u VF%s (%pe)\n", + num_vfs, str_plural(num_vfs), ERR_PTR(err)); + return err; +} + +static int pf_disable_vfs(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + struct pci_dev *pdev = to_pci_dev(dev); + u16 num_vfs = pci_num_vf(pdev); + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_sriov_dbg(xe, "disabling %u VF%s\n", num_vfs, str_plural(num_vfs)); + + if (!num_vfs) + return 0; + + pf_engine_activity_stats(xe, num_vfs, false); + + xe_sriov_pf_sysfs_unlink_vfs(xe, num_vfs); + + pci_disable_sriov(pdev); + + xe_sriov_pf_reprovision_default(xe); + + pf_reset_vfs(xe, num_vfs); + + xe_sriov_pf_unprovision_vfs(xe, num_vfs); + + /* not needed anymore - see pf_enable_vfs() */ + xe_pm_runtime_put(xe); + + pf_finish_vfs_enabling(xe); + + xe_sriov_info(xe, "Disabled %u VF%s\n", num_vfs, str_plural(num_vfs)); + return 0; +} + +/** + * xe_pci_sriov_configure - Configure SR-IOV (enable/disable VFs). + * @pdev: the &pci_dev + * @num_vfs: number of VFs to enable or zero to disable all VFs + * + * This is the Xe implementation of struct pci_driver.sriov_configure callback. + * + * This callback will be called by the PCI subsystem to enable or disable SR-IOV + * Virtual Functions (VFs) as requested by the used via the PCI sysfs interface. + * + * Return: number of configured VFs or a negative error code on failure. + */ +int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + + if (!IS_SRIOV_PF(xe)) + return -ENODEV; + + if (num_vfs < 0) + return -EINVAL; + + if (num_vfs > xe_sriov_pf_get_totalvfs(xe)) + return -ERANGE; + + if (num_vfs && pci_num_vf(pdev)) + return -EBUSY; + + guard(xe_pm_runtime)(xe); + if (num_vfs > 0) + return pf_enable_vfs(xe, num_vfs); + else + return pf_disable_vfs(xe); +} + +/** + * xe_pci_sriov_get_vf_pdev() - Lookup the VF's PCI device using the VF identifier. + * @pdev: the PF's &pci_dev + * @vfid: VF identifier (1-based) + * + * The caller must decrement the reference count by calling pci_dev_put(). + * + * Return: the VF's &pci_dev or NULL if the VF device was not found. + */ +struct pci_dev *xe_pci_sriov_get_vf_pdev(struct pci_dev *pdev, unsigned int vfid) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + + xe_assert(xe, dev_is_pf(&pdev->dev)); + xe_assert(xe, vfid); + xe_assert(xe, vfid <= pci_sriov_get_totalvfs(pdev)); + + return pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus), + pdev->bus->number, + pci_iov_virtfn_devfn(pdev, vfid - 1)); +} diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.h b/drivers/gpu/drm/xe/xe_pci_sriov.h new file mode 100644 index 000000000..b9105d71d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_sriov.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_PCI_SRIOV_H_ +#define _XE_PCI_SRIOV_H_ + +struct pci_dev; + +#ifdef CONFIG_PCI_IOV +int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs); +struct pci_dev *xe_pci_sriov_get_vf_pdev(struct pci_dev *pdev, unsigned int vfid); +#else +static inline int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h new file mode 100644 index 000000000..fed509ff6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_PCI_TYPES_H_ +#define _XE_PCI_TYPES_H_ + +#include + +#include "xe_platform_types.h" + +struct xe_subplatform_desc { + enum xe_subplatform subplatform; + const char *name; + const u16 *pciidlist; +}; + +struct xe_device_desc { + /* Should only ever be set for platforms without GMD_ID */ + const struct xe_ip *pre_gmdid_graphics_ip; + /* Should only ever be set for platforms without GMD_ID */ + const struct xe_ip *pre_gmdid_media_ip; + + const char *platform_name; + const struct xe_subplatform_desc *subplatforms; + + enum xe_platform platform; + + u8 dma_mask_size; + u8 max_remote_tiles:2; + u8 max_gt_per_tile:2; + u8 multi_lrc_mask; + u8 va_bits; + u8 vm_max_level; + u8 vram_flags; + + u8 require_force_probe:1; + u8 is_dgfx:1; + + u8 has_cached_pt:1; + u8 has_display:1; + u8 has_drm_ras:1; + u8 has_fan_control:1; + u8 has_flat_ccs:1; + u8 has_gsc_nvm:1; + u8 has_heci_gscfi:1; + u8 has_heci_cscfi:1; + u8 has_i2c:1; + u8 has_late_bind:1; + u8 has_llc:1; + u8 has_mbx_power_limits:1; + u8 has_mbx_thermal_info:1; + u8 has_mert:1; + u8 has_pre_prod_wa:1; + u8 has_page_reclaim_hw_assist:1; + u8 has_pxp:1; + u8 has_soc_remapper_sysctrl:1; + u8 has_soc_remapper_telem:1; + u8 has_sriov:1; + u8 has_sysctrl:1; + u8 needs_scratch:1; + u8 skip_guc_pc:1; + u8 skip_pcode:1; + u8 needs_shared_vf_gt_wq:1; +}; + +struct xe_graphics_desc { + u64 hw_engine_mask; /* hardware engines provided by graphics IP */ + u16 multi_queue_engine_class_mask; /* bitmask of engine classes which support multi queue */ + u8 num_geometry_xecore_fuse_regs; + u8 num_compute_xecore_fuse_regs; + + u8 has_access_counter:1; + u8 has_asid:1; + u8 has_atomic_enable_pte_bit:1; + u8 has_indirect_ring_state:1; + u8 has_range_tlb_inval:1; + u8 has_ctx_tlb_inval:1; + u8 has_usm:1; + u8 has_64bit_timestamp:1; + u8 has_uncorrectable_error_reporting:1; +}; + +struct xe_media_desc { + u64 hw_engine_mask; /* hardware engines provided by media IP */ + + u8 has_indirect_ring_state:1; + u8 has_uncorrectable_error_reporting:1; +}; + +struct xe_ip { + unsigned int verx100; + const char *name; + const void *desc; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c new file mode 100644 index 000000000..ccc3bdeed --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pcode.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_pcode.h" + +#include +#include +#include + +#include + +#include "regs/xe_pmt.h" +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pcode_api.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_vsec.h" + +/** + * DOC: PCODE + * + * Xe PCODE is the component responsible for interfacing with the PCODE + * firmware. + * It shall provide a very simple ABI to other Xe components, but be the + * single and consolidated place that will communicate with PCODE. All read + * and write operations to PCODE will be internal and private to this component. + * + * What's next: + * - PCODE hw metrics + * - PCODE for display operations + */ + +static int pcode_mailbox_status(struct xe_tile *tile) +{ + const char *err_str; + int err_decode; + u32 err; + +#define CASE_ERR(_err, _err_decode, _err_str) \ + case _err: \ + err_decode = _err_decode; \ + err_str = _err_str; \ + break + + err = xe_mmio_read32(&tile->mmio, PCODE_MAILBOX) & PCODE_ERROR_MASK; + switch (err) { + CASE_ERR(PCODE_ILLEGAL_CMD, -ENXIO, "Illegal Command"); + CASE_ERR(PCODE_TIMEOUT, -ETIMEDOUT, "Timed out"); + CASE_ERR(PCODE_ILLEGAL_DATA, -EINVAL, "Illegal Data"); + CASE_ERR(PCODE_ILLEGAL_SUBCOMMAND, -ENXIO, "Illegal Subcommand"); + CASE_ERR(PCODE_LOCKED, -EBUSY, "PCODE Locked"); + CASE_ERR(PCODE_GT_RATIO_OUT_OF_RANGE, -EOVERFLOW, "GT ratio out of range"); + CASE_ERR(PCODE_REJECTED, -EACCES, "PCODE Rejected"); + default: + err_decode = -EPROTO; + err_str = "Unknown"; + } + + if (err) { + drm_err(&tile_to_xe(tile)->drm, "PCODE Mailbox failed: %d %s", + err_decode, err_str); + + return err_decode; + } + + return 0; +#undef CASE_ERR +} + +static int __pcode_mailbox_rw(struct xe_tile *tile, u32 mbox, u32 *data0, u32 *data1, + unsigned int timeout_ms, bool return_data, + bool atomic) +{ + struct xe_mmio *mmio = &tile->mmio; + int err; + + if (tile_to_xe(tile)->info.skip_pcode) + return 0; + + if ((xe_mmio_read32(mmio, PCODE_MAILBOX) & PCODE_READY) != 0) + return -EAGAIN; + + xe_mmio_write32(mmio, PCODE_DATA0, *data0); + xe_mmio_write32(mmio, PCODE_DATA1, data1 ? *data1 : 0); + xe_mmio_write32(mmio, PCODE_MAILBOX, PCODE_READY | mbox); + + err = xe_mmio_wait32(mmio, PCODE_MAILBOX, PCODE_READY, 0, + timeout_ms * USEC_PER_MSEC, NULL, atomic); + if (err) + return err; + + if (return_data) { + *data0 = xe_mmio_read32(mmio, PCODE_DATA0); + if (data1) + *data1 = xe_mmio_read32(mmio, PCODE_DATA1); + } + + return pcode_mailbox_status(tile); +} + +static int pcode_mailbox_rw(struct xe_tile *tile, u32 mbox, u32 *data0, u32 *data1, + unsigned int timeout_ms, bool return_data, + bool atomic) +{ + if (tile_to_xe(tile)->info.skip_pcode) + return 0; + + lockdep_assert_held(&tile->pcode.lock); + + return __pcode_mailbox_rw(tile, mbox, data0, data1, timeout_ms, return_data, atomic); +} + +int xe_pcode_write_timeout(struct xe_tile *tile, u32 mbox, u32 data, int timeout) +{ + int err; + + mutex_lock(&tile->pcode.lock); + err = pcode_mailbox_rw(tile, mbox, &data, NULL, timeout, false, false); + mutex_unlock(&tile->pcode.lock); + + return err; +} + +int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0, u32 data1, int timeout) +{ + int err; + + mutex_lock(&tile->pcode.lock); + err = pcode_mailbox_rw(tile, mbox, &data0, &data1, timeout, false, false); + mutex_unlock(&tile->pcode.lock); + + return err; +} + +int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1) +{ + int err; + + mutex_lock(&tile->pcode.lock); + err = pcode_mailbox_rw(tile, mbox, val, val1, 1, true, false); + mutex_unlock(&tile->pcode.lock); + + return err; +} + +static int pcode_try_request(struct xe_tile *tile, u32 mbox, + u32 request, u32 reply_mask, u32 reply, + u32 *status, bool atomic, int timeout_us, bool locked) +{ + int slept, wait = 10; + + xe_tile_assert(tile, timeout_us > 0); + + for (slept = 0; slept < timeout_us; slept += wait) { + if (locked) + *status = pcode_mailbox_rw(tile, mbox, &request, NULL, 1, true, + atomic); + else + *status = __pcode_mailbox_rw(tile, mbox, &request, NULL, 1, true, + atomic); + if ((*status == 0) && ((request & reply_mask) == reply)) + return 0; + + if (atomic) + udelay(wait); + else + usleep_range(wait, wait << 1); + wait <<= 1; + } + + return -ETIMEDOUT; +} + +/** + * xe_pcode_request - send PCODE request until acknowledgment + * @tile: tile + * @mbox: PCODE mailbox ID the request is targeted for + * @request: request ID + * @reply_mask: mask used to check for request acknowledgment + * @reply: value used to check for request acknowledgment + * @timeout_base_ms: timeout for polling with preemption enabled + * + * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE + * reports an error or an overall timeout of @timeout_base_ms+50 ms expires. + * The request is acknowledged once the PCODE reply dword equals @reply after + * applying @reply_mask. Polling is first attempted with preemption enabled + * for @timeout_base_ms and if this times out for another 50 ms with + * preemption disabled. + * + * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some + * other error as reported by PCODE. + */ +int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms) +{ + u32 status; + int ret; + + xe_tile_assert(tile, timeout_base_ms <= 3); + + mutex_lock(&tile->pcode.lock); + + ret = pcode_try_request(tile, mbox, request, reply_mask, reply, &status, + false, timeout_base_ms * 1000, true); + if (!ret) + goto out; + + /* + * The above can time out if the number of requests was low (2 in the + * worst case) _and_ PCODE was busy for some reason even after a + * (queued) request and @timeout_base_ms delay. As a workaround retry + * the poll with preemption disabled to maximize the number of + * requests. Increase the timeout from @timeout_base_ms to 50ms to + * account for interrupts that could reduce the number of these + * requests, and for any quirks of the PCODE firmware that delays + * the request completion. + */ + drm_err(&tile_to_xe(tile)->drm, + "PCODE timeout, retrying with preemption disabled\n"); + preempt_disable(); + ret = pcode_try_request(tile, mbox, request, reply_mask, reply, &status, + true, 50 * 1000, true); + preempt_enable(); + +out: + mutex_unlock(&tile->pcode.lock); + return status ? status : ret; +} +/** + * xe_pcode_init_min_freq_table - Initialize PCODE's QOS frequency table + * @tile: tile instance + * @min_gt_freq: Minimal (RPn) GT frequency in units of 50MHz. + * @max_gt_freq: Maximal (RP0) GT frequency in units of 50MHz. + * + * This function initialize PCODE's QOS frequency table for a proper minimal + * frequency/power steering decision, depending on the current requested GT + * frequency. For older platforms this was a more complete table including + * the IA freq. However for the latest platforms this table become a simple + * 1-1 Ring vs GT frequency. Even though, without setting it, PCODE might + * not take the right decisions for some memory frequencies and affect latency. + * + * It returns 0 on success, and -ERROR number on failure, -EINVAL if max + * frequency is higher then the minimal, and other errors directly translated + * from the PCODE Error returns: + * - -ENXIO: "Illegal Command" + * - -ETIMEDOUT: "Timed out" + * - -EINVAL: "Illegal Data" + * - -ENXIO, "Illegal Subcommand" + * - -EBUSY: "PCODE Locked" + * - -EOVERFLOW, "GT ratio out of range" + * - -EACCES, "PCODE Rejected" + * - -EPROTO, "Unknown" + */ +int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq, + u32 max_gt_freq) +{ + int ret; + u32 freq; + + if (!tile_to_xe(tile)->info.has_llc) + return 0; + + if (max_gt_freq <= min_gt_freq) + return -EINVAL; + + mutex_lock(&tile->pcode.lock); + for (freq = min_gt_freq; freq <= max_gt_freq; freq++) { + u32 data = freq << PCODE_FREQ_RING_RATIO_SHIFT | freq; + + ret = pcode_mailbox_rw(tile, PCODE_WRITE_MIN_FREQ_TABLE, + &data, NULL, 1, false, false); + if (ret) + goto unlock; + } + +unlock: + mutex_unlock(&tile->pcode.lock); + return ret; +} + +/** + * xe_pcode_ready - Ensure PCODE is initialized + * @xe: xe instance + * @locked: true if lock held, false otherwise + * + * PCODE init mailbox is polled only on root gt of root tile + * as the root tile provides the initialization is complete only + * after all the tiles have completed the initialization. + * Called only on early probe without locks and with locks in + * resume path. + * + * Returns 0 on success, and -error number on failure. + */ +int xe_pcode_ready(struct xe_device *xe, bool locked) +{ + u32 status, request = DGFX_GET_INIT_STATUS; + struct xe_tile *tile = xe_device_get_root_tile(xe); + int timeout_us = 180000000; /* 3 min */ + int ret; + + if (xe->info.skip_pcode) + return 0; + + if (!IS_DGFX(xe)) + return 0; + + if (locked) + mutex_lock(&tile->pcode.lock); + + ret = pcode_try_request(tile, DGFX_PCODE_STATUS, request, + DGFX_INIT_STATUS_COMPLETE, + DGFX_INIT_STATUS_COMPLETE, + &status, false, timeout_us, locked); + + if (locked) + mutex_unlock(&tile->pcode.lock); + + if (ret) + drm_err(&xe->drm, + "PCODE initialization timedout after: 3 min\n"); + + return ret; +} + +/** + * xe_pcode_init_early() - Initialize components of PCODE + * @tile: tile instance + * + * This function initializes the xe_pcode component. + * To be called once only during probe. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pcode_init_early(struct xe_tile *tile) +{ + return drmm_mutex_init(&tile_to_xe(tile)->drm, &tile->pcode.lock); +} + +/** + * xe_pcode_probe_early: initializes PCODE + * @xe: xe instance + * + * This function checks the initialization status of PCODE + * To be called once only during early probe without locks. + * + * Returns 0 on success, error code otherwise + */ +int xe_pcode_probe_early(struct xe_device *xe) +{ + return xe_pcode_ready(xe, false); +} +ALLOW_ERROR_INJECTION(xe_pcode_probe_early, ERRNO); /* See xe_pci_probe */ + +/** + * xe_get_pcode_version - Read pcode version via PMT telemetry + * @xe: xe instance + * @version: pointer to struct xe_pcode_version to store version info + * + * Reads the pcode version from PMT telemetry and fills the + * provided @version structure. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_get_pcode_version(struct xe_device *xe, struct xe_pcode_version *version) +{ + int ret = 0; + + guard(xe_pm_runtime)(xe); + + ret = xe_pmt_telem_read(xe->drm.dev, + xe_mmio_read32(xe_root_tile_mmio(xe), PUNIT_TELEMETRY_GUID), + (u64 *)version, PUNIT_VERSION_OFFSET, sizeof(*version)); + if (ret != sizeof(*version)) { + xe_warn(xe, "pcode version read from PMT failed, ret %pe\n", ERR_PTR(ret)); + return ret; + } + xe_dbg(xe, "pcode version major %u minor %u engg %u\n", version->major, + version->minor, version->engg); + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h new file mode 100644 index 000000000..8fb3e4ba1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pcode.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PCODE_H_ +#define _XE_PCODE_H_ + +#include + +struct drm_device; +struct xe_device; +struct xe_tile; + +struct xe_pcode_version { + u16 minor; + u16 major; + u32 engg; +}; + +int xe_pcode_init_early(struct xe_tile *tile); +int xe_pcode_probe_early(struct xe_device *xe); +int xe_pcode_ready(struct xe_device *xe, bool locked); +int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq, + u32 max_gt_freq); +int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1); +int xe_pcode_write_timeout(struct xe_tile *tile, u32 mbox, u32 val, + int timeout_ms); +int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0, + u32 data1, int timeout); +int xe_get_pcode_version(struct xe_device *xe, struct xe_pcode_version *version); + +#define xe_pcode_write(tile, mbox, val) \ + xe_pcode_write_timeout(tile, mbox, val, 1) + +int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_ms); + +#define PCODE_MBOX(mbcmd, param1, param2)\ + (FIELD_PREP(PCODE_MB_COMMAND, mbcmd)\ + | FIELD_PREP(PCODE_MB_PARAM1, param1)\ + | FIELD_PREP(PCODE_MB_PARAM2, param2)) + +#endif diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h new file mode 100644 index 000000000..94575c476 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pcode_api.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PCODE_API_H_ +#define _XE_PCODE_API_H_ + +/* Internal to xe_pcode */ + +#include "regs/xe_reg_defs.h" + +#define PCODE_MAILBOX XE_REG(0x138124) +#define PCODE_READY REG_BIT(31) +#define PCODE_MB_PARAM2 REG_GENMASK(23, 16) +#define PCODE_MB_PARAM1 REG_GENMASK(15, 8) +#define PCODE_MB_COMMAND REG_GENMASK(7, 0) +#define PCODE_ERROR_MASK 0xFF +#define PCODE_SUCCESS 0x0 +#define PCODE_ILLEGAL_CMD 0x1 +#define PCODE_TIMEOUT 0x2 +#define PCODE_ILLEGAL_DATA 0x3 +#define PCODE_ILLEGAL_SUBCOMMAND 0x4 +#define PCODE_LOCKED 0x6 +#define PCODE_GT_RATIO_OUT_OF_RANGE 0x10 +#define PCODE_REJECTED 0x11 + +#define PCODE_DATA0 XE_REG(0x138128) +#define PCODE_DATA1 XE_REG(0x13812C) + +/* Min Freq QOS Table */ +#define PCODE_WRITE_MIN_FREQ_TABLE 0x8 +#define PCODE_READ_MIN_FREQ_TABLE 0x9 +#define PCODE_FREQ_RING_RATIO_SHIFT 16 + +/* PCODE Init */ +#define DGFX_PCODE_STATUS 0x7E +#define DGFX_GET_INIT_STATUS 0x0 +#define DGFX_INIT_STATUS_COMPLETE 0x1 +#define DGFX_LINK_DOWNGRADE_STATUS REG_BIT(31) + +#define PCODE_POWER_SETUP 0x7C +#define POWER_SETUP_SUBCOMMAND_READ_I1 0x4 +#define POWER_SETUP_SUBCOMMAND_WRITE_I1 0x5 +#define POWER_SETUP_I1_WATTS REG_BIT(31) +#define POWER_SETUP_I1_SHIFT 6 /* 10.6 fixed point format */ +#define POWER_SETUP_I1_DATA_MASK REG_GENMASK(15, 0) + +#define READ_PSYSGPU_POWER_LIMIT 0x6 +#define WRITE_PSYSGPU_POWER_LIMIT 0x7 +#define READ_PACKAGE_POWER_LIMIT 0x8 +#define WRITE_PACKAGE_POWER_LIMIT 0x9 +#define READ_PL_FROM_PCODE 0x0 +#define READ_PL_FROM_FW 0x1 +#define READ_PL_ACCEPTED 0x2 + +#define PCODE_THERMAL_INFO 0x25 +#define READ_THERMAL_LIMITS 0x0 +#define READ_THERMAL_CONFIG 0x1 +#define READ_THERMAL_DATA 0x2 +#define PCIE_SENSOR_GROUP_ID 0x2 +#define PCIE_SENSOR_MASK REG_GENMASK(31, 16) + +#define PCODE_LATE_BINDING 0x5C +#define GET_CAPABILITY_STATUS 0x0 +#define V1_FAN_SUPPORTED REG_BIT(0) +#define VR_PARAMS_SUPPORTED REG_BIT(3) +#define V1_FAN_PROVISIONED REG_BIT(16) +#define VR_PARAMS_PROVISIONED REG_BIT(19) +#define GET_VERSION_LOW 0x1 +#define GET_VERSION_HIGH 0x2 +#define MAJOR_VERSION_MASK REG_GENMASK(31, 16) +#define MINOR_VERSION_MASK REG_GENMASK(15, 0) +#define HOTFIX_VERSION_MASK REG_GENMASK(31, 16) +#define BUILD_VERSION_MASK REG_GENMASK(15, 0) +#define FAN_TABLE 1 +#define VR_CONFIG 2 + +#define PCODE_FREQUENCY_CONFIG 0x6e +/* Frequency Config Sub Commands (param1) */ +#define PCODE_MBOX_FC_SC_READ_FUSED_P0 0x0 +#define PCODE_MBOX_FC_SC_READ_FUSED_PN 0x1 +/* Domain IDs (param2) */ +#define PCODE_MBOX_DOMAIN_HBM 0x2 + +#define FAN_SPEED_CONTROL 0x7D +#define FSC_READ_NUM_FANS 0x4 + +#define PCODE_SCRATCH(x) XE_REG(0x138320 + ((x) * 4)) +/* PCODE_SCRATCH0 */ +#define BREADCRUMB_VERSION REG_GENMASK(31, 29) +#define AUXINFO_REG_OFFSET REG_GENMASK(17, 15) +#define OVERFLOW_REG_OFFSET REG_GENMASK(14, 12) +#define HISTORY_TRACKING REG_BIT(11) +#define OVERFLOW_SUPPORT REG_BIT(10) +#define AUXINFO_SUPPORT REG_BIT(9) +#define FDO_MODE REG_BIT(4) +#define BOOT_STATUS REG_GENMASK(3, 1) +#define CRITICAL_FAILURE 4 +#define NON_CRITICAL_FAILURE 7 + +/* Auxiliary info bits */ +#define AUXINFO_HISTORY_OFFSET REG_GENMASK(31, 29) + +#define BMG_PCIE_CAP XE_REG(0x138340) +#define LINK_DOWNGRADE REG_GENMASK(1, 0) +#define DOWNGRADE_CAPABLE 2 + +#endif diff --git a/drivers/gpu/drm/xe/xe_platform_types.h b/drivers/gpu/drm/xe/xe_platform_types.h new file mode 100644 index 000000000..3447848b7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_platform_types.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PLATFORM_TYPES_H_ +#define _XE_PLATFORM_TYPES_H_ + +/* + * Keep this in graphics version based order and chronological order within a + * version + */ +enum xe_platform { + XE_PLATFORM_UNINITIALIZED = 0, + XE_TIGERLAKE, + XE_ROCKETLAKE, + XE_ALDERLAKE_S, + XE_ALDERLAKE_P, + XE_ALDERLAKE_N, + XE_DG1, + XE_DG2, + XE_PVC, + XE_METEORLAKE, + XE_LUNARLAKE, + XE_BATTLEMAGE, + XE_PANTHERLAKE, + XE_NOVALAKE_S, + XE_CRESCENTISLAND, + XE_NOVALAKE_P, +}; + +enum xe_subplatform { + XE_SUBPLATFORM_UNINITIALIZED = 0, + XE_SUBPLATFORM_NONE, + XE_SUBPLATFORM_ALDERLAKE_P_RPLU, + XE_SUBPLATFORM_ALDERLAKE_S_RPLS, + XE_SUBPLATFORM_DG2_G10, + XE_SUBPLATFORM_DG2_G11, + XE_SUBPLATFORM_DG2_G12, + XE_SUBPLATFORM_BATTLEMAGE_G21, +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c new file mode 100644 index 000000000..a5289a9df --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -0,0 +1,1037 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_pm.h" + +#include +#include +#include +#include + +#include +#include + +#include "display/xe_display.h" +#include "xe_bo.h" +#include "xe_bo_evict.h" +#include "xe_device.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_idle.h" +#include "xe_i2c.h" +#include "xe_irq.h" +#include "xe_late_bind_fw.h" +#include "xe_pcode.h" +#include "xe_printk.h" +#include "xe_pxp.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_sysctrl.h" +#include "xe_trace.h" +#include "xe_vm.h" +#include "xe_wa.h" + +/** + * DOC: Xe Power Management + * + * Xe PM implements the main routines for both system level suspend states and + * for the opportunistic runtime suspend states. + * + * System Level Suspend (S-States) - In general this is OS initiated suspend + * driven by ACPI for achieving S0ix (a.k.a. S2idle, freeze), S3 (suspend to ram), + * S4 (disk). The main functions here are `xe_pm_suspend` and `xe_pm_resume`. They + * are the main point for the suspend to and resume from these states. + * + * PCI Device Suspend (D-States) - This is the opportunistic PCIe device low power + * state D3, controlled by the PCI subsystem and ACPI with the help from the + * runtime_pm infrastructure. + * PCI D3 is special and can mean D3hot, where Vcc power is on for keeping memory + * alive and quicker low latency resume or D3Cold where Vcc power is off for + * better power savings. + * The Vcc control of PCI hierarchy can only be controlled at the PCI root port + * level, while the device driver can be behind multiple bridges/switches and + * paired with other devices. For this reason, the PCI subsystem cannot perform + * the transition towards D3Cold. The lowest runtime PM possible from the PCI + * subsystem is D3hot. Then, if all these paired devices in the same root port + * are in D3hot, ACPI will assist here and run its own methods (_PR3 and _OFF) + * to perform the transition from D3hot to D3cold. Xe may disallow this + * transition by calling pci_d3cold_disable(root_pdev) before going to runtime + * suspend. It will be based on runtime conditions such as VRAM usage for a + * quick and low latency resume for instance. + * + * Runtime PM - This infrastructure provided by the Linux kernel allows the + * device drivers to indicate when the can be runtime suspended, so the device + * could be put at D3 (if supported), or allow deeper package sleep states + * (PC-states), and/or other low level power states. Xe PM component provides + * `xe_pm_runtime_suspend` and `xe_pm_runtime_resume` functions that PCI + * subsystem will call before transition to/from runtime suspend. + * + * Also, Xe PM provides get and put functions that Xe driver will use to + * indicate activity. In order to avoid locking complications with the memory + * management, whenever possible, these get and put functions needs to be called + * from the higher/outer levels. + * The main cases that need to be protected from the outer levels are: IOCTL, + * sysfs, debugfs, dma-buf sharing, GPU execution. + * + * This component is not responsible for GT idleness (RC6) nor GT frequency + * management (RPS). + */ + +#ifdef CONFIG_LOCKDEP +static struct lockdep_map xe_pm_runtime_d3cold_map = { + .name = "xe_rpm_d3cold_map" +}; + +static struct lockdep_map xe_pm_runtime_nod3cold_map = { + .name = "xe_rpm_nod3cold_map" +}; + +static struct lockdep_map xe_pm_block_lockdep_map = { + .name = "xe_pm_block_map", +}; +#endif + +static void xe_pm_block_begin_signalling(void) +{ + lock_acquire_shared_recursive(&xe_pm_block_lockdep_map, 0, 1, NULL, _RET_IP_); +} + +static void xe_pm_block_end_signalling(void) +{ + lock_release(&xe_pm_block_lockdep_map, _RET_IP_); +} + +/** + * xe_pm_might_block_on_suspend() - Annotate that the code might block on suspend + * + * Annotation to use where the code might block or seize to make + * progress pending resume completion. + */ +void xe_pm_might_block_on_suspend(void) +{ + lock_map_acquire(&xe_pm_block_lockdep_map); + lock_map_release(&xe_pm_block_lockdep_map); +} + +/** + * xe_pm_block_on_suspend() - Block pending suspend. + * @xe: The xe device about to be suspended. + * + * Block if the pm notifier has start evicting bos, to avoid + * racing and validating those bos back. The function is + * annotated to ensure no locks are held that are also grabbed + * in the pm notifier or the device suspend / resume. + * This is intended to be used by freezable tasks only. + * (Not freezable workqueues), with the intention that the function + * returns %-ERESTARTSYS when tasks are frozen during suspend, + * and allows the task to freeze. The caller must be able to + * handle the %-ERESTARTSYS. + * + * Return: %0 on success, %-ERESTARTSYS on signal pending or + * if freezing requested. + */ +int xe_pm_block_on_suspend(struct xe_device *xe) +{ + xe_pm_might_block_on_suspend(); + + return wait_for_completion_interruptible(&xe->pm_block); +} + +/** + * xe_rpm_reclaim_safe() - Whether runtime resume can be done from reclaim context + * @xe: The xe device. + * + * Return: true if it is safe to runtime resume from reclaim context. + * false otherwise. + */ +bool xe_rpm_reclaim_safe(const struct xe_device *xe) +{ + return !xe->d3cold.capable; +} + +static void xe_rpm_lockmap_acquire(const struct xe_device *xe) +{ + lock_map_acquire(xe_rpm_reclaim_safe(xe) ? + &xe_pm_runtime_nod3cold_map : + &xe_pm_runtime_d3cold_map); +} + +static void xe_rpm_lockmap_release(const struct xe_device *xe) +{ + lock_map_release(xe_rpm_reclaim_safe(xe) ? + &xe_pm_runtime_nod3cold_map : + &xe_pm_runtime_d3cold_map); +} + +/** + * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle + * @xe: xe device instance + * + * Return: 0 on success + */ +int xe_pm_suspend(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + int err; + + drm_dbg(&xe->drm, "Suspending device\n"); + xe_pm_block_begin_signalling(); + trace_xe_pm_suspend(xe, __builtin_return_address(0)); + + err = xe_pxp_pm_suspend(xe->pxp); + if (err) + goto err; + + xe_late_bind_wait_for_worker_completion(&xe->late_bind); + + for_each_gt(gt, xe, id) + xe_gt_suspend_prepare(gt); + + xe_display_pm_suspend(xe); + + /* FIXME: Super racey... */ + err = xe_bo_evict_all(xe); + if (err) + goto err_display; + + for_each_gt(gt, xe, id) { + err = xe_gt_suspend(gt); + if (err) + goto err_display; + } + + xe_irq_suspend(xe); + + xe_display_pm_suspend_late(xe); + + xe_i2c_pm_suspend(xe); + + drm_dbg(&xe->drm, "Device suspended\n"); + xe_pm_block_end_signalling(); + + return 0; + +err_display: + xe_display_pm_resume(xe); + xe_pxp_pm_resume(xe->pxp); +err: + drm_dbg(&xe->drm, "Device suspend failed %d\n", err); + xe_pm_block_end_signalling(); + return err; +} + +/** + * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0 + * @xe: xe device instance + * + * Return: 0 on success + */ +int xe_pm_resume(struct xe_device *xe) +{ + struct xe_tile *tile; + struct xe_gt *gt; + u8 id; + int err; + + xe_pm_block_begin_signalling(); + drm_dbg(&xe->drm, "Resuming device\n"); + trace_xe_pm_resume(xe, __builtin_return_address(0)); + + for_each_gt(gt, xe, id) + xe_gt_idle_disable_c6(gt); + + for_each_tile(tile, xe, id) + xe_wa_apply_tile_workarounds(tile); + + err = xe_pcode_ready(xe, true); + if (err) + return err; + + xe_display_pm_resume_early(xe); + + /* + * This only restores pinned memory which is the memory required for the + * GT(s) to resume. + */ + err = xe_bo_restore_early(xe); + if (err) + goto err; + + xe_i2c_pm_resume(xe, true); + + xe_sysctrl_pm_resume(xe); + + xe_irq_resume(xe); + + for_each_gt(gt, xe, id) { + err = xe_gt_resume(gt); + if (err) + break; + } + + /* + * Try to bring up display before bailing from GT resume failure, + * so we don't leave the user clueless with a blank screen. + */ + xe_display_pm_resume(xe); + if (err) + goto err; + + err = xe_bo_restore_late(xe); + if (err) + goto err; + + xe_pxp_pm_resume(xe->pxp); + + if (IS_VF_CCS_READY(xe)) + xe_sriov_vf_ccs_register_context(xe); + + xe_late_bind_fw_load(&xe->late_bind); + + drm_dbg(&xe->drm, "Device resumed\n"); + xe_pm_block_end_signalling(); + return 0; +err: + drm_dbg(&xe->drm, "Device resume failed %d\n", err); + xe_pm_block_end_signalling(); + return err; +} + +static bool xe_pm_pci_d3cold_capable(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability */ + if (!pci_pme_capable(root_pdev, PCI_D3cold)) { + drm_dbg(&xe->drm, "d3cold: PME# not supported\n"); + return false; + } + + /* D3Cold requires _PR3 power resource */ + if (!pci_pr3_present(root_pdev)) { + drm_dbg(&xe->drm, "d3cold: ACPI _PR3 not present\n"); + return false; + } + + return true; +} + +static void xe_pm_runtime_init(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + + /* Our current VFs do not support RPM. so, disable it */ + if (IS_SRIOV_VF(xe)) + return; + + /* + * Disable the system suspend direct complete optimization. + * We need to ensure that the regular device suspend/resume functions + * are called since our runtime_pm cannot guarantee local memory + * eviction for d3cold. + * TODO: Check HDA audio dependencies claimed by i915, and then enforce + * this option to integrated graphics as well. + */ + if (IS_DGFX(xe)) + dev_pm_set_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE); + + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, 1000); + pm_runtime_set_active(dev); + pm_runtime_allow(dev); + pm_runtime_mark_last_busy(dev); + pm_runtime_put(dev); +} + +/** + * xe_pm_init_early() - Initialize Xe Power Management + * @xe: the &xe_device instance + * + * Initialize everything that is a "software-only" state that does not + * require access to any of the device's hardware data. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pm_init_early(struct xe_device *xe) +{ + int err; + + init_completion(&xe->pm_block); + complete_all(&xe->pm_block); + INIT_LIST_HEAD(&xe->rebind_resume_list); + INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list); + + err = drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock); + if (err) + return err; + + err = drmm_mutex_init(&xe->drm, &xe->d3cold.lock); + if (err) + return err; + + err = drmm_mutex_init(&xe->drm, &xe->rebind_resume_lock); + if (err) + return err; + + return 0; +} +ALLOW_ERROR_INJECTION(xe_pm_init_early, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_pm_probe() - Initialize Xe Power Management + * @xe: the &xe_device instance + * + * Check d3cold capability. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pm_probe(struct xe_device *xe) +{ + xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe); + xe_dbg(xe, "d3cold: capable=%s\n", str_yes_no(xe->d3cold.capable)); + + return 0; +} + +static u32 vram_threshold_value(struct xe_device *xe) +{ + if (xe->info.platform == XE_BATTLEMAGE) { + const char *product_name; + + product_name = dmi_get_system_info(DMI_PRODUCT_NAME); + if (product_name && strstr(product_name, "NUC13RNG")) { + drm_warn(&xe->drm, "BMG + D3Cold not supported on this platform\n"); + return 0; + } + } + + return DEFAULT_VRAM_THRESHOLD; +} + +static void xe_pm_wake_rebind_workers(struct xe_device *xe) +{ + struct xe_vm *vm, *next; + + mutex_lock(&xe->rebind_resume_lock); + list_for_each_entry_safe(vm, next, &xe->rebind_resume_list, + preempt.pm_activate_link) { + list_del_init(&vm->preempt.pm_activate_link); + xe_vm_resume_rebind_worker(vm); + } + mutex_unlock(&xe->rebind_resume_lock); +} + +static int xe_pm_notifier_callback(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct xe_device *xe = container_of(nb, struct xe_device, pm_notifier); + int err = 0; + + switch (action) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + { + struct xe_validation_ctx ctx; + + reinit_completion(&xe->pm_block); + xe_pm_block_begin_signalling(); + xe_pm_runtime_get(xe); + (void)xe_validation_ctx_init(&ctx, &xe->val, NULL, + (struct xe_val_flags) {.exclusive = true}); + err = xe_bo_evict_all_user(xe); + xe_validation_ctx_fini(&ctx); + if (err) + drm_dbg(&xe->drm, "Notifier evict user failed (%d)\n", err); + + err = xe_bo_notifier_prepare_all_pinned(xe); + if (err) + drm_dbg(&xe->drm, "Notifier prepare pin failed (%d)\n", err); + /* + * Keep the runtime pm reference until post hibernation / post suspend to + * avoid a runtime suspend interfering with evicted objects or backup + * allocations. + */ + xe_pm_block_end_signalling(); + break; + } + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + complete_all(&xe->pm_block); + xe_pm_wake_rebind_workers(xe); + xe_bo_notifier_unprepare_all_pinned(xe); + xe_pm_runtime_put(xe); + break; + } + + return NOTIFY_DONE; +} + +/** + * xe_pm_init - Initialize Xe Power Management + * @xe: xe device instance + * + * This component is responsible for System and Device sleep states. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_pm_init(struct xe_device *xe) +{ + u32 vram_threshold; + int err; + + xe->pm_notifier.notifier_call = xe_pm_notifier_callback; + err = register_pm_notifier(&xe->pm_notifier); + if (err) + return err; + + /* For now suspend/resume is only allowed with GuC */ + if (!xe_device_uc_enabled(xe)) + return 0; + + if (xe->d3cold.capable) { + vram_threshold = vram_threshold_value(xe); + err = xe_pm_set_vram_threshold(xe, vram_threshold); + if (err) + goto err_unregister; + } + + xe_pm_runtime_init(xe); + return 0; + +err_unregister: + unregister_pm_notifier(&xe->pm_notifier); + return err; +} + +static void xe_pm_runtime_fini(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + + /* Our current VFs do not support RPM. so, disable it */ + if (IS_SRIOV_VF(xe)) + return; + + pm_runtime_get_sync(dev); + pm_runtime_forbid(dev); +} + +/** + * xe_pm_fini - Finalize PM + * @xe: xe device instance + */ +void xe_pm_fini(struct xe_device *xe) +{ + if (xe_device_uc_enabled(xe)) + xe_pm_runtime_fini(xe); + + unregister_pm_notifier(&xe->pm_notifier); +} + +static void xe_pm_write_callback_task(struct xe_device *xe, + struct task_struct *task) +{ + WRITE_ONCE(xe->pm_callback_task, task); + + /* + * Just in case it's somehow possible for our writes to be reordered to + * the extent that something else re-uses the task written in + * pm_callback_task. For example after returning from the callback, but + * before the reordered write that resets pm_callback_task back to NULL. + */ + smp_mb(); /* pairs with xe_pm_read_callback_task */ +} + +struct task_struct *xe_pm_read_callback_task(struct xe_device *xe) +{ + smp_mb(); /* pairs with xe_pm_write_callback_task */ + + return READ_ONCE(xe->pm_callback_task); +} + +/** + * xe_pm_runtime_suspended - Check if runtime_pm state is suspended + * @xe: xe device instance + * + * This does not provide any guarantee that the device is going to remain + * suspended as it might be racing with the runtime state transitions. + * It can be used only as a non-reliable assertion, to ensure that we are not in + * the sleep state while trying to access some memory for instance. + * + * Returns true if PCI device is suspended, false otherwise. + */ +bool xe_pm_runtime_suspended(struct xe_device *xe) +{ + return pm_runtime_suspended(xe->drm.dev); +} + +/** + * xe_pm_runtime_suspend - Prepare our device for D3hot/D3Cold + * @xe: xe device instance + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_pm_runtime_suspend(struct xe_device *xe) +{ + struct xe_bo *bo, *on; + struct xe_gt *gt; + u8 id; + int err = 0; + + trace_xe_pm_runtime_suspend(xe, __builtin_return_address(0)); + /* Disable access_ongoing asserts and prevent recursive pm calls */ + xe_pm_write_callback_task(xe, current); + + /* + * The actual xe_pm_runtime_put() is always async underneath, so + * exactly where that is called should makes no difference to us. However + * we still need to be very careful with the locks that this callback + * acquires and the locks that are acquired and held by any callers of + * xe_runtime_pm_get(). We already have the matching annotation + * on that side, but we also need it here. For example lockdep should be + * able to tell us if the following scenario is in theory possible: + * + * CPU0 | CPU1 (kworker) + * lock(A) | + * | xe_pm_runtime_suspend() + * | lock(A) + * xe_pm_runtime_get() | + * + * This will clearly deadlock since rpm core needs to wait for + * xe_pm_runtime_suspend() to complete, but here we are holding lock(A) + * on CPU0 which prevents CPU1 making forward progress. With the + * annotation here and in xe_pm_runtime_get() lockdep will see + * the potential lock inversion and give us a nice splat. + */ + xe_rpm_lockmap_acquire(xe); + + err = xe_pxp_pm_suspend(xe->pxp); + if (err) + goto out; + + /* + * Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify + * also checks and deletes bo entry from user fault list. + */ + mutex_lock(&xe->mem_access.vram_userfault.lock); + list_for_each_entry_safe(bo, on, + &xe->mem_access.vram_userfault.list, vram_userfault_link) + xe_bo_runtime_pm_release_mmap_offset(bo); + mutex_unlock(&xe->mem_access.vram_userfault.lock); + + xe_display_pm_runtime_suspend(xe); + + if (xe->d3cold.allowed) { + err = xe_bo_evict_all(xe); + if (err) + goto out_resume; + } + + for_each_gt(gt, xe, id) { + err = xe->d3cold.allowed ? xe_gt_suspend(gt) : xe_gt_runtime_suspend(gt); + if (err) + goto out_resume; + } + + xe_irq_suspend(xe); + + xe_display_pm_runtime_suspend_late(xe); + + xe_i2c_pm_suspend(xe); + + xe_rpm_lockmap_release(xe); + xe_pm_write_callback_task(xe, NULL); + return 0; + +out_resume: + xe_display_pm_runtime_resume(xe); + xe_pxp_pm_resume(xe->pxp); +out: + xe_rpm_lockmap_release(xe); + xe_pm_write_callback_task(xe, NULL); + return err; +} + +/** + * xe_pm_runtime_resume - Waking up from D3hot/D3Cold + * @xe: xe device instance + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_pm_runtime_resume(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + int err = 0; + + trace_xe_pm_runtime_resume(xe, __builtin_return_address(0)); + /* Disable access_ongoing asserts and prevent recursive pm calls */ + xe_pm_write_callback_task(xe, current); + + xe_rpm_lockmap_acquire(xe); + + if (xe->d3cold.allowed) { + for_each_gt(gt, xe, id) + xe_gt_idle_disable_c6(gt); + + err = xe_pcode_ready(xe, true); + if (err) + goto out; + + xe_display_pm_resume_early(xe); + + /* + * This only restores pinned memory which is the memory + * required for the GT(s) to resume. + */ + err = xe_bo_restore_early(xe); + if (err) + goto out; + } + + xe_i2c_pm_resume(xe, xe->d3cold.allowed); + + if (xe->d3cold.allowed) + xe_sysctrl_pm_resume(xe); + + xe_display_pm_runtime_resume_early(xe); + + xe_irq_resume(xe); + + for_each_gt(gt, xe, id) { + err = xe->d3cold.allowed ? xe_gt_resume(gt) : xe_gt_runtime_resume(gt); + if (err) + break; + } + + /* + * Try to bring up display before bailing from GT resume failure, + * so we don't leave the user clueless with a blank screen. + */ + xe_display_pm_runtime_resume(xe); + if (err) + goto out; + + if (xe->d3cold.allowed) { + err = xe_bo_restore_late(xe); + if (err) + goto out; + } + + xe_pxp_pm_resume(xe->pxp); + + if (IS_VF_CCS_READY(xe)) + xe_sriov_vf_ccs_register_context(xe); + + if (xe->d3cold.allowed) + xe_late_bind_fw_load(&xe->late_bind); + +out: + xe_rpm_lockmap_release(xe); + xe_pm_write_callback_task(xe, NULL); + return err; +} + +/* + * For places where resume is synchronous it can be quite easy to deadlock + * if we are not careful. Also in practice it might be quite timing + * sensitive to ever see the 0 -> 1 transition with the callers locks + * held, so deadlocks might exist but are hard for lockdep to ever see. + * With this in mind, help lockdep learn about the potentially scary + * stuff that can happen inside the runtime_resume callback by acquiring + * a dummy lock (it doesn't protect anything and gets compiled out on + * non-debug builds). Lockdep then only needs to see the + * xe_pm_runtime_xxx_map -> runtime_resume callback once, and then can + * hopefully validate all the (callers_locks) -> xe_pm_runtime_xxx_map. + * For example if the (callers_locks) are ever grabbed in the + * runtime_resume callback, lockdep should give us a nice splat. + */ +static void xe_rpm_might_enter_cb(const struct xe_device *xe) +{ + xe_rpm_lockmap_acquire(xe); + xe_rpm_lockmap_release(xe); +} + +/* + * Prime the lockdep maps for known locking orders that need to + * be supported but that may not always occur on all systems. + */ +static void xe_pm_runtime_lockdep_prime(void) +{ + struct dma_resv lockdep_resv; + + dma_resv_init(&lockdep_resv); + lock_map_acquire(&xe_pm_runtime_d3cold_map); + /* D3Cold takes the dma_resv locks to evict bos */ + dma_resv_lock(&lockdep_resv, NULL); + dma_resv_unlock(&lockdep_resv); + lock_map_release(&xe_pm_runtime_d3cold_map); + + /* Shrinkers might like to wake up the device under reclaim. */ + fs_reclaim_acquire(GFP_KERNEL); + lock_map_acquire(&xe_pm_runtime_nod3cold_map); + lock_map_release(&xe_pm_runtime_nod3cold_map); + fs_reclaim_release(GFP_KERNEL); +} + +/** + * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously + * @xe: xe device instance + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime)) is + * be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which + * can break scope-based handling, or when the lifetime of the runtime PM + * reference does not match a specific scope (e.g., runtime PM obtained in one + * function and released in a different one). + */ +void xe_pm_runtime_get(struct xe_device *xe) +{ + trace_xe_pm_runtime_get(xe, __builtin_return_address(0)); + pm_runtime_get_noresume(xe->drm.dev); + + if (xe_pm_read_callback_task(xe) == current) + return; + + xe_rpm_might_enter_cb(xe); + pm_runtime_resume(xe->drm.dev); +} + +/** + * xe_pm_runtime_put - Put the runtime_pm reference back and mark as idle + * @xe: xe device instance + */ +void xe_pm_runtime_put(struct xe_device *xe) +{ + trace_xe_pm_runtime_put(xe, __builtin_return_address(0)); + if (xe_pm_read_callback_task(xe) == current) { + pm_runtime_put_noidle(xe->drm.dev); + } else { + pm_runtime_mark_last_busy(xe->drm.dev); + pm_runtime_put(xe->drm.dev); + } +} + +/** + * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl + * @xe: xe device instance + * + * When possible, scope-based runtime PM (through + * ACQUIRE(xe_pm_runtime_ioctl, ...)) is be preferred over direct usage of this + * function. Manual get/put handling should only be used when the function + * contains goto-based logic which can break scope-based handling, or when the + * lifetime of the runtime PM reference does not match a specific scope (e.g., + * runtime PM obtained in one function and released in a different one). + * + * Returns: Any number greater than or equal to 0 for success, negative error + * code otherwise. + */ +int xe_pm_runtime_get_ioctl(struct xe_device *xe) +{ + trace_xe_pm_runtime_get_ioctl(xe, __builtin_return_address(0)); + if (WARN_ON(xe_pm_read_callback_task(xe) == current)) + return -ELOOP; + + xe_rpm_might_enter_cb(xe); + return pm_runtime_get_sync(xe->drm.dev); +} + +/** + * xe_pm_runtime_get_if_active - Get a runtime_pm reference if device active + * @xe: xe device instance + * + * Return: True if device is awake (regardless the previous number of references) + * and a new reference was taken, false otherwise. + */ +bool xe_pm_runtime_get_if_active(struct xe_device *xe) +{ + return pm_runtime_get_if_active(xe->drm.dev) > 0; +} + +/** + * xe_pm_runtime_get_if_in_use - Get a new reference if device is active with previous ref taken + * @xe: xe device instance + * + * Return: True if device is awake, a previous reference had been already taken, + * and a new reference was now taken, false otherwise. + */ +bool xe_pm_runtime_get_if_in_use(struct xe_device *xe) +{ + if (xe_pm_read_callback_task(xe) == current) { + /* The device is awake, grab the ref and move on */ + pm_runtime_get_noresume(xe->drm.dev); + return true; + } + + return pm_runtime_get_if_in_use(xe->drm.dev) > 0; +} + +/* + * Very unreliable! Should only be used to suppress the false positive case + * in the missing outer rpm protection warning. + */ +static bool xe_pm_suspending_or_resuming(struct xe_device *xe) +{ +#ifdef CONFIG_PM + struct device *dev = xe->drm.dev; + + return dev->power.runtime_status == RPM_SUSPENDING || + dev->power.runtime_status == RPM_RESUMING || + pm_suspend_in_progress(); +#else + return false; +#endif +} + +/** + * xe_pm_runtime_get_noresume - Bump runtime PM usage counter without resuming + * @xe: xe device instance + * + * This function should be used in inner places where it is surely already + * protected by outer-bound callers of `xe_pm_runtime_get`. + * It will warn if not protected. + * The reference should be put back after this function regardless, since it + * will always bump the usage counter, regardless. + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime_noresume)) + * is be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which can + * break scope-based handling, or when the lifetime of the runtime PM reference + * does not match a specific scope (e.g., runtime PM obtained in one function + * and released in a different one). + */ +void xe_pm_runtime_get_noresume(struct xe_device *xe) +{ + bool ref; + + ref = xe_pm_runtime_get_if_in_use(xe); + + if (!ref) { + pm_runtime_get_noresume(xe->drm.dev); + drm_WARN(&xe->drm, !xe_pm_suspending_or_resuming(xe), + "Missing outer runtime PM protection\n"); + } +} + +/** + * xe_pm_runtime_resume_and_get - Resume, then get a runtime_pm ref if awake. + * @xe: xe device instance + * + * Returns: True if device is awake and the reference was taken, false otherwise. + */ +bool xe_pm_runtime_resume_and_get(struct xe_device *xe) +{ + if (xe_pm_read_callback_task(xe) == current) { + /* The device is awake, grab the ref and move on */ + pm_runtime_get_noresume(xe->drm.dev); + return true; + } + + xe_rpm_might_enter_cb(xe); + return pm_runtime_resume_and_get(xe->drm.dev) >= 0; +} + +/** + * xe_pm_assert_unbounded_bridge - Disable PM on unbounded pcie parent bridge + * @xe: xe device instance + */ +void xe_pm_assert_unbounded_bridge(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct pci_dev *bridge = pci_upstream_bridge(pdev); + + if (!bridge) + return; + + if (!bridge->driver) { + drm_warn(&xe->drm, "unbounded parent pci bridge, device won't support any PM support.\n"); + device_set_pm_not_required(&pdev->dev); + } +} + +/** + * xe_pm_set_vram_threshold - Set a VRAM threshold for allowing/blocking D3Cold + * @xe: xe device instance + * @threshold: VRAM size in MiB for the D3cold threshold + * + * Return: + * * 0 - success + * * -EINVAL - invalid argument + */ +int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold) +{ + struct ttm_resource_manager *man; + u32 vram_total_mb = 0; + int i; + + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) + vram_total_mb += DIV_ROUND_UP_ULL(man->size, 1024 * 1024); + } + + drm_dbg(&xe->drm, "Total vram %u mb\n", vram_total_mb); + + if (threshold > vram_total_mb) + return -EINVAL; + + mutex_lock(&xe->d3cold.lock); + xe->d3cold.vram_threshold = threshold; + mutex_unlock(&xe->d3cold.lock); + + return 0; +} + +/** + * xe_pm_d3cold_allowed_toggle - Check conditions to toggle d3cold.allowed + * @xe: xe device instance + * + * To be called during runtime_pm idle callback. + * Check for all the D3Cold conditions ahead of runtime suspend. + */ +void xe_pm_d3cold_allowed_toggle(struct xe_device *xe) +{ + struct ttm_resource_manager *man; + u32 total_vram_used_mb = 0; + u64 vram_used; + int i; + + if (!xe->d3cold.capable) { + xe->d3cold.allowed = false; + return; + } + + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = ttm_resource_manager_usage(man); + total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024); + } + } + + mutex_lock(&xe->d3cold.lock); + + if (total_vram_used_mb < xe->d3cold.vram_threshold) + xe->d3cold.allowed = true; + else + xe->d3cold.allowed = false; + + mutex_unlock(&xe->d3cold.lock); +} + +/** + * xe_pm_module_init() - Perform xe_pm specific module initialization. + * + * Return: 0 on success. Currently doesn't fail. + */ +int __init xe_pm_module_init(void) +{ + xe_pm_runtime_lockdep_prime(); + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h new file mode 100644 index 000000000..6d5ab09cb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PM_H_ +#define _XE_PM_H_ + +#include +#include + +#define DEFAULT_VRAM_THRESHOLD 300 /* in MB */ + +struct xe_device; + +int xe_pm_suspend(struct xe_device *xe); +int xe_pm_resume(struct xe_device *xe); + +int xe_pm_init_early(struct xe_device *xe); +int xe_pm_probe(struct xe_device *xe); +int xe_pm_init(struct xe_device *xe); +void xe_pm_fini(struct xe_device *xe); +bool xe_pm_runtime_suspended(struct xe_device *xe); +int xe_pm_runtime_suspend(struct xe_device *xe); +int xe_pm_runtime_resume(struct xe_device *xe); +void xe_pm_runtime_get(struct xe_device *xe); +int xe_pm_runtime_get_ioctl(struct xe_device *xe); +void xe_pm_runtime_put(struct xe_device *xe); +bool xe_pm_runtime_get_if_active(struct xe_device *xe); +bool xe_pm_runtime_get_if_in_use(struct xe_device *xe); +void xe_pm_runtime_get_noresume(struct xe_device *xe); +bool xe_pm_runtime_resume_and_get(struct xe_device *xe); +void xe_pm_assert_unbounded_bridge(struct xe_device *xe); +int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold); +void xe_pm_d3cold_allowed_toggle(struct xe_device *xe); +bool xe_rpm_reclaim_safe(const struct xe_device *xe); +struct task_struct *xe_pm_read_callback_task(struct xe_device *xe); +int xe_pm_block_on_suspend(struct xe_device *xe); +void xe_pm_might_block_on_suspend(void); +int xe_pm_module_init(void); + +static inline void __xe_pm_runtime_noop(struct xe_device *xe) {} + +DEFINE_GUARD(xe_pm_runtime, struct xe_device *, + xe_pm_runtime_get(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD(xe_pm_runtime_noresume, struct xe_device *, + xe_pm_runtime_get_noresume(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD_COND(xe_pm_runtime, _ioctl, xe_pm_runtime_get_ioctl(_T), _RET >= 0) + +/* + * Used when a function needs to release runtime PM in all possible cases + * and error paths, but the wakeref was already acquired by a different + * function (i.e., get() has already happened so only a put() is needed). + */ +DEFINE_GUARD(xe_pm_runtime_release_only, struct xe_device *, + __xe_pm_runtime_noop(_T), xe_pm_runtime_put(_T)); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pmu.c b/drivers/gpu/drm/xe/xe_pmu.c new file mode 100644 index 000000000..ce1b110f7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pmu.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt_idle.h" +#include "xe_guc_engine_activity.h" +#include "xe_guc_pc.h" +#include "xe_hw_engine.h" +#include "xe_pm.h" +#include "xe_pmu.h" +#include "xe_sriov_pf_helpers.h" + +/** + * DOC: Xe PMU (Performance Monitoring Unit) + * + * Expose events/counters like GT-C6 residency, GT frequency and per-class-engine + * activity to user land via the perf interface. Events are per device. + * + * All events are listed in sysfs: + * + * $ ls -ld /sys/bus/event_source/devices/xe_* + * $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/events/ + * $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/format/ + * + * The following format parameters are available to read events, + * but only few are valid with each event: + * + * gt[60:63] Selects gt for the event + * engine_class[20:27] Selects engine-class for event + * engine_instance[12:19] Selects the engine-instance for the event + * function[44:59] Selects the function of the event (SRIOV enabled) + * + * For engine specific events (engine-*), gt, engine_class and engine_instance parameters must be + * set as populated by DRM_XE_DEVICE_QUERY_ENGINES and function if SRIOV is enabled. + * + * For gt specific events (gt-*) gt parameter must be passed. All other parameters will be 0. + * + * The standard perf tool can be used to grep for a certain event as well. + * Example: + * + * $ perf list | grep gt-c6 + * + * To sample a specific event for a GT at regular intervals: + * + * $ perf stat -e -I + */ + +#define XE_PMU_EVENT_GT_MASK GENMASK_ULL(63, 60) +#define XE_PMU_EVENT_FUNCTION_MASK GENMASK_ULL(59, 44) +#define XE_PMU_EVENT_ENGINE_CLASS_MASK GENMASK_ULL(27, 20) +#define XE_PMU_EVENT_ENGINE_INSTANCE_MASK GENMASK_ULL(19, 12) +#define XE_PMU_EVENT_ID_MASK GENMASK_ULL(11, 0) + +static unsigned int config_to_event_id(u64 config) +{ + return FIELD_GET(XE_PMU_EVENT_ID_MASK, config); +} + +static unsigned int config_to_function_id(u64 config) +{ + return FIELD_GET(XE_PMU_EVENT_FUNCTION_MASK, config); +} + +static unsigned int config_to_engine_class(u64 config) +{ + return FIELD_GET(XE_PMU_EVENT_ENGINE_CLASS_MASK, config); +} + +static unsigned int config_to_engine_instance(u64 config) +{ + return FIELD_GET(XE_PMU_EVENT_ENGINE_INSTANCE_MASK, config); +} + +static unsigned int config_to_gt_id(u64 config) +{ + return FIELD_GET(XE_PMU_EVENT_GT_MASK, config); +} + +#define XE_PMU_EVENT_GT_C6_RESIDENCY 0x01 +#define XE_PMU_EVENT_ENGINE_ACTIVE_TICKS 0x02 +#define XE_PMU_EVENT_ENGINE_TOTAL_TICKS 0x03 +#define XE_PMU_EVENT_GT_ACTUAL_FREQUENCY 0x04 +#define XE_PMU_EVENT_GT_REQUESTED_FREQUENCY 0x05 + +static struct xe_gt *event_to_gt(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + u64 gt = config_to_gt_id(event->attr.config); + + return xe_device_get_gt(xe, gt); +} + +static struct xe_hw_engine *event_to_hwe(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct drm_xe_engine_class_instance eci; + u64 config = event->attr.config; + struct xe_hw_engine *hwe; + + eci.engine_class = config_to_engine_class(config); + eci.engine_instance = config_to_engine_instance(config); + eci.gt_id = config_to_gt_id(config); + + hwe = xe_hw_engine_lookup(xe, eci); + if (!hwe || xe_hw_engine_is_reserved(hwe)) + return NULL; + + return hwe; +} + +static bool is_engine_event(u64 config) +{ + unsigned int event_id = config_to_event_id(config); + + return (event_id == XE_PMU_EVENT_ENGINE_TOTAL_TICKS || + event_id == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS); +} + +static bool is_gt_frequency_event(struct perf_event *event) +{ + u32 id = config_to_event_id(event->attr.config); + + return id == XE_PMU_EVENT_GT_ACTUAL_FREQUENCY || + id == XE_PMU_EVENT_GT_REQUESTED_FREQUENCY; +} + +static bool event_gt_forcewake(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + u64 config = event->attr.config; + struct xe_gt *gt; + unsigned int *fw_ref; + + if (!is_engine_event(config) && !is_gt_frequency_event(event)) + return true; + + gt = xe_device_get_gt(xe, config_to_gt_id(config)); + + fw_ref = kzalloc_obj(*fw_ref); + if (!fw_ref) + return false; + + *fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (!*fw_ref) { + kfree(fw_ref); + return false; + } + + event->pmu_private = fw_ref; + + return true; +} + +static bool event_supported(struct xe_pmu *pmu, unsigned int gt_id, + unsigned int id) +{ + struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); + struct xe_gt *gt = xe_device_get_gt(xe, gt_id); + + if (!gt) + return false; + + return id < sizeof(pmu->supported_events) * BITS_PER_BYTE && + pmu->supported_events & BIT_ULL(id); +} + +static bool event_param_valid(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + unsigned int engine_class, engine_instance, function_id; + u64 config = event->attr.config; + struct xe_gt *gt; + + gt = xe_device_get_gt(xe, config_to_gt_id(config)); + if (!gt) + return false; + + engine_class = config_to_engine_class(config); + engine_instance = config_to_engine_instance(config); + function_id = config_to_function_id(config); + + switch (config_to_event_id(config)) { + case XE_PMU_EVENT_GT_C6_RESIDENCY: + case XE_PMU_EVENT_GT_ACTUAL_FREQUENCY: + case XE_PMU_EVENT_GT_REQUESTED_FREQUENCY: + if (engine_class || engine_instance || function_id) + return false; + break; + case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS: + case XE_PMU_EVENT_ENGINE_TOTAL_TICKS: + if (!event_to_hwe(event)) + return false; + + /* PF(0) and total vfs when SRIOV is enabled */ + if (IS_SRIOV_PF(xe)) { + if (function_id > xe_sriov_pf_get_totalvfs(xe)) + return false; + } else if (function_id) { + return false; + } + + break; + } + + return true; +} + +static void xe_pmu_event_destroy(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_gt *gt; + unsigned int *fw_ref = event->pmu_private; + + if (fw_ref) { + gt = xe_device_get_gt(xe, config_to_gt_id(event->attr.config)); + xe_force_wake_put(gt_to_fw(gt), *fw_ref); + kfree(fw_ref); + event->pmu_private = NULL; + } + + drm_WARN_ON(&xe->drm, event->parent); + xe_pm_runtime_put(xe); + drm_dev_put(&xe->drm); +} + +static int xe_pmu_event_init(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + unsigned int id, gt; + + if (!pmu->registered) + return -ENODEV; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* unsupported modes and filters */ + if (event->attr.sample_period) /* no sampling */ + return -EINVAL; + + if (event->cpu < 0) + return -EINVAL; + + gt = config_to_gt_id(event->attr.config); + id = config_to_event_id(event->attr.config); + if (!event_supported(pmu, gt, id)) + return -ENOENT; + + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + if (!event_param_valid(event)) + return -ENOENT; + + if (!event->parent) { + drm_dev_get(&xe->drm); + xe_pm_runtime_get(xe); + if (!event_gt_forcewake(event)) { + xe_pm_runtime_put(xe); + drm_dev_put(&xe->drm); + return -EINVAL; + } + event->destroy = xe_pmu_event_destroy; + } + + return 0; +} + +static u64 read_engine_events(struct xe_gt *gt, struct perf_event *event) +{ + struct xe_hw_engine *hwe; + unsigned int function_id; + u64 config, val = 0; + + config = event->attr.config; + function_id = config_to_function_id(config); + + hwe = event_to_hwe(event); + if (config_to_event_id(config) == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS) + val = xe_guc_engine_activity_active_ticks(>->uc.guc, hwe, function_id); + else + val = xe_guc_engine_activity_total_ticks(>->uc.guc, hwe, function_id); + + return val; +} + +static u64 __xe_pmu_event_read(struct perf_event *event) +{ + struct xe_gt *gt = event_to_gt(event); + + if (!gt) + return 0; + + switch (config_to_event_id(event->attr.config)) { + case XE_PMU_EVENT_GT_C6_RESIDENCY: + return xe_gt_idle_residency_msec(>->gtidle); + case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS: + case XE_PMU_EVENT_ENGINE_TOTAL_TICKS: + return read_engine_events(gt, event); + case XE_PMU_EVENT_GT_ACTUAL_FREQUENCY: + return xe_guc_pc_get_act_freq(>->uc.guc.pc); + case XE_PMU_EVENT_GT_REQUESTED_FREQUENCY: + return xe_guc_pc_get_cur_freq_fw(>->uc.guc.pc); + } + + return 0; +} + +static void xe_pmu_event_update(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 prev, new; + + prev = local64_read(&hwc->prev_count); + do { + new = __xe_pmu_event_read(event); + } while (!local64_try_cmpxchg(&hwc->prev_count, &prev, new)); + + /* + * GT frequency is not a monotonically increasing counter, so add the + * instantaneous value instead. + */ + if (is_gt_frequency_event(event)) + local64_add(new, &event->count); + else + local64_add(new - prev, &event->count); +} + +static void xe_pmu_event_read(struct perf_event *event) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + + if (!pmu->registered) { + event->hw.state = PERF_HES_STOPPED; + return; + } + + xe_pmu_event_update(event); +} + +static void xe_pmu_enable(struct perf_event *event) +{ + /* + * Store the current counter value so we can report the correct delta + * for all listeners. Even when the event was already enabled and has + * an existing non-zero value. + */ + local64_set(&event->hw.prev_count, __xe_pmu_event_read(event)); +} + +static void xe_pmu_event_start(struct perf_event *event, int flags) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + + if (!pmu->registered) + return; + + xe_pmu_enable(event); + event->hw.state = 0; +} + +static void xe_pmu_event_stop(struct perf_event *event, int flags) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + + if (pmu->registered) + if (flags & PERF_EF_UPDATE) + xe_pmu_event_update(event); + + event->hw.state = PERF_HES_STOPPED; +} + +static int xe_pmu_event_add(struct perf_event *event, int flags) +{ + struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + + if (!pmu->registered) + return -ENODEV; + + if (flags & PERF_EF_START) + xe_pmu_event_start(event, flags); + + return 0; +} + +static void xe_pmu_event_del(struct perf_event *event, int flags) +{ + xe_pmu_event_stop(event, PERF_EF_UPDATE); +} + +PMU_FORMAT_ATTR(gt, "config:60-63"); +PMU_FORMAT_ATTR(function, "config:44-59"); +PMU_FORMAT_ATTR(engine_class, "config:20-27"); +PMU_FORMAT_ATTR(engine_instance, "config:12-19"); +PMU_FORMAT_ATTR(event, "config:0-11"); + +static struct attribute *pmu_format_attrs[] = { + &format_attr_event.attr, + &format_attr_engine_class.attr, + &format_attr_engine_instance.attr, + &format_attr_function.attr, + &format_attr_gt.attr, + NULL, +}; + +static const struct attribute_group pmu_format_attr_group = { + .name = "format", + .attrs = pmu_format_attrs, +}; + +static ssize_t event_attr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct perf_pmu_events_attr *pmu_attr = + container_of(attr, struct perf_pmu_events_attr, attr); + + return sysfs_emit(buf, "event=%#04llx\n", pmu_attr->id); +} + +#define XE_EVENT_ATTR(name_, v_, id_) \ + PMU_EVENT_ATTR(name_, pmu_event_ ## v_, id_, event_attr_show) + +#define XE_EVENT_ATTR_UNIT(name_, v_, unit_) \ + PMU_EVENT_ATTR_STRING(name_.unit, pmu_event_unit_ ## v_, unit_) + +#define XE_EVENT_ATTR_GROUP(v_, id_, ...) \ + static struct attribute *pmu_attr_ ##v_[] = { \ + __VA_ARGS__, \ + NULL \ + }; \ + static umode_t is_visible_##v_(struct kobject *kobj, \ + struct attribute *attr, int idx) \ + { \ + struct perf_pmu_events_attr *pmu_attr; \ + struct xe_pmu *pmu; \ + \ + pmu_attr = container_of(attr, typeof(*pmu_attr), attr.attr); \ + pmu = container_of(dev_get_drvdata(kobj_to_dev(kobj)), \ + typeof(*pmu), base); \ + \ + return event_supported(pmu, 0, id_) ? attr->mode : 0; \ + } \ + static const struct attribute_group pmu_group_ ##v_ = { \ + .name = "events", \ + .attrs = pmu_attr_ ## v_, \ + .is_visible = is_visible_ ## v_, \ + } + +#define XE_EVENT_ATTR_SIMPLE(name_, v_, id_, unit_) \ + XE_EVENT_ATTR(name_, v_, id_) \ + XE_EVENT_ATTR_UNIT(name_, v_, unit_) \ + XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr, \ + &pmu_event_unit_ ##v_.attr.attr) + +#define XE_EVENT_ATTR_NOUNIT(name_, v_, id_) \ + XE_EVENT_ATTR(name_, v_, id_) \ + XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr) + +XE_EVENT_ATTR_SIMPLE(gt-c6-residency, gt_c6_residency, XE_PMU_EVENT_GT_C6_RESIDENCY, "ms"); +XE_EVENT_ATTR_NOUNIT(engine-active-ticks, engine_active_ticks, XE_PMU_EVENT_ENGINE_ACTIVE_TICKS); +XE_EVENT_ATTR_NOUNIT(engine-total-ticks, engine_total_ticks, XE_PMU_EVENT_ENGINE_TOTAL_TICKS); +XE_EVENT_ATTR_SIMPLE(gt-actual-frequency, gt_actual_frequency, + XE_PMU_EVENT_GT_ACTUAL_FREQUENCY, "MHz"); +XE_EVENT_ATTR_SIMPLE(gt-requested-frequency, gt_requested_frequency, + XE_PMU_EVENT_GT_REQUESTED_FREQUENCY, "MHz"); + +static struct attribute *pmu_empty_event_attrs[] = { + /* Empty - all events are added as groups with .attr_update() */ + NULL, +}; + +static const struct attribute_group pmu_events_attr_group = { + .name = "events", + .attrs = pmu_empty_event_attrs, +}; + +static const struct attribute_group *pmu_events_attr_update[] = { + &pmu_group_gt_c6_residency, + &pmu_group_engine_active_ticks, + &pmu_group_engine_total_ticks, + &pmu_group_gt_actual_frequency, + &pmu_group_gt_requested_frequency, + NULL, +}; + +static void set_supported_events(struct xe_pmu *pmu) +{ + struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); + struct xe_gt *gt; + int id; + + /* If there are no GTs, don't support any GT-related events */ + if (xe->info.gt_count == 0) + return; + + if (!xe->info.skip_guc_pc) { + pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_C6_RESIDENCY); + pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_ACTUAL_FREQUENCY); + pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_REQUESTED_FREQUENCY); + } + + /* Find the first available GT to query engine event capabilities */ + for_each_gt(gt, xe, id) + break; + + if (xe_guc_engine_activity_supported(>->uc.guc)) { + pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_ACTIVE_TICKS); + pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_TOTAL_TICKS); + } +} + +/** + * xe_pmu_unregister() - Remove/cleanup PMU registration + * @arg: Ptr to pmu + */ +static void xe_pmu_unregister(void *arg) +{ + struct xe_pmu *pmu = arg; + struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); + + if (!pmu->registered) + return; + + pmu->registered = false; + + perf_pmu_unregister(&pmu->base); + kfree(pmu->name); +} + +/** + * xe_pmu_register() - Define basic PMU properties for Xe and add event callbacks. + * @pmu: the PMU object + * + * Returns 0 on success and an appropriate error code otherwise + */ +int xe_pmu_register(struct xe_pmu *pmu) +{ + struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); + static const struct attribute_group *attr_groups[] = { + &pmu_format_attr_group, + &pmu_events_attr_group, + NULL + }; + int ret = -ENOMEM; + char *name; + + BUILD_BUG_ON(XE_MAX_GT_PER_TILE != XE_PMU_MAX_GT); + + if (IS_SRIOV_VF(xe)) + return 0; + + name = kasprintf(GFP_KERNEL, "xe_%s", + dev_name(xe->drm.dev)); + if (!name) + goto err; + + /* tools/perf reserves colons as special. */ + strreplace(name, ':', '_'); + + pmu->name = name; + pmu->base.attr_groups = attr_groups; + pmu->base.attr_update = pmu_events_attr_update; + pmu->base.scope = PERF_PMU_SCOPE_SYS_WIDE; + pmu->base.module = THIS_MODULE; + pmu->base.task_ctx_nr = perf_invalid_context; + pmu->base.event_init = xe_pmu_event_init; + pmu->base.add = xe_pmu_event_add; + pmu->base.del = xe_pmu_event_del; + pmu->base.start = xe_pmu_event_start; + pmu->base.stop = xe_pmu_event_stop; + pmu->base.read = xe_pmu_event_read; + + set_supported_events(pmu); + + ret = perf_pmu_register(&pmu->base, pmu->name, -1); + if (ret) + goto err_name; + + pmu->registered = true; + + return devm_add_action_or_reset(xe->drm.dev, xe_pmu_unregister, pmu); + +err_name: + kfree(name); +err: + drm_err(&xe->drm, "Failed to register PMU (ret=%d)!\n", ret); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_pmu.h b/drivers/gpu/drm/xe/xe_pmu.h new file mode 100644 index 000000000..60c37126f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pmu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PMU_H_ +#define _XE_PMU_H_ + +#include "xe_pmu_types.h" + +#if IS_ENABLED(CONFIG_PERF_EVENTS) +int xe_pmu_register(struct xe_pmu *pmu); +#else +static inline int xe_pmu_register(struct xe_pmu *pmu) { return 0; } +#endif + +#endif + diff --git a/drivers/gpu/drm/xe/xe_pmu_types.h b/drivers/gpu/drm/xe/xe_pmu_types.h new file mode 100644 index 000000000..f5ba4d566 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pmu_types.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PMU_TYPES_H_ +#define _XE_PMU_TYPES_H_ + +#include +#include + +#define XE_PMU_MAX_GT 2 + +/** + * struct xe_pmu - PMU related data per Xe device + * + * Stores per device PMU info that includes event/perf attributes and sampling + * counters across all GTs for this device. + */ +struct xe_pmu { + /** + * @base: PMU base. + */ + struct pmu base; + /** + * @registered: PMU is registered and not in the unregistering process. + */ + bool registered; + /** + * @name: Name as registered with perf core. + */ + const char *name; + /** + * @supported_events: Bitmap of supported events, indexed by event id + */ + u64 supported_events; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.c b/drivers/gpu/drm/xe/xe_preempt_fence.c new file mode 100644 index 000000000..4aa570fe7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_preempt_fence.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_preempt_fence.h" + +#include + +#include "xe_exec_queue.h" +#include "xe_gt_printk.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_vm.h" + +static void preempt_fence_work_func(struct work_struct *w) +{ + bool cookie = dma_fence_begin_signalling(); + struct xe_preempt_fence *pfence = + container_of(w, typeof(*pfence), preempt_work); + struct xe_exec_queue *q = pfence->q; + + if (pfence->error) { + dma_fence_set_error(&pfence->base, pfence->error); + } else if (!q->ops->reset_status(q)) { + int err = q->ops->suspend_wait(q); + + if (err == -EAGAIN) { + xe_gt_dbg(q->gt, "PREEMPT FENCE RETRY guc_id=%d", + q->guc->id); + queue_work(q->vm->xe->preempt_fence_wq, + &pfence->preempt_work); + dma_fence_end_signalling(cookie); + return; + } + + if (err) + dma_fence_set_error(&pfence->base, err); + } else { + dma_fence_set_error(&pfence->base, -ENOENT); + } + + dma_fence_signal(&pfence->base); + /* + * Opt for keep everything in the fence critical section. This looks really strange since we + * have just signalled the fence, however the preempt fences are all signalled via single + * global ordered-wq, therefore anything that happens in this callback can easily block + * progress on the entire wq, which itself may prevent other published preempt fences from + * ever signalling. Therefore try to keep everything here in the callback in the fence + * critical section. For example if something below grabs a scary lock like vm->lock, + * lockdep should complain since we also hold that lock whilst waiting on preempt fences to + * complete. + */ + xe_vm_queue_rebind_worker(q->vm); + xe_exec_queue_put(q); + dma_fence_end_signalling(cookie); +} + +static const char * +preempt_fence_get_driver_name(struct dma_fence *fence) +{ + return "xe"; +} + +static const char * +preempt_fence_get_timeline_name(struct dma_fence *fence) +{ + return "preempt"; +} + +static bool preempt_fence_enable_signaling(struct dma_fence *fence) +{ + struct xe_preempt_fence *pfence = + container_of(fence, typeof(*pfence), base); + struct xe_exec_queue *q = pfence->q; + + pfence->error = q->ops->suspend(q); + /* + * Record a successful suspend so the rebind worker only resumes queues + * that were actually suspended; a failed suspend() leaves the queue + * un-suspended and must not be paired with a resume(). + */ + if (!pfence->error) + WRITE_ONCE(q->lr.suspended, true); + queue_work(q->vm->xe->preempt_fence_wq, &pfence->preempt_work); + return true; +} + +static const struct dma_fence_ops preempt_fence_ops = { + .get_driver_name = preempt_fence_get_driver_name, + .get_timeline_name = preempt_fence_get_timeline_name, + .enable_signaling = preempt_fence_enable_signaling, +}; + +/** + * xe_preempt_fence_alloc() - Allocate a preempt fence with minimal + * initialization + * + * Allocate a preempt fence, and initialize its list head. + * If the preempt_fence allocated has been armed with + * xe_preempt_fence_arm(), it must be freed using dma_fence_put(). If not, + * it must be freed using xe_preempt_fence_free(). + * + * Return: A struct xe_preempt_fence pointer used for calling into + * xe_preempt_fence_arm() or xe_preempt_fence_free(). + * An error pointer on error. + */ +struct xe_preempt_fence *xe_preempt_fence_alloc(void) +{ + struct xe_preempt_fence *pfence; + + pfence = kmalloc_obj(*pfence); + if (!pfence) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&pfence->link); + INIT_WORK(&pfence->preempt_work, preempt_fence_work_func); + + return pfence; +} + +/** + * xe_preempt_fence_free() - Free a preempt fence allocated using + * xe_preempt_fence_alloc(). + * @pfence: pointer obtained from xe_preempt_fence_alloc(); + * + * Free a preempt fence that has not yet been armed. + */ +void xe_preempt_fence_free(struct xe_preempt_fence *pfence) +{ + list_del(&pfence->link); + kfree(pfence); +} + +/** + * xe_preempt_fence_arm() - Arm a preempt fence allocated using + * xe_preempt_fence_alloc(). + * @pfence: The struct xe_preempt_fence pointer returned from + * xe_preempt_fence_alloc(). + * @q: The struct xe_exec_queue used for arming. + * @context: The dma-fence context used for arming. + * @seqno: The dma-fence seqno used for arming. + * + * Inserts the preempt fence into @context's timeline, takes @link off any + * list, and registers the struct xe_exec_queue as the xe_engine to be preempted. + * + * Return: A pointer to a struct dma_fence embedded into the preempt fence. + * This function doesn't error. + */ +struct dma_fence * +xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, + u64 context, u32 seqno) +{ + list_del_init(&pfence->link); + pfence->q = xe_exec_queue_get(q); + spin_lock_init(&pfence->lock); + dma_fence_init(&pfence->base, &preempt_fence_ops, + &pfence->lock, context, seqno); + + return &pfence->base; +} + +/** + * xe_preempt_fence_create() - Helper to create and arm a preempt fence. + * @q: The struct xe_exec_queue used for arming. + * @context: The dma-fence context used for arming. + * @seqno: The dma-fence seqno used for arming. + * + * Allocates and inserts the preempt fence into @context's timeline, + * and registers @e as the struct xe_exec_queue to be preempted. + * + * Return: A pointer to the resulting struct dma_fence on success. An error + * pointer on error. In particular if allocation fails it returns + * ERR_PTR(-ENOMEM); + */ +struct dma_fence * +xe_preempt_fence_create(struct xe_exec_queue *q, + u64 context, u32 seqno) +{ + struct xe_preempt_fence *pfence; + + pfence = xe_preempt_fence_alloc(); + if (IS_ERR(pfence)) + return ERR_CAST(pfence); + + return xe_preempt_fence_arm(pfence, q, context, seqno); +} + +bool xe_fence_is_xe_preempt(const struct dma_fence *fence) +{ + return fence->ops == &preempt_fence_ops; +} diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.h b/drivers/gpu/drm/xe/xe_preempt_fence.h new file mode 100644 index 000000000..9406c6fea --- /dev/null +++ b/drivers/gpu/drm/xe/xe_preempt_fence.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PREEMPT_FENCE_H_ +#define _XE_PREEMPT_FENCE_H_ + +#include "xe_preempt_fence_types.h" + +struct list_head; + +struct dma_fence * +xe_preempt_fence_create(struct xe_exec_queue *q, + u64 context, u32 seqno); + +struct xe_preempt_fence *xe_preempt_fence_alloc(void); + +void xe_preempt_fence_free(struct xe_preempt_fence *pfence); + +struct dma_fence * +xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, + u64 context, u32 seqno); + +static inline struct xe_preempt_fence * +to_preempt_fence(struct dma_fence *fence) +{ + return container_of(fence, struct xe_preempt_fence, base); +} + +/** + * xe_preempt_fence_link() - Return a link used to keep unarmed preempt + * fences on a list. + * @pfence: Pointer to the preempt fence. + * + * The link is embedded in the struct xe_preempt_fence. Use + * link_to_preempt_fence() to convert back to the preempt fence. + * + * Return: A pointer to an embedded struct list_head. + */ +static inline struct list_head * +xe_preempt_fence_link(struct xe_preempt_fence *pfence) +{ + return &pfence->link; +} + +/** + * to_preempt_fence_from_link() - Convert back to a preempt fence pointer + * from a link obtained with xe_preempt_fence_link(). + * @link: The struct list_head obtained from xe_preempt_fence_link(). + * + * Return: A pointer to the embedding struct xe_preempt_fence. + */ +static inline struct xe_preempt_fence * +to_preempt_fence_from_link(struct list_head *link) +{ + return container_of(link, struct xe_preempt_fence, link); +} + +bool xe_fence_is_xe_preempt(const struct dma_fence *fence); +#endif diff --git a/drivers/gpu/drm/xe/xe_preempt_fence_types.h b/drivers/gpu/drm/xe/xe_preempt_fence_types.h new file mode 100644 index 000000000..ac125c697 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_preempt_fence_types.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PREEMPT_FENCE_TYPES_H_ +#define _XE_PREEMPT_FENCE_TYPES_H_ + +#include +#include + +struct xe_exec_queue; + +/** + * struct xe_preempt_fence - Xe preempt fence + * + * hardware and triggers a callback once the xe_engine is complete. + */ +struct xe_preempt_fence { + /** @base: dma fence base */ + struct dma_fence base; + /** @link: link into list of pending preempt fences */ + struct list_head link; + /** @q: exec queue for this preempt fence */ + struct xe_exec_queue *q; + /** @preempt_work: work struct which issues preemption */ + struct work_struct preempt_work; + /** @lock: dma-fence fence lock */ + spinlock_t lock; + /** @error: preempt fence is in error state */ + int error; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_printk.h b/drivers/gpu/drm/xe/xe_printk.h new file mode 100644 index 000000000..c5be2385a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_printk.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PRINTK_H_ +#define _XE_PRINTK_H_ + +#include + +#include "xe_device_types.h" + +#define __XE_PRINTK_FMT(_xe, _fmt, _args...) _fmt, ##_args + +#define xe_printk(_xe, _level, _fmt, ...) \ + drm_##_level(&(_xe)->drm, __XE_PRINTK_FMT((_xe), _fmt, ## __VA_ARGS__)) + +#define xe_err(_xe, _fmt, ...) \ + xe_printk((_xe), err, _fmt, ##__VA_ARGS__) + +#define xe_err_once(_xe, _fmt, ...) \ + xe_printk((_xe), err_once, _fmt, ##__VA_ARGS__) + +#define xe_err_ratelimited(_xe, _fmt, ...) \ + xe_printk((_xe), err_ratelimited, _fmt, ##__VA_ARGS__) + +#define xe_warn(_xe, _fmt, ...) \ + xe_printk((_xe), warn, _fmt, ##__VA_ARGS__) + +#define xe_notice(_xe, _fmt, ...) \ + xe_printk((_xe), notice, _fmt, ##__VA_ARGS__) + +#define xe_info(_xe, _fmt, ...) \ + xe_printk((_xe), info, _fmt, ##__VA_ARGS__) + +#define xe_dbg(_xe, _fmt, ...) \ + xe_printk((_xe), dbg, _fmt, ##__VA_ARGS__) + +#define xe_WARN_type(_xe, _type, _condition, _fmt, ...) \ + drm_WARN##_type(&(_xe)->drm, _condition, _fmt, ## __VA_ARGS__) + +#define xe_WARN(_xe, _condition, _fmt, ...) \ + xe_WARN_type((_xe),, _condition, __XE_PRINTK_FMT((_xe), _fmt, ## __VA_ARGS__)) + +#define xe_WARN_ONCE(_xe, _condition, _fmt, ...) \ + xe_WARN_type((_xe), _ONCE, _condition, __XE_PRINTK_FMT((_xe), _fmt, ## __VA_ARGS__)) + +#define xe_WARN_ON(_xe, _condition) \ + xe_WARN((_xe), _condition, "%s(%s)", "WARN_ON", __stringify(_condition)) + +#define xe_WARN_ON_ONCE(_xe, _condition) \ + xe_WARN_ONCE((_xe), _condition, "%s(%s)", "WARN_ON_ONCE", __stringify(_condition)) + +static inline void __xe_printfn_err(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_device *xe = p->arg; + + xe_err(xe, "%pV", vaf); +} + +static inline void __xe_printfn_info(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_device *xe = p->arg; + + xe_info(xe, "%pV", vaf); +} + +static inline void __xe_printfn_dbg(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_device *xe = p->arg; + struct drm_printer ddp; + + /* + * The original xe_dbg() callsite annotations are useless here, + * redirect to the tweaked drm_dbg_printer() instead. + */ + ddp = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL); + ddp.origin = p->origin; + + drm_printf(&ddp, __XE_PRINTK_FMT(xe, "%pV", vaf)); +} + +/** + * xe_err_printer - Construct a &drm_printer that outputs to xe_err() + * @xe: the &xe_device pointer to use in xe_err() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_err_printer(struct xe_device *xe) +{ + struct drm_printer p = { + .printfn = __xe_printfn_err, + .arg = xe, + }; + return p; +} + +/** + * xe_info_printer - Construct a &drm_printer that outputs to xe_info() + * @xe: the &xe_device pointer to use in xe_info() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_info_printer(struct xe_device *xe) +{ + struct drm_printer p = { + .printfn = __xe_printfn_info, + .arg = xe, + }; + return p; +} + +/** + * xe_dbg_printer - Construct a &drm_printer that outputs like xe_dbg() + * @xe: the &xe_device pointer to use in xe_dbg() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_dbg_printer(struct xe_device *xe) +{ + struct drm_printer p = { + .printfn = __xe_printfn_dbg, + .arg = xe, + .origin = (const void *)_THIS_IP_, + }; + return p; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_psmi.c b/drivers/gpu/drm/xe/xe_psmi.c new file mode 100644 index 000000000..899b01f72 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_psmi.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_configfs.h" +#include "xe_psmi.h" + +/* + * PSMI capture support + * + * Requirement for PSMI capture is to have a physically contiguous buffer. The + * PSMI tool owns doing all necessary configuration (MMIO register writes are + * done from user-space). However, KMD needs to provide the PSMI tool with the + * required physical address of the base of PSMI buffer in case of VRAM. + * + * VRAM backed PSMI buffer: + * Buffer is allocated as GEM object and with XE_BO_CREATE_PINNED_BIT flag which + * creates a contiguous allocation. The physical address is returned from + * psmi_debugfs_capture_addr_show(). PSMI tool can mmap the buffer via the + * PCIBAR through sysfs. + * + * SYSTEM memory backed PSMI buffer: + * Interface here does not support allocating from SYSTEM memory region. The + * PSMI tool needs to allocate memory themselves using hugetlbfs. In order to + * get the physical address, user-space can query /proc/[pid]/pagemap. As an + * alternative, CMA debugfs could also be used to allocate reserved CMA memory. + */ + +static bool psmi_enabled(struct xe_device *xe) +{ + return xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev)); +} + +static void psmi_free_object(struct xe_bo *bo) +{ + xe_bo_lock(bo, NULL); + xe_bo_unpin(bo); + xe_bo_unlock(bo); + xe_bo_put(bo); +} + +/* + * Free PSMI capture buffer objects. + */ +static void psmi_cleanup(struct xe_device *xe) +{ + unsigned long id, region_mask = xe->psmi.region_mask; + struct xe_bo *bo; + + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + bo = xe->psmi.capture_obj[id]; + if (bo) { + psmi_free_object(bo); + xe->psmi.capture_obj[id] = NULL; + } + } +} + +static struct xe_bo *psmi_alloc_object(struct xe_device *xe, + unsigned int id, size_t bo_size) +{ + struct xe_tile *tile; + + xe_assert(xe, id); + xe_assert(xe, bo_size); + + tile = &xe->tiles[id - 1]; + + /* VRAM: Allocate GEM object for the capture buffer */ + return xe_bo_create_pin_range_novm(xe, tile, bo_size, 0, ~0ull, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_PINNED | + XE_BO_FLAG_PINNED_LATE_RESTORE | + XE_BO_FLAG_NEEDS_CPU_ACCESS); +} + +/* + * Allocate PSMI capture buffer objects (via debugfs set function), based on + * which regions the user has selected in region_mask. @size: size in bytes + * (should be power of 2) + * + * Always release/free the current buffer objects before attempting to allocate + * new ones. Size == 0 will free all current buffers. + * + * Note, we don't write any registers as the capture tool is already configuring + * all PSMI registers itself via mmio space. + */ +static int psmi_resize_object(struct xe_device *xe, size_t size) +{ + unsigned long id, region_mask = xe->psmi.region_mask; + struct xe_bo *bo = NULL; + int err = 0; + + /* if resizing, free currently allocated buffers first */ + psmi_cleanup(xe); + + /* can set size to 0, in which case, now done */ + if (!size) + return 0; + + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + bo = psmi_alloc_object(xe, id, size); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + break; + } + xe->psmi.capture_obj[id] = bo; + + drm_info(&xe->drm, + "PSMI capture size requested: %zu bytes, allocated: %lu:%zu\n", + size, id, bo ? xe_bo_size(bo) : 0); + } + + /* on error, reverse what was allocated */ + if (err) + psmi_cleanup(xe); + + return err; +} + +/* + * Returns an address for the capture tool to use to find start of capture + * buffer. Capture tool requires the capability to have a buffer allocated per + * each tile (VRAM region), thus we return an address for each region. + */ +static int psmi_debugfs_capture_addr_show(struct seq_file *m, void *data) +{ + struct xe_device *xe = m->private; + unsigned long id, region_mask; + struct xe_bo *bo; + u64 val; + + region_mask = xe->psmi.region_mask; + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + /* VRAM region */ + bo = xe->psmi.capture_obj[id]; + if (!bo) + continue; + + /* pinned, so don't need bo_lock */ + val = __xe_bo_addr(bo, 0, PAGE_SIZE); + seq_printf(m, "%ld: 0x%llx\n", id, val); + } + + return 0; +} + +/* + * Return capture buffer size, using the size from first allocated object that + * is found. This works because all objects must be of the same size. + */ +static int psmi_debugfs_capture_size_get(void *data, u64 *val) +{ + unsigned long id, region_mask; + struct xe_device *xe = data; + struct xe_bo *bo; + + region_mask = xe->psmi.region_mask; + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + bo = xe->psmi.capture_obj[id]; + if (bo) { + *val = xe_bo_size(bo); + return 0; + } + } + + /* no capture objects are allocated */ + *val = 0; + + return 0; +} + +/* + * Set size of PSMI capture buffer. This triggers the allocation of capture + * buffer in each memory region as specified with prior write to + * psmi_capture_region_mask. + */ +static int psmi_debugfs_capture_size_set(void *data, u64 val) +{ + struct xe_device *xe = data; + + /* user must have specified at least one region */ + if (!xe->psmi.region_mask) + return -EINVAL; + + return psmi_resize_object(xe, val); +} + +static int psmi_debugfs_capture_region_mask_get(void *data, u64 *val) +{ + struct xe_device *xe = data; + + *val = xe->psmi.region_mask; + + return 0; +} + +/* + * Select VRAM regions for multi-tile devices, only allowed when buffer is not + * currently allocated. + */ +static int psmi_debugfs_capture_region_mask_set(void *data, u64 region_mask) +{ + struct xe_device *xe = data; + u64 size = 0; + + /* SMEM is not supported (see comments at top of file) */ + if (region_mask & 0x1) + return -EOPNOTSUPP; + + /* input bitmask should contain only valid TTM regions */ + if (!region_mask || region_mask & ~xe->info.mem_region_mask) + return -EINVAL; + + /* only allow setting mask if buffer is not yet allocated */ + psmi_debugfs_capture_size_get(xe, &size); + if (size) + return -EBUSY; + + xe->psmi.region_mask = region_mask; + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(psmi_debugfs_capture_addr); + +DEFINE_DEBUGFS_ATTRIBUTE(psmi_debugfs_capture_region_mask_fops, + psmi_debugfs_capture_region_mask_get, + psmi_debugfs_capture_region_mask_set, + "0x%llx\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(psmi_debugfs_capture_size_fops, + psmi_debugfs_capture_size_get, + psmi_debugfs_capture_size_set, + "%lld\n"); + +void xe_psmi_debugfs_register(struct xe_device *xe) +{ + struct drm_minor *minor; + + if (!psmi_enabled(xe)) + return; + + minor = xe->drm.primary; + if (!minor->debugfs_root) + return; + + debugfs_create_file("psmi_capture_addr", + 0400, minor->debugfs_root, xe, + &psmi_debugfs_capture_addr_fops); + + debugfs_create_file("psmi_capture_region_mask", + 0600, minor->debugfs_root, xe, + &psmi_debugfs_capture_region_mask_fops); + + debugfs_create_file("psmi_capture_size", + 0600, minor->debugfs_root, xe, + &psmi_debugfs_capture_size_fops); +} + +static void psmi_fini(void *arg) +{ + psmi_cleanup(arg); +} + +int xe_psmi_init(struct xe_device *xe) +{ + if (!psmi_enabled(xe)) + return 0; + + return devm_add_action(xe->drm.dev, psmi_fini, xe); +} diff --git a/drivers/gpu/drm/xe/xe_psmi.h b/drivers/gpu/drm/xe/xe_psmi.h new file mode 100644 index 000000000..b1dfba80d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_psmi.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PSMI_H_ +#define _XE_PSMI_H_ + +struct xe_device; + +int xe_psmi_init(struct xe_device *xe); +void xe_psmi_debugfs_register(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c new file mode 100644 index 000000000..a07316a45 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -0,0 +1,2934 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_pt.h" + +#include "regs/xe_gtt_defs.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_drm_client.h" +#include "xe_exec_queue.h" +#include "xe_gt.h" +#include "xe_gt_stats.h" +#include "xe_migrate.h" +#include "xe_page_reclaim.h" +#include "xe_pat.h" +#include "xe_pt_types.h" +#include "xe_pt_walk.h" +#include "xe_res_cursor.h" +#include "xe_sched_job.h" +#include "xe_svm.h" +#include "xe_sync.h" +#include "xe_tlb_inval_job.h" +#include "xe_trace.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_userptr.h" +#include "xe_vm.h" + +struct xe_pt_dir { + struct xe_pt pt; + /** @children: Array of page-table child nodes */ + struct xe_ptw *children[XE_PDES]; + /** @staging: Array of page-table staging nodes */ + struct xe_ptw *staging[XE_PDES]; +}; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) +#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr)) +#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr) +#else +#define xe_pt_set_addr(__xe_pt, __addr) +#define xe_pt_addr(__xe_pt) 0ull +#endif + +static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48}; +static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48}; + +#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1) + +static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt) +{ + return container_of(pt, struct xe_pt_dir, pt); +} + +static struct xe_pt * +xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index) +{ + return container_of(pt_dir->staging[index], struct xe_pt, base); +} + +static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, + unsigned int level) +{ + struct xe_device *xe = tile_to_xe(tile); + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); + u8 id = tile->id; + + if (!xe_vm_has_scratch(vm)) + return 0; + + if (level > MAX_HUGEPTE_LEVEL) + return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, + 0); + + return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | + XE_PTE_NULL; +} + +static void xe_pt_free(struct xe_pt *pt) +{ + if (pt->level) + kfree(as_xe_pt_dir(pt)); + else + kfree(pt); +} + +/** + * xe_pt_create() - Create a page-table. + * @vm: The vm to create for. + * @tile: The tile to create for. + * @level: The page-table level. + * @exec: The drm_exec object used to lock the vm. + * + * Allocate and initialize a single struct xe_pt metadata structure. Also + * create the corresponding page-table bo, but don't initialize it. If the + * level is grater than zero, then it's assumed to be a directory page- + * table and the directory structure is also allocated and initialized to + * NULL pointers. + * + * Return: A valid struct xe_pt pointer on success, Pointer error code on + * error. + */ +struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, + unsigned int level, struct drm_exec *exec) +{ + struct xe_pt *pt; + struct xe_bo *bo; + u32 bo_flags; + int err; + + if (level) { + struct xe_pt_dir *dir = kzalloc_obj(*dir); + + pt = (dir) ? &dir->pt : NULL; + } else { + pt = kzalloc_obj(*pt); + } + if (!pt) + return ERR_PTR(-ENOMEM); + + bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE | + XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE; + if (vm->xef) /* userspace */ + bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM; + + pt->level = level; + + drm_WARN_ON(&vm->xe->drm, IS_ERR_OR_NULL(exec)); + bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, + ttm_bo_type_kernel, + bo_flags, exec); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + goto err_kfree; + } + pt->bo = bo; + pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; + pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL; + + if (vm->xef) + xe_drm_client_add_bo(vm->xef->client, pt->bo); + xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL); + + return pt; + +err_kfree: + xe_pt_free(pt); + return ERR_PTR(err); +} +ALLOW_ERROR_INJECTION(xe_pt_create, ERRNO); + +/** + * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero + * entries. + * @tile: The tile the scratch pagetable of which to use. + * @vm: The vm we populate for. + * @pt: The pagetable the bo of which to initialize. + * + * Populate the page-table bo of @pt with entries pointing into the tile's + * scratch page-table tree if any. Otherwise populate with zeros. + */ +void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, + struct xe_pt *pt) +{ + struct iosys_map *map = &pt->bo->vmap; + u64 empty; + int i; + + if (!xe_vm_has_scratch(vm)) { + /* + * FIXME: Some memory is allocated already allocated to zero? + * Find out which memory that is and avoid this memset... + */ + xe_map_memset(vm->xe, map, 0, 0, SZ_4K); + } else { + empty = __xe_pt_empty_pte(tile, vm, pt->level); + for (i = 0; i < XE_PDES; i++) + xe_pt_write(vm->xe, map, i, empty); + } +} + +/** + * xe_pt_shift() - Return the ilog2 value of the size of the address range of + * a page-table at a certain level. + * @level: The level. + * + * Return: The ilog2 value of the size of the address range of a page-table + * at level @level. + */ +unsigned int xe_pt_shift(unsigned int level) +{ + return XE_PTE_SHIFT + XE_PDE_SHIFT * level; +} + +/** + * xe_pt_destroy() - Destroy a page-table tree. + * @pt: The root of the page-table tree to destroy. + * @flags: vm flags. Currently unused. + * @deferred: List head of lockless list for deferred putting. NULL for + * immediate putting. + * + * Puts the page-table bo, recursively calls xe_pt_destroy on all children + * and finally frees @pt. TODO: Can we remove the @flags argument? + */ +void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) +{ + int i; + + if (!pt) + return; + + XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); + xe_bo_unpin(pt->bo); + xe_bo_put_deferred(pt->bo, deferred); + + if (pt->level > 0 && pt->num_live) { + struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); + + for (i = 0; i < XE_PDES; i++) { + if (xe_pt_entry_staging(pt_dir, i)) + xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags, + deferred); + } + } + xe_pt_free(pt); +} + +/** + * xe_pt_clear() - Clear a page-table. + * @xe: xe device. + * @pt: The page-table. + * + * Clears page-table by setting to zero. + */ +void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt) +{ + struct iosys_map *map = &pt->bo->vmap; + + xe_map_memset(xe, map, 0, 0, SZ_4K); +} + +/** + * DOC: Pagetable building + * + * Below we use the term "page-table" for both page-directories, containing + * pointers to lower level page-directories or page-tables, and level 0 + * page-tables that contain only page-table-entries pointing to memory pages. + * + * When inserting an address range in an already existing page-table tree + * there will typically be a set of page-tables that are shared with other + * address ranges, and a set that are private to this address range. + * The set of shared page-tables can be at most two per level, + * and those can't be updated immediately because the entries of those + * page-tables may still be in use by the gpu for other mappings. Therefore + * when inserting entries into those, we instead stage those insertions by + * adding insertion data into struct xe_vm_pgtable_update structures. This + * data, (subtrees for the cpu and page-table-entries for the gpu) is then + * added in a separate commit step. CPU-data is committed while still under the + * vm lock, the object lock and for userptr, the notifier lock in read mode. + * The GPU async data is committed either by the GPU or CPU after fulfilling + * relevant dependencies. + * For non-shared page-tables (and, in fact, for shared ones that aren't + * existing at the time of staging), we add the data in-place without the + * special update structures. This private part of the page-table tree will + * remain disconnected from the vm page-table tree until data is committed to + * the shared page tables of the vm tree in the commit phase. + */ + +struct xe_pt_update { + /** @update: The update structure we're building for this parent. */ + struct xe_vm_pgtable_update *update; + /** @parent: The parent. Used to detect a parent change. */ + struct xe_pt *parent; + /** @preexisting: Whether the parent was pre-existing or allocated */ + bool preexisting; +}; + +/** + * struct xe_pt_stage_bind_walk - Walk state for the stage_bind walk. + */ +struct xe_pt_stage_bind_walk { + /** @base: The base class. */ + struct xe_pt_walk base; + + /* Input parameters for the walk */ + /** @vm: The vm we're building for. */ + struct xe_vm *vm; + /** @tile: The tile we're building for. */ + struct xe_tile *tile; + /** @default_vram_pte: PTE flag only template for VRAM. No address is associated */ + u64 default_vram_pte; + /** @default_system_pte: PTE flag only template for System. No address is associated */ + u64 default_system_pte; + /** @dma_offset: DMA offset to add to the PTE. */ + u64 dma_offset; + /** + * @needs_64K: This address range enforces 64K alignment and + * granularity on VRAM. + */ + bool needs_64K; + /** @clear_pt: clear page table entries during the bind walk */ + bool clear_pt; + /** + * @target_leaf_level: Page-table level at which to emit leaf PTEs + * 0 for normal 4K/64K mappings, 1 for 2M huge pages, and 2 for 1G huge + * pages. The walk still traverses from the root down; this field tells + * xe_pt_stage_bind_entry() to treat the selected level as a leaf instead + * of descending further. + */ + u32 target_leaf_level; + /** + * @vma: VMA being mapped + */ + struct xe_vma *vma; + + /* Also input, but is updated during the walk*/ + /** @curs: The DMA address cursor. */ + struct xe_res_cursor *curs; + /** @va_curs_start: The Virtual address corresponding to @curs->start */ + u64 va_curs_start; + + /* Output */ + /** @wupd: Walk output data for page-table updates. */ + struct xe_walk_update { + /** @wupd.entries: Caller provided storage. */ + struct xe_vm_pgtable_update *entries; + /** @wupd.num_used_entries: Number of update @entries used. */ + unsigned int num_used_entries; + /** @wupd.updates: Tracks the update entry at a given level */ + struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1]; + } wupd; + + /* Walk state */ + /** + * @l0_end_addr: The end address of the current l0 leaf. Used for + * 64K granularity detection. + */ + u64 l0_end_addr; + /** @addr_64K: The start address of the current 64K chunk. */ + u64 addr_64K; + /** @found_64K: Whether @add_64K actually points to a 64K chunk. */ + bool found_64K; +}; + +static int +xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, + pgoff_t offset, bool alloc_entries) +{ + struct xe_pt_update *upd = &wupd->updates[parent->level]; + struct xe_vm_pgtable_update *entry; + + /* + * For *each level*, we could only have one active + * struct xt_pt_update at any one time. Once we move on to a + * new parent and page-directory, the old one is complete, and + * updates are either already stored in the build tree or in + * @wupd->entries + */ + if (likely(upd->parent == parent)) + return 0; + + upd->parent = parent; + upd->preexisting = true; + + if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) + return -EINVAL; + + entry = wupd->entries + wupd->num_used_entries++; + upd->update = entry; + entry->ofs = offset; + entry->pt_bo = parent->bo; + entry->pt = parent; + entry->flags = 0; + entry->qwords = 0; + entry->pt_bo->update_index = -1; + + if (alloc_entries) { + entry->pt_entries = kmalloc_objs(*entry->pt_entries, XE_PDES); + if (!entry->pt_entries) + return -ENOMEM; + } + + return 0; +} + +/* + * NOTE: This is a very frequently called function so we allow ourselves + * to annotate (using branch prediction hints) the fastpath of updating a + * non-pre-existing pagetable with leaf ptes. + */ +static int +xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, + pgoff_t offset, struct xe_pt *xe_child, u64 pte) +{ + struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; + struct xe_pt_update *child_upd = xe_child ? + &xe_walk->wupd.updates[xe_child->level] : NULL; + int ret; + + ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); + if (unlikely(ret)) + return ret; + + /* + * Register this new pagetable so that it won't be recognized as + * a shared pagetable by a subsequent insertion. + */ + if (unlikely(child_upd)) { + child_upd->update = NULL; + child_upd->parent = xe_child; + child_upd->preexisting = false; + } + + if (likely(!upd->preexisting)) { + /* Continue building a non-connected subtree. */ + struct iosys_map *map = &parent->bo->vmap; + + if (unlikely(xe_child)) { + parent->base.children[offset] = &xe_child->base; + parent->base.staging[offset] = &xe_child->base; + } + + xe_pt_write(xe_walk->vm->xe, map, offset, pte); + parent->num_live++; + } else { + /* Shared pt. Stage update. */ + unsigned int idx; + struct xe_vm_pgtable_update *entry = upd->update; + + idx = offset - entry->ofs; + entry->pt_entries[idx].pt = xe_child; + entry->pt_entries[idx].pte = pte; + entry->qwords++; + } + + return 0; +} + +static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, + struct xe_pt_stage_bind_walk *xe_walk) +{ + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); + u64 size, dma; + + if (level > MAX_HUGEPTE_LEVEL) + return false; + + /* Does the virtual range requested cover a huge pte? */ + if (!xe_pt_covers(addr, next, level, &xe_walk->base)) + return false; + + /* null VMA's and purged BO's do not have dma addresses */ + if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo))) + return true; + + /* if we are clearing page table, no dma addresses*/ + if (xe_walk->clear_pt) + return true; + + /* Does the DMA segment cover the whole pte? */ + if (next - xe_walk->va_curs_start > xe_walk->curs->size) + return false; + + /* Is the DMA address huge PTE size aligned? */ + size = next - addr; + dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); + + return IS_ALIGNED(dma, size); +} + +/* + * Scan the requested mapping to check whether it can be done entirely + * with 64K PTEs. + */ +static bool +xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) +{ + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); + struct xe_res_cursor curs = *xe_walk->curs; + + if (!IS_ALIGNED(addr, SZ_64K)) + return false; + + if (next > xe_walk->l0_end_addr) + return false; + + /* null VMA's and purged BO's do not have dma addresses */ + if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo))) + return true; + + xe_res_next(&curs, addr - xe_walk->va_curs_start); + for (; addr < next; addr += SZ_64K) { + if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K) + return false; + + xe_res_next(&curs, SZ_64K); + } + + return addr == next; +} + +/* + * For non-compact "normal" 4K level-0 pagetables, we want to try to group + * addresses together in 64K-contigous regions to add a 64K TLB hint for the + * device to the PTE. + * This function determines whether the address is part of such a + * segment. For VRAM in normal pagetables, this is strictly necessary on + * some devices. + */ +static bool +xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) +{ + /* Address is within an already found 64k region */ + if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) + return true; + + xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); + xe_walk->addr_64K = addr; + + return xe_walk->found_64K; +} + +static bool xe_pt_huge_leaf_allowed(u64 addr, u64 next, unsigned int level, + struct xe_pt_stage_bind_walk *xe_walk) +{ + if (xe_walk->clear_pt) + return xe_pt_hugepte_possible(addr, next, level, xe_walk); + + if (!xe_debug_page_size_supported(xe_walk->vm->xe)) + return xe_pt_hugepte_possible(addr, next, level, xe_walk); + + if (!xe_walk->target_leaf_level) + return xe_pt_hugepte_possible(addr, next, level, xe_walk); + + if (level == xe_walk->target_leaf_level) + return xe_pt_hugepte_possible(addr, next, level, xe_walk); + + return false; +} + +static bool xe_pt_exact_leaf_required_but_invalid(u64 addr, u64 next, + unsigned int level, + struct xe_pt_stage_bind_walk *xe_walk) +{ + struct xe_device *xe = xe_walk->vm->xe; + + if (!xe_debug_page_size_mode_not_none(xe)) + return false; + + return !xe_walk->clear_pt && + xe_walk->target_leaf_level && + level == xe_walk->target_leaf_level && + !xe_pt_hugepte_possible(addr, next, level, xe_walk); +} + +static int +xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_stage_bind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + u16 pat_index = xe_walk->vma->attr.pat_index; + struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); + struct xe_vm *vm = xe_walk->vm; + struct xe_pt *xe_child; + bool covers; + int ret = 0; + u64 pte; + + if (xe_pt_exact_leaf_required_but_invalid(addr, next, level, xe_walk)) + return -EINVAL; + + /* + * Is this a leaf entry? + * Always create a 4K leaf at level 0. For huge pages (level > 0), + * validate alignment and size with xe_pt_hugepte_possible(). + * When target_leaf_level is non-zero, only that huge-page level is + * accepted for normal bind walks. Clear walks remain unconstrained so + * existing huge leaves can be cleared without descending further. + */ + if (level == 0 || xe_pt_huge_leaf_allowed(addr, next, level, xe_walk)) { + struct xe_res_cursor *curs = xe_walk->curs; + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); + bool is_null_or_purged = xe_vma_is_null(xe_walk->vma) || + (bo && xe_bo_is_purged(bo)); + bool is_vram = is_null_or_purged ? false : xe_res_is_vram(curs); + + XE_WARN_ON(xe_walk->va_curs_start != addr); + + if (xe_walk->clear_pt) { + pte = 0; + } else { + /* + * For purged BOs, treat like null VMAs - pass address 0. + * The pte_encode_vma will set XE_PTE_NULL flag for scratch mapping. + */ + pte = vm->pt_ops->pte_encode_vma(is_null_or_purged ? 0 : + xe_res_dma(curs) + + xe_walk->dma_offset, + xe_walk->vma, + pat_index, level); + if (!is_null_or_purged) + pte |= is_vram ? xe_walk->default_vram_pte : + xe_walk->default_system_pte; + + /* + * Set the XE_PTE_PS64 hint if possible, otherwise if + * this device *requires* 64K PTE size for VRAM, fail. + */ + if (level == 0 && !xe_parent->is_compact) { + if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) { + xe_walk->vma->gpuva.flags |= + XE_VMA_PTE_64K; + pte |= XE_PTE_PS64; + } else if (XE_WARN_ON(xe_walk->needs_64K && + is_vram)) { + return -EINVAL; + } + } + } + + ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte); + if (unlikely(ret)) + return ret; + + if (!is_null_or_purged && !xe_walk->clear_pt) + xe_res_next(curs, next - addr); + xe_walk->va_curs_start = next; + xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); + *action = ACTION_CONTINUE; + + return ret; + } + + /* + * Descending to lower level. Determine if we need to allocate a + * new page table or -directory, which we do if there is no + * previous one or there is one we can completely replace. + */ + if (level == 1) { + walk->shifts = xe_normal_pt_shifts; + xe_walk->l0_end_addr = next; + } + + covers = xe_pt_covers(addr, next, level, &xe_walk->base); + if (covers || !*child) { + u64 flags = 0; + + xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1, + xe_vm_validation_exec(vm)); + if (IS_ERR(xe_child)) + return PTR_ERR(xe_child); + + xe_pt_set_addr(xe_child, + round_down(addr, 1ull << walk->shifts[level])); + + if (!covers) + xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); + + *child = &xe_child->base; + + /* + * Prefer the compact pagetable layout for L0 if possible. Only + * possible if VMA covers entire 2MB region as compact 64k and + * 4k pages cannot be mixed within a 2MB region. + * TODO: Suballocate the pt bo to avoid wasting a lot of + * memory. + */ + if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && + covers && xe_pt_scan_64K(addr, next, xe_walk)) { + walk->shifts = xe_compact_pt_shifts; + xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; + flags |= XE_PDE_64K; + xe_child->is_compact = true; + } + + pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0) | flags; + ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child, + pte); + } + + *action = ACTION_SUBTREE; + return ret; +} + +static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { + .pt_entry = xe_pt_stage_bind_entry, +}; + +/* + * Default atomic expectations for different allocation scenarios are as follows: + * + * 1. Traditional API: When the VM is not in LR mode: + * - Device atomics are expected to function with all allocations. + * + * 2. Compute/SVM API: When the VM is in LR mode: + * - Device atomics are the default behavior when the bo is placed in a single region. + * - In all other cases device atomics will be disabled with AE=0 until an application + * request differently using a ioctl like madvise. + */ +static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma) +{ + if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU) + return false; + + return true; +} + +static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma) +{ + struct xe_device *xe = vm->xe; + struct xe_bo *bo = xe_vma_bo(vma); + + if (!xe->info.has_device_atomics_on_smem || + vma->attr.atomic_access == DRM_XE_ATOMIC_CPU) + return false; + + if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE) + return true; + + /* + * If a SMEM+LMEM allocation is backed by SMEM, a device + * atomics will cause a gpu page fault and which then + * gets migrated to LMEM, bind such allocations with + * device atomics enabled. + */ + return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) || + (bo && xe_bo_has_single_placement(bo)))); +} + +static u32 xe_pt_target_leaf_level_from_bo(struct xe_device *xe, + struct xe_vma *vma) +{ + struct xe_bo *bo = xe_vma_bo(vma); + + if (!xe_debug_page_size_mode_not_none(xe)) + return 0; + + if (!bo || !xe_bo_is_vram(bo) || !(bo->flags & XE_BO_FLAG_USER)) + return 0; + + if (bo->flags & XE_BO_FLAG_NEEDS_1G) + return 2; + + if (bo->flags & XE_BO_FLAG_NEEDS_2M) + return 1; + + return 0; +} + +/** + * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address + * range. + * @tile: The tile we're building for. + * @vma: The vma indicating the address range. + * @range: The range indicating the address range. + * @entries: Storage for the update entries used for connecting the tree to + * the main tree at commit time. + * @num_entries: On output contains the number of @entries used. + * @clear_pt: Clear the page table entries. + * + * This function builds a disconnected page-table tree for a given address + * range. The tree is connected to the main vm tree for the gpu using + * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind(). + * The function builds xe_vm_pgtable_update structures for already existing + * shared page-tables, and non-existing shared and non-shared page-tables + * are built and populated directly. + * + * Return 0 on success, negative error code on error. + */ +static int +xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, + struct xe_svm_range *range, + struct xe_vm_pgtable_update *entries, + u32 *num_entries, bool clear_pt) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_bo *bo = xe_vma_bo(vma); + struct xe_res_cursor curs = {}; + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_pt_stage_bind_walk xe_walk = { + .base = { + .ops = &xe_pt_stage_bind_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + .staging = true, + }, + .vm = vm, + .tile = tile, + .curs = &curs, + .va_curs_start = range ? xe_svm_range_start(range) : + xe_vma_start(vma), + .vma = vma, + .wupd.entries = entries, + .clear_pt = clear_pt, + }; + struct xe_pt *pt = vm->pt_root[tile->id]; + int ret; + bool is_purged = false; + + /* + * Check if BO is purged: + * - Scratch VMs: Use scratch PTEs (XE_PTE_NULL) for safe zero reads + * - Non-scratch VMs: Clear PTEs to zero (non-present) to avoid mapping to phys addr 0 + * + * For non-scratch VMs, we force clear_pt=true so leaf PTEs become completely + * zero instead of creating a PRESENT mapping to physical address 0. + */ + if (bo && xe_bo_is_purged(bo)) { + is_purged = true; + + /* + * For non-scratch VMs, a NULL rebind should use zero PTEs + * (non-present), not a present PTE to phys 0. + */ + if (!xe_vm_has_scratch(vm)) + xe_walk.clear_pt = true; + } + + if (range) { + /* Move this entire thing to xe_svm.c? */ + xe_svm_notifier_lock(vm); + if (!xe_svm_range_pages_valid(range)) { + xe_svm_range_debug(range, "BIND PREPARE - RETRY"); + xe_svm_notifier_unlock(vm); + return -EAGAIN; + } + if (xe_svm_range_has_dma_mapping(range)) { + xe_res_first_dma(range->pages.dma_addr, 0, + xe_svm_range_size(range), + &curs); + xe_svm_range_debug(range, "BIND PREPARE - MIXED"); + } else { + xe_assert(xe, false); + } + /* + * Note, when unlocking the resource cursor dma addresses may become + * stale, but the bind will be aborted anyway at commit time. + */ + xe_svm_notifier_unlock(vm); + } + + xe_walk.target_leaf_level = xe_pt_target_leaf_level_from_bo(xe, vma); + xe_walk.needs_64K = (vm->flags & XE_VM_FLAG_64K); + if (clear_pt) { + xe_assert(xe, !range); + curs.size = xe_vma_size(vma); + goto walk_pt; + } + + if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { + xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0; + xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ? + XE_USM_PPGTT_PTE_AE : 0; + } + + xe_walk.default_vram_pte |= XE_PPGTT_PTE_DM; + xe_walk.dma_offset = (bo && !is_purged) ? vram_region_gpu_offset(bo->ttm.resource) : 0; + if (!range) + xe_bo_assert_held(bo); + + if (!xe_vma_is_null(vma) && !range && !is_purged) { + if (xe_vma_is_userptr(vma)) + xe_res_first_dma(to_userptr_vma(vma)->userptr.pages.dma_addr, 0, + xe_vma_size(vma), &curs); + else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) + xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), + xe_vma_size(vma), &curs); + else + xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma), + xe_vma_size(vma), &curs); + } else if (!range) { + curs.size = xe_vma_size(vma); + } + +walk_pt: + ret = xe_pt_walk_range(&pt->base, pt->level, + range ? xe_svm_range_start(range) : xe_vma_start(vma), + range ? xe_svm_range_end(range) : xe_vma_end(vma), + &xe_walk.base); + + *num_entries = xe_walk.wupd.num_used_entries; + return ret; +} + +/** + * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a + * shared pagetable. + * @addr: The start address within the non-shared pagetable. + * @end: The end address within the non-shared pagetable. + * @level: The level of the non-shared pagetable. + * @walk: Walk info. The function adjusts the walk action. + * @action: next action to perform (see enum page_walk_action) + * @offset: Ignored on input, First non-shared entry on output. + * @end_offset: Ignored on input, Last non-shared entry + 1 on output. + * + * A non-shared page-table has some entries that belong to the address range + * and others that don't. This function determines the entries that belong + * fully to the address range. Depending on level, some entries may + * partially belong to the address range (that can't happen at level 0). + * The function detects that and adjust those offsets to not include those + * partial entries. Iff it does detect partial entries, we know that there must + * be shared page tables also at lower levels, so it adjusts the walk action + * accordingly. + * + * Return: true if there were non-shared entries, false otherwise. + */ +static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level, + struct xe_pt_walk *walk, + enum page_walk_action *action, + pgoff_t *offset, pgoff_t *end_offset) +{ + u64 size = 1ull << walk->shifts[level]; + + *offset = xe_pt_offset(addr, level, walk); + *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset; + + if (!level) + return true; + + /* + * If addr or next are not size aligned, there are shared pts at lower + * level, so in that case traverse down the subtree + */ + *action = ACTION_CONTINUE; + if (!IS_ALIGNED(addr, size)) { + *action = ACTION_SUBTREE; + (*offset)++; + } + + if (!IS_ALIGNED(end, size)) { + *action = ACTION_SUBTREE; + (*end_offset)--; + } + + return *end_offset > *offset; +} + +struct xe_pt_zap_ptes_walk { + /** @base: The walk base-class */ + struct xe_pt_walk base; + + /* Input parameters for the walk */ + /** @tile: The tile we're building for */ + struct xe_tile *tile; + + /* Output */ + /** @needs_invalidate: Whether we need to invalidate TLB*/ + bool needs_invalidate; +}; + +static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_zap_ptes_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_pt *xe_child; + pgoff_t end_offset; + + XE_WARN_ON(!level); + + /* + * Below would be unexpected behavior that needs to be root caused + * but better warn and bail than crash the driver. + */ + if (XE_WARN_ON(!*child)) + return 0; + + xe_child = container_of(*child, typeof(*xe_child), base); + + /* + * Note that we're called from an entry callback, and we're dealing + * with the child of that entry rather than the parent, so need to + * adjust level down. + */ + if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, + &end_offset)) { + xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, + offset * sizeof(u64), 0, + (end_offset - offset) * sizeof(u64)); + xe_walk->needs_invalidate = true; + } + + return 0; +} + +static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { + .pt_entry = xe_pt_zap_ptes_entry, +}; + +/** + * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range + * @tile: The tile we're zapping for. + * @vma: GPU VMA detailing address range. + * + * Eviction and Userptr invalidation needs to be able to zap the + * gpu ptes of a given address range in pagefaulting mode. + * In order to be able to do that, that function needs access to the shared + * page-table entrieaso it can either clear the leaf PTEs or + * clear the pointers to lower-level page-tables. The caller is required + * to hold the necessary locks to ensure neither the page-table connectivity + * nor the page-table entries of the range is updated from under us. + * + * Return: Whether ptes were actually updated and a TLB invalidation is + * required. + */ +bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma) +{ + struct xe_pt_zap_ptes_walk xe_walk = { + .base = { + .ops = &xe_pt_zap_ptes_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + }, + .tile = tile, + }; + struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; + u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated); + + if (xe_vma_bo(vma)) + xe_bo_assert_held(xe_vma_bo(vma)); + else if (xe_vma_is_userptr(vma)) + lockdep_assert_held(&xe_vma_vm(vma)->svm.gpusvm.notifier_lock); + + if (!(pt_mask & BIT(tile->id))) + return false; + + (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), + xe_vma_end(vma), &xe_walk.base); + + return xe_walk.needs_invalidate; +} + +/** + * xe_pt_zap_ptes_range() - Zap (zero) gpu ptes of a SVM range + * @tile: The tile we're zapping for. + * @vm: The VM we're zapping for. + * @range: The SVM range we're zapping for. + * + * SVM invalidation needs to be able to zap the gpu ptes of a given address + * range. In order to be able to do that, that function needs access to the + * shared page-table entries so it can either clear the leaf PTEs or + * clear the pointers to lower-level page-tables. The caller is required + * to hold the SVM notifier lock. + * + * Return: Whether ptes were actually updated and a TLB invalidation is + * required. + */ +bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, + struct xe_svm_range *range) +{ + struct xe_pt_zap_ptes_walk xe_walk = { + .base = { + .ops = &xe_pt_zap_ptes_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + }, + .tile = tile, + }; + struct xe_pt *pt = vm->pt_root[tile->id]; + u8 pt_mask = (range->tile_present & ~range->tile_invalidated); + + /* + * Locking rules: + * + * - notifier_lock (write): full protection against page table changes + * and MMU notifier invalidations. + * + * - notifier_lock (read) + vm_lock (write): combined protection against + * invalidations and concurrent page table modifications. (e.g., madvise) + * + */ + lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || + (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && + lockdep_is_held_type(&vm->lock, 0))); + + if (!(pt_mask & BIT(tile->id))) + return false; + + (void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range), + xe_svm_range_end(range), &xe_walk.base); + + return xe_walk.needs_invalidate; +} + +static void +xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile, + struct iosys_map *map, void *data, + u32 qword_ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update) +{ + struct xe_pt_entry *ptes = update->pt_entries; + u64 *ptr = data; + u32 i; + + /* + * @qword_ofs is the absolute entry offset within the page table, while + * @ptes is indexed relative to @update->ofs (its first entry). The GPU + * path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized + * chunks, calling this with an advancing @qword_ofs but a fresh @data + * pointer per chunk, so translate back into a @ptes index rather than + * assuming the chunk starts at ptes[0]. + */ + for (i = 0; i < num_qwords; i++) { + u32 idx = qword_ofs - update->ofs + i; + + if (map) + xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * + sizeof(u64), u64, ptes[idx].pte); + else + ptr[i] = ptes[idx].pte; + } +} + +static void xe_pt_cancel_bind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries) +{ + u32 i, j; + + for (i = 0; i < num_entries; i++) { + struct xe_pt *pt = entries[i].pt; + + if (!pt) + continue; + + if (pt->level) { + for (j = 0; j < entries[i].qwords; j++) + xe_pt_destroy(entries[i].pt_entries[j].pt, + xe_vma_vm(vma)->flags, NULL); + } + + kfree(entries[i].pt_entries); + entries[i].pt_entries = NULL; + entries[i].qwords = 0; + } +} + +#define XE_INVALID_VMA ((struct xe_vma *)(0xdeaddeadull)) + +static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma) +{ + struct xe_vm *vm; + + if (vma == XE_INVALID_VMA) + return; + + vm = xe_vma_vm(vma); + lockdep_assert_held(&vm->lock); + + if (!xe_vma_has_no_bo(vma)) + dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); + + xe_vm_assert_held(vm); +} + +static void xe_pt_commit_locks_assert(struct xe_vma *vma) +{ + struct xe_vm *vm; + + if (vma == XE_INVALID_VMA) + return; + + vm = xe_vma_vm(vma); + xe_pt_commit_prepare_locks_assert(vma); + + if (xe_vma_is_userptr(vma)) + xe_svm_assert_held_read_or_inject_write(vm); +} + +static void xe_pt_commit(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries, struct llist_head *deferred) +{ + u32 i, j; + + xe_pt_commit_locks_assert(vma); + + for (i = 0; i < num_entries; i++) { + struct xe_pt *pt = entries[i].pt; + struct xe_pt_dir *pt_dir; + + if (!pt->level) + continue; + + pt_dir = as_xe_pt_dir(pt); + for (j = 0; j < entries[i].qwords; j++) { + struct xe_pt *oldpte = entries[i].pt_entries[j].pt; + int j_ = j + entries[i].ofs; + + pt_dir->children[j_] = pt_dir->staging[j_]; + xe_pt_destroy(oldpte, (vma == XE_INVALID_VMA) ? 0 : + xe_vma_vm(vma)->flags, deferred); + } + } +} + +static void xe_pt_abort_bind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries, bool rebind) +{ + int i, j; + + xe_pt_commit_prepare_locks_assert(vma); + + for (i = num_entries - 1; i >= 0; --i) { + struct xe_pt *pt = entries[i].pt; + struct xe_pt_dir *pt_dir; + + if (!rebind) + pt->num_live -= entries[i].qwords; + + if (!pt->level) + continue; + + pt_dir = as_xe_pt_dir(pt); + for (j = 0; j < entries[i].qwords; j++) { + u32 j_ = j + entries[i].ofs; + struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_); + struct xe_pt *oldpte = entries[i].pt_entries[j].pt; + + pt_dir->staging[j_] = oldpte ? &oldpte->base : 0; + xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL); + } + } +} + +static void xe_pt_commit_prepare_bind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries, bool rebind) +{ + u32 i, j; + + xe_pt_commit_prepare_locks_assert(vma); + + for (i = 0; i < num_entries; i++) { + struct xe_pt *pt = entries[i].pt; + struct xe_pt_dir *pt_dir; + + if (!rebind) + pt->num_live += entries[i].qwords; + + if (!pt->level) + continue; + + pt_dir = as_xe_pt_dir(pt); + for (j = 0; j < entries[i].qwords; j++) { + u32 j_ = j + entries[i].ofs; + struct xe_pt *newpte = entries[i].pt_entries[j].pt; + struct xe_pt *oldpte = NULL; + + if (xe_pt_entry_staging(pt_dir, j_)) + oldpte = xe_pt_entry_staging(pt_dir, j_); + + pt_dir->staging[j_] = &newpte->base; + entries[i].pt_entries[j].pt = oldpte; + } + } +} + +static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries, + u32 num_entries) +{ + u32 i; + + for (i = 0; i < num_entries; i++) + kfree(entries[i].pt_entries); +} + +static int +xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma, + struct xe_svm_range *range, + struct xe_vm_pgtable_update *entries, + u32 *num_entries, bool invalidate_on_bind) +{ + int err; + + *num_entries = 0; + err = xe_pt_stage_bind(tile, vma, range, entries, num_entries, + invalidate_on_bind); + if (!err) + xe_tile_assert(tile, *num_entries); + + return err; +} + +static void xe_vm_dbg_print_entries(struct xe_device *xe, + const struct xe_vm_pgtable_update *entries, + unsigned int num_entries, bool bind) +#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) +{ + unsigned int i; + + vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind", + num_entries); + for (i = 0; i < num_entries; i++) { + const struct xe_vm_pgtable_update *entry = &entries[i]; + struct xe_pt *xe_pt = entry->pt; + u64 page_size = 1ull << xe_pt_shift(xe_pt->level); + u64 end; + u64 start; + + xe_assert(xe, !entry->pt->is_compact); + start = entry->ofs * page_size; + end = start + page_size * entry->qwords; + vm_dbg(&xe->drm, + "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n", + i, xe_pt->level, entry->ofs, entry->qwords, + xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0); + } +} +#else +{} +#endif + +static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs) +{ + int i; + + for (i = 0; i < num_syncs; i++) { + struct dma_fence *fence = syncs[i].fence; + + if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &fence->flags)) + return false; + } + + return true; +} + +static int job_test_add_deps(struct xe_sched_job *job, + struct dma_resv *resv, + enum dma_resv_usage usage) +{ + if (!job) { + if (!dma_resv_test_signaled(resv, usage)) + return -ETIME; + + return 0; + } + + return xe_sched_job_add_deps(job, resv, usage); +} + +static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job) +{ + struct xe_bo *bo = xe_vma_bo(vma); + + xe_bo_assert_held(bo); + + if (bo && !bo->vm) + return job_test_add_deps(job, bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL); + + return 0; +} + +static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, + struct xe_sched_job *job) +{ + int err = 0; + + /* + * No need to check for is_cpu_addr_mirror here as vma_add_deps is a + * NOP if VMA is_cpu_addr_mirror + */ + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if (!op->map.immediate && xe_vm_in_fault_mode(vm)) + break; + + err = vma_add_deps(op->map.vma, job); + break; + case DRM_GPUVA_OP_REMAP: + if (op->remap.prev) + err = vma_add_deps(op->remap.prev, job); + if (!err && op->remap.next) + err = vma_add_deps(op->remap.next, job); + break; + case DRM_GPUVA_OP_UNMAP: + break; + case DRM_GPUVA_OP_PREFETCH: + err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job); + break; + case DRM_GPUVA_OP_DRIVER: + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE"); + } + + return err; +} + +static int xe_pt_vm_dependencies(struct xe_sched_job *job, + struct xe_tlb_inval_job *ijob, + struct xe_tlb_inval_job *mjob, + struct xe_vm *vm, + struct xe_vma_ops *vops, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_range_fence_tree *rftree) +{ + struct xe_range_fence *rtfence; + struct dma_fence *fence; + struct xe_vma_op *op; + int err = 0, i; + + xe_vm_assert_held(vm); + + if (!job && !no_in_syncs(vops->syncs, vops->num_syncs)) + return -ETIME; + + if (!job && !xe_exec_queue_is_idle(pt_update_ops->q)) + return -ETIME; + + if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) { + err = job_test_add_deps(job, xe_vm_resv(vm), + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_BOOKKEEP : + DMA_RESV_USAGE_KERNEL); + if (err) + return err; + } + + rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start, + pt_update_ops->last); + while (rtfence) { + fence = rtfence->fence; + + if (!dma_fence_is_signaled(fence)) { + /* + * Is this a CPU update? GPU is busy updating, so return + * an error + */ + if (!job) + return -ETIME; + + dma_fence_get(fence); + err = drm_sched_job_add_dependency(&job->drm, fence); + if (err) + return err; + } + + rtfence = xe_range_fence_tree_next(rtfence, + pt_update_ops->start, + pt_update_ops->last); + } + + list_for_each_entry(op, &vops->list, link) { + err = op_add_deps(vm, op, job); + if (err) + return err; + } + + for (i = 0; job && !err && i < vops->num_syncs; i++) + err = xe_sync_entry_add_deps(&vops->syncs[i], job); + + if (job) { + if (ijob) { + err = xe_tlb_inval_job_alloc_dep(ijob); + if (err) + return err; + } + + if (mjob) { + err = xe_tlb_inval_job_alloc_dep(mjob); + if (err) + return err; + } + } + + return err; +} + +static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) +{ + struct xe_vma_ops *vops = pt_update->vops; + struct xe_vm *vm = vops->vm; + struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id]; + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[pt_update->tile_id]; + + return xe_pt_vm_dependencies(pt_update->job, pt_update->ijob, + pt_update->mjob, vm, pt_update->vops, + pt_update_ops, rftree); +} + +#if IS_ENABLED(CONFIG_DRM_GPUSVM) +/* + * Acquire/release the svm notifier_lock around xe_pt_svm_userptr_pre_commit() + * and the matching late release in xe_pt_update_ops_run(). Read mode by + * default; write mode when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, + * because a userptr op in this critical section may invoke the injected + * xe_vma_userptr_force_invalidate() path that calls + * drm_gpusvm_unmap_pages() with ctx->in_notifier=true, which requires the + * lock held for write. + */ +static void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + down_write(&vm->svm.gpusvm.notifier_lock); +#else + xe_svm_notifier_lock(vm); +#endif +} + +static void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + up_write(&vm->svm.gpusvm.notifier_lock); +#else + xe_svm_notifier_unlock(vm); +#endif +} +#else +static inline void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) { } +static inline void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) { } +#endif + +#if IS_ENABLED(CONFIG_DRM_GPUSVM) +#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT + +static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) +{ + u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; + static u32 count; + + if (count++ % divisor == divisor - 1) { + uvma->userptr.divisor = divisor << 1; + return true; + } + + return false; +} + +#else + +static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) +{ + return false; +} + +#endif + +static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, + struct xe_vm_pgtable_update_ops *pt_update) +{ + struct xe_userptr_vma *uvma; + unsigned long notifier_seq; + + xe_svm_assert_held_read_or_inject_write(vm); + + if (!xe_vma_is_userptr(vma)) + return 0; + + uvma = to_userptr_vma(vma); + if (xe_pt_userptr_inject_eagain(uvma)) + xe_vma_userptr_force_invalidate(uvma); + + notifier_seq = uvma->userptr.pages.notifier_seq; + + if (!mmu_interval_read_retry(&uvma->userptr.notifier, + notifier_seq)) + return 0; + + if (xe_vm_in_fault_mode(vm)) + return -EAGAIN; + + /* + * Just continue the operation since exec or rebind worker + * will take care of rebinding. + */ + return 0; +} + +static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, + struct xe_vm_pgtable_update_ops *pt_update) +{ + int err = 0; + + xe_svm_assert_held_read_or_inject_write(vm); + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if (!op->map.immediate && xe_vm_in_fault_mode(vm)) + break; + + err = vma_check_userptr(vm, op->map.vma, pt_update); + break; + case DRM_GPUVA_OP_REMAP: + if (op->remap.prev && !op->remap.skip_prev) + err = vma_check_userptr(vm, op->remap.prev, pt_update); + if (!err && op->remap.next && !op->remap.skip_next) + err = vma_check_userptr(vm, op->remap.next, pt_update); + break; + case DRM_GPUVA_OP_UNMAP: + break; + case DRM_GPUVA_OP_PREFETCH: + if (xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va))) { + struct xe_svm_range *range = op->map_range.range; + unsigned long i; + + xe_assert(vm->xe, + xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va))); + xa_for_each(&op->prefetch_range.range, i, range) { + xe_svm_range_debug(range, "PRE-COMMIT"); + + if (!xe_svm_range_pages_valid(range)) { + xe_svm_range_debug(range, "PRE-COMMIT - RETRY"); + return -ENODATA; + } + } + } else { + err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), pt_update); + } + break; +#if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) + case DRM_GPUVA_OP_DRIVER: + if (op->subop == XE_VMA_SUBOP_MAP_RANGE) { + struct xe_svm_range *range = op->map_range.range; + + xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma)); + + xe_svm_range_debug(range, "PRE-COMMIT"); + + if (!xe_svm_range_pages_valid(range)) { + xe_svm_range_debug(range, "PRE-COMMIT - RETRY"); + return -EAGAIN; + } + } + break; +#endif + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE"); + } + + return err; +} + +static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) +{ + struct xe_vm *vm = pt_update->vops->vm; + struct xe_vma_ops *vops = pt_update->vops; + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[pt_update->tile_id]; + struct xe_vma_op *op; + int err; + + err = xe_pt_pre_commit(pt_update); + if (err) + return err; + + xe_pt_svm_userptr_notifier_lock(vm); + + list_for_each_entry(op, &vops->list, link) { + err = op_check_svm_userptr(vm, op, pt_update_ops); + if (err) { + xe_pt_svm_userptr_notifier_unlock(vm); + break; + } + } + + return err; +} +#endif + +struct xe_pt_stage_unbind_walk { + /** @base: The pagewalk base-class. */ + struct xe_pt_walk base; + + /* Input parameters for the walk */ + /** @tile: The tile we're unbinding from. */ + struct xe_tile *tile; + + /** + * @modified_start: Walk range start, modified to include any + * shared pagetables that we're the only user of and can thus + * treat as private. + */ + u64 modified_start; + /** @modified_end: Walk range start, modified like @modified_start. */ + u64 modified_end; + + /** @prl: Backing pointer to page reclaim list in pt_update_ops */ + struct xe_page_reclaim_list *prl; + + /* Output */ + /* @wupd: Structure to track the page-table updates we're building */ + struct xe_walk_update wupd; +}; + +/* + * Check whether this range is the only one populating this pagetable, + * and in that case, update the walk range checks so that higher levels don't + * view us as a shared pagetable. + */ +static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, + const struct xe_pt *child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_stage_unbind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + unsigned int shift = walk->shifts[level]; + u64 size = 1ull << shift; + + if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) && + ((next - addr) >> shift) == child->num_live) { + u64 size = 1ull << walk->shifts[level + 1]; + + *action = ACTION_CONTINUE; + + if (xe_walk->modified_start >= addr) + xe_walk->modified_start = round_down(addr, size); + if (xe_walk->modified_end <= next) + xe_walk->modified_end = round_up(next, size); + + return true; + } + + return false; +} + +static int generate_reclaim_entry(struct xe_tile *tile, + struct xe_page_reclaim_list *prl, + u64 pte, struct xe_pt *xe_child) +{ + struct xe_gt *gt = tile->primary_gt; + struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries; + bool is_2m = xe_child->level == 1 && (pte & XE_PDE_PS_2M); + bool is_64k = xe_child->level == 0 && ((pte & XE_PTE_PS64) || xe_child->is_compact); + u32 page_shift = is_2m ? ilog2(SZ_2M) : is_64k ? ilog2(SZ_64K) : ilog2(SZ_4K); + /* Physical address bits start at page shift: 2M->[51:21], 64K->[51:16], 4K->[51:12] */ + u64 phys_addr = pte & XE_PAGE_ADDR_MASK(page_shift); + /* Page address is relative to 4K page regardless of entry level */ + u64 phys_page = phys_addr >> XE_PTE_SHIFT; + int num_entries = prl->num_entries; + u32 reclamation_size = page_shift - XE_PTE_SHIFT; + + xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL); + xe_tile_assert(tile, reclaim_entries); + xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1); + + if (!xe_page_reclaim_list_valid(prl)) + return -EINVAL; + + /** + * reclamation_size indicates the size of the page to be + * invalidated and flushed from non-coherent cache. + * Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes. + * Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim + */ + if (is_2m) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT, 1); + } else if (is_64k) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1); + } else if (xe_child->level == 0) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1); + } else { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "unsupported PTE level=%u pte=%#llx", + xe_child->level, pte); + return -EINVAL; + } + + reclaim_entries[num_entries].qw = + FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) | + FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) | + FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) | + FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20); + prl->num_entries++; + vm_dbg(&tile_to_xe(tile)->drm, + "PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n", + xe_child->level, pte, reclamation_size, num_entries); + + return 0; +} + +static int add_pte_to_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl, + struct xe_pt *xe_child, u64 pte, u64 addr) +{ + /* + * In rare scenarios, pte may not be written yet due to racy conditions. + * In such cases, invalidate the PRL and fallback to full PPC invalidation. + */ + if (!pte) { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "found zero pte at addr=%#llx", addr); + return -EINVAL; + } + + /* Ensure it is a defined page */ + xe_tile_assert(tile, xe_child->level == 0 || + (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G))); + + /* Account for NULL terminated entry on end (-1) */ + if (prl->num_entries >= XE_PAGE_RECLAIM_MAX_ENTRIES - 1) { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "overflow while adding pte=%#llx", pte); + return -ENOSPC; + } + + return generate_reclaim_entry(tile, prl, pte, xe_child); +} + +static bool add_compact_pt_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl, + struct xe_device *xe, struct xe_pt *compact_pt, u64 addr) +{ + struct iosys_map *map = &compact_pt->bo->vmap; + + for (pgoff_t i = 0; i < SZ_2M / SZ_64K && xe_page_reclaim_list_valid(prl); i++) { + u64 pte = xe_map_rd(xe, map, i * sizeof(u64), u64); + + if (add_pte_to_prl(tile, prl, compact_pt, pte, addr + i * SZ_64K)) + break; + } + + return xe_page_reclaim_list_valid(prl); +} + +static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + struct xe_pt_stage_unbind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_page_reclaim_list *prl = xe_walk->prl; + struct xe_tile *tile = xe_walk->tile; + struct xe_device *xe = tile_to_xe(tile); + pgoff_t first = xe_pt_offset(addr, xe_child->level, walk); + bool killed; + + XE_WARN_ON(!*child); + XE_WARN_ON(!level); + /* Check for leaf node */ + if (prl && xe_page_reclaim_list_valid(prl) && + xe_child->level <= MAX_HUGEPTE_LEVEL) { + struct iosys_map *leaf_map = &xe_child->bo->vmap; + pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk); + + for (pgoff_t i = 0; i < count; i++) { + u64 pte; + + /* + * If not a leaf pt, skip unless non-leaf pt is interleaved between + * leaf ptes which causes the page walk to skip over the child leaves + */ + if (xe_child->base.children && xe_child->base.children[first + i]) { + u64 pt_size = 1ULL << walk->shifts[xe_child->level]; + bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) || + (i == count - 1 && !IS_ALIGNED(next, pt_size)); + struct xe_pt *child_pt = + container_of(xe_child->base.children[first + i], + struct xe_pt, base); + + /* Compact PTs always fill a full 2M-aligned slot, never an edge. */ + XE_WARN_ON(child_pt->is_compact && edge_pt); + if (edge_pt) + continue; + + /* Walker never descends into compact PTs, descend now */ + if (child_pt->is_compact) { + if (!add_compact_pt_prl(tile, prl, xe, child_pt, + addr + (u64)i * pt_size)) + break; + } else { + xe_page_reclaim_list_abort(tile->primary_gt, + prl, + "PT is skipped by walk at level=%u offset=%lu", + xe_child->level, first + i); + break; + } + continue; + } + + pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64); + + if (add_pte_to_prl(tile, prl, xe_child, pte, addr)) + break; + + /* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */ + if (pte & XE_PTE_PS64) + i += 15; /* Skip other 15 consecutive 4K pages in the 64K page */ + } + } + + killed = xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); + + /* + * Verify if any PTE are potentially dropped at non-leaf levels, either from being + * killed or the page walk covers the region. + */ + if (prl && xe_page_reclaim_list_valid(prl) && + xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) { + bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base); + + /* + * If aborting page walk early (kill) or page walk completes the full range + * we need to invalidate the PRL. + */ + if (killed || covered) + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "kill at level=%u addr=%#llx next=%#llx num_live=%u", + level, addr, next, xe_child->num_live); + } + + return 0; +} + +static int +xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_stage_unbind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + pgoff_t end_offset; + u64 size = 1ull << walk->shifts[--level]; + int err; + + if (!IS_ALIGNED(addr, size)) + addr = xe_walk->modified_start; + if (!IS_ALIGNED(next, size)) + next = xe_walk->modified_end; + + /* Parent == *child is the root pt. Don't kill it. */ + if (parent != *child && + xe_pt_check_kill(addr, next, level, xe_child, action, walk)) + return 0; + + if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset, + &end_offset)) + return 0; + + err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true); + if (err) + return err; + + xe_walk->wupd.updates[level].update->qwords = end_offset - offset; + + return 0; +} + +static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { + .pt_entry = xe_pt_stage_unbind_entry, + .pt_post_descend = xe_pt_stage_unbind_post_descend, +}; + +/** + * xe_pt_stage_unbind() - Build page-table update structures for an unbind + * operation + * @tile: The tile we're unbinding for. + * @vm: The vm + * @vma: The vma we're unbinding. + * @range: The range we're unbinding. + * @entries: Caller-provided storage for the update structures. + * + * Builds page-table update structures for an unbind operation. The function + * will attempt to remove all page-tables that we're the only user + * of, and for that to work, the unbind operation must be committed in the + * same critical section that blocks racing binds to the same page-table tree. + * + * Return: The number of entries used. + */ +static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, + struct xe_vm *vm, + struct xe_vma *vma, + struct xe_svm_range *range, + struct xe_vm_pgtable_update *entries) +{ + u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma); + u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma); + struct xe_vm_pgtable_update_op *pt_update_op = + container_of(entries, struct xe_vm_pgtable_update_op, entries[0]); + struct xe_pt_stage_unbind_walk xe_walk = { + .base = { + .ops = &xe_pt_stage_unbind_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + .staging = true, + }, + .tile = tile, + .modified_start = start, + .modified_end = end, + .wupd.entries = entries, + .prl = pt_update_op->prl, + }; + struct xe_pt *pt = vm->pt_root[tile->id]; + + (void)xe_pt_walk_shared(&pt->base, pt->level, start, end, + &xe_walk.base); + + return xe_walk.wupd.num_used_entries; +} + +static void +xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, + struct xe_tile *tile, struct iosys_map *map, + void *ptr, u32 qword_ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update) +{ + struct xe_vm *vm = pt_update->vops->vm; + u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level); + int i; + + if (map && map->is_iomem) + for (i = 0; i < num_qwords; ++i) + xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * + sizeof(u64), u64, empty); + else if (map) + memset64(map->vaddr + qword_ofs * sizeof(u64), empty, + num_qwords); + else + memset64(ptr, empty, num_qwords); +} + +static void xe_pt_abort_unbind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries) +{ + int i, j; + + xe_pt_commit_prepare_locks_assert(vma); + + for (i = num_entries - 1; i >= 0; --i) { + struct xe_vm_pgtable_update *entry = &entries[i]; + struct xe_pt *pt = entry->pt; + struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); + + pt->num_live += entry->qwords; + + if (!pt->level) + continue; + + for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) + pt_dir->staging[j] = + entries[i].pt_entries[j - entry->ofs].pt ? + &entries[i].pt_entries[j - entry->ofs].pt->base : NULL; + } +} + +static void +xe_pt_commit_prepare_unbind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries) +{ + int i, j; + + xe_pt_commit_prepare_locks_assert(vma); + + for (i = 0; i < num_entries; ++i) { + struct xe_vm_pgtable_update *entry = &entries[i]; + struct xe_pt *pt = entry->pt; + struct xe_pt_dir *pt_dir; + + pt->num_live -= entry->qwords; + if (!pt->level) + continue; + + pt_dir = as_xe_pt_dir(pt); + for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) { + entry->pt_entries[j - entry->ofs].pt = + xe_pt_entry_staging(pt_dir, j); + pt_dir->staging[j] = NULL; + } + } +} + +static void +xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops, + u64 start, u64 end) +{ + u64 last; + u32 current_op = pt_update_ops->current_op; + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + int i, level = 0; + + for (i = 0; i < pt_op->num_entries; i++) { + const struct xe_vm_pgtable_update *entry = &pt_op->entries[i]; + + if (entry->pt->level > level) + level = entry->pt->level; + } + + /* Greedy (non-optimal) calculation but simple */ + start = ALIGN_DOWN(start, 0x1ull << xe_pt_shift(level)); + last = ALIGN(end, 0x1ull << xe_pt_shift(level)) - 1; + + if (start < pt_update_ops->start) + pt_update_ops->start = start; + if (last > pt_update_ops->last) + pt_update_ops->last = last; +} + +static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma) +{ + int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0; + + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) + return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, + xe->info.tile_count << shift); + + return 0; +} + +static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma *vma, bool invalidate_on_bind) +{ + u32 current_op = pt_update_ops->current_op; + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + int err; + + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + xe_bo_assert_held(xe_vma_bo(vma)); + + vm_dbg(&xe_vma_vm(vma)->xe->drm, + "Preparing bind, with range [%llx...%llx)\n", + xe_vma_start(vma), xe_vma_end(vma) - 1); + + pt_op->vma = NULL; + pt_op->bind = true; + pt_op->rebind = BIT(tile->id) & vma->tile_present; + + err = vma_reserve_fences(tile_to_xe(tile), vma); + if (err) + return err; + + err = xe_pt_prepare_bind(tile, vma, NULL, pt_op->entries, + &pt_op->num_entries, invalidate_on_bind); + if (!err) { + xe_tile_assert(tile, pt_op->num_entries <= + ARRAY_SIZE(pt_op->entries)); + xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, + pt_op->num_entries, true); + + xe_pt_update_ops_rfence_interval(pt_update_ops, + xe_vma_start(vma), + xe_vma_end(vma)); + ++pt_update_ops->current_op; + pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma); + + /* + * If rebind, we have to invalidate TLB on !LR vms to invalidate + * cached PTEs point to freed memory. On LR vms this is done + * automatically when the context is re-enabled by the rebind worker, + * or in fault mode it was invalidated on PTE zapping. + * + * If rebind, we have to invalidate TLB on context based TLB invalidation + * LR vms, as they cannot be relied on context re-enable. + * + * If !rebind, and scratch enabled VMs, there is a chance the scratch + * PTE is already cached in the TLB so it needs to be invalidated. + * On !LR VMs this is done in the ring ops preceding a batch, but on + * LR, in particular on user-space batch buffer chaining, it needs to + * be done here. + */ + if ((!pt_op->rebind && xe_vm_has_scratch(vm) && + xe_vm_in_lr_mode(vm))) + pt_update_ops->needs_invalidation = true; + else if (pt_op->rebind && xe_vm_in_preempt_fence_mode(vm) && + vm->xe->info.has_ctx_tlb_inval) + pt_update_ops->needs_invalidation = true; + else if (pt_op->rebind && !xe_vm_in_lr_mode(vm)) + /* We bump also if batch_invalidate_tlb is true */ + vm->tlb_flush_seqno++; + + vma->tile_staged |= BIT(tile->id); + pt_op->vma = vma; + xe_pt_commit_prepare_bind(vma, pt_op->entries, + pt_op->num_entries, pt_op->rebind); + } else { + xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); + } + + return err; +} + +static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma *vma, struct xe_svm_range *range) +{ + u32 current_op = pt_update_ops->current_op; + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + int err; + + xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma)); + + vm_dbg(&xe_vma_vm(vma)->xe->drm, + "Preparing bind, with range [%lx...%lx)\n", + xe_svm_range_start(range), xe_svm_range_end(range) - 1); + + pt_op->vma = NULL; + pt_op->bind = true; + pt_op->rebind = BIT(tile->id) & range->tile_present; + + err = xe_pt_prepare_bind(tile, vma, range, pt_op->entries, + &pt_op->num_entries, false); + if (!err) { + xe_tile_assert(tile, pt_op->num_entries <= + ARRAY_SIZE(pt_op->entries)); + xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, + pt_op->num_entries, true); + + xe_pt_update_ops_rfence_interval(pt_update_ops, + xe_svm_range_start(range), + xe_svm_range_end(range)); + ++pt_update_ops->current_op; + pt_update_ops->needs_svm_lock = true; + + pt_op->vma = vma; + xe_pt_commit_prepare_bind(vma, pt_op->entries, + pt_op->num_entries, pt_op->rebind); + } else { + xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); + } + + return err; +} + +static int unbind_op_prepare(struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma *vma) +{ + struct xe_device *xe = tile_to_xe(tile); + u32 current_op = pt_update_ops->current_op; + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + int err; + + if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) + return 0; + + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + xe_bo_assert_held(xe_vma_bo(vma)); + + vm_dbg(&xe_vma_vm(vma)->xe->drm, + "Preparing unbind, with range [%llx...%llx)\n", + xe_vma_start(vma), xe_vma_end(vma) - 1); + + pt_op->vma = vma; + pt_op->bind = false; + pt_op->rebind = false; + /* + * Maintain one PRL located in pt_update_ops that all others in unbind op reference. + * Ensure that PRL is allocated only once, and if invalidated, remains an invalidated PRL. + */ + if (xe->info.has_page_reclaim_hw_assist && + xe_page_reclaim_list_is_new(&pt_update_ops->prl)) + xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl); + + /* Page reclaim may not be needed due to other features, so skip the corresponding VMA */ + pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) && + !xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL; + + err = vma_reserve_fences(tile_to_xe(tile), vma); + if (err) + return err; + + pt_op->num_entries = xe_pt_stage_unbind(tile, xe_vma_vm(vma), + vma, NULL, pt_op->entries); + + xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, + pt_op->num_entries, false); + xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma), + xe_vma_end(vma)); + ++pt_update_ops->current_op; + pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma); + pt_update_ops->needs_invalidation = true; + + xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries); + + return 0; +} + +static bool +xe_pt_op_check_range_skip_invalidation(struct xe_vm_pgtable_update_op *pt_op, + struct xe_svm_range *range) +{ + struct xe_vm_pgtable_update *update = pt_op->entries; + + XE_WARN_ON(!pt_op->num_entries); + + /* + * We can't skip the invalidation if we are removing PTEs that span more + * than the range, do some checks to ensure we are removing PTEs that + * are invalid. + */ + + if (pt_op->num_entries > 1) + return false; + + if (update->pt->level == 0) + return true; + + if (update->pt->level == 1) + return xe_svm_range_size(range) >= SZ_2M; + + return false; +} + +static int unbind_range_prepare(struct xe_vm *vm, + struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_svm_range *range) +{ + u32 current_op = pt_update_ops->current_op; + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + + if (!(range->tile_present & BIT(tile->id))) + return 0; + + vm_dbg(&vm->xe->drm, + "Preparing unbind, with range [%lx...%lx)\n", + xe_svm_range_start(range), xe_svm_range_end(range) - 1); + + pt_op->vma = XE_INVALID_VMA; + pt_op->bind = false; + pt_op->rebind = false; + pt_op->prl = NULL; + + pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range, + pt_op->entries); + + xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, + pt_op->num_entries, false); + xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range), + xe_svm_range_end(range)); + ++pt_update_ops->current_op; + pt_update_ops->needs_svm_lock = true; + pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) || + xe_vm_has_valid_gpu_mapping(tile, range->tile_present, + range->tile_invalidated) || + !xe_pt_op_check_range_skip_invalidation(pt_op, range); + + xe_pt_commit_prepare_unbind(XE_INVALID_VMA, pt_op->entries, + pt_op->num_entries); + + return 0; +} + +static int op_prepare(struct xe_vm *vm, + struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma_op *op) +{ + int err = 0; + + xe_vm_assert_held(vm); + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if ((!op->map.immediate && xe_vm_in_fault_mode(vm) && + !op->map.invalidate_on_bind) || + (op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) + break; + + err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma, + op->map.invalidate_on_bind); + pt_update_ops->wait_vm_kernel = true; + break; + case DRM_GPUVA_OP_REMAP: + { + struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); + + if (xe_vma_is_cpu_addr_mirror(old)) + break; + + err = unbind_op_prepare(tile, pt_update_ops, old); + + if (!err && op->remap.prev && !op->remap.skip_prev) { + err = bind_op_prepare(vm, tile, pt_update_ops, + op->remap.prev, false); + pt_update_ops->wait_vm_bookkeep = true; + } + if (!err && op->remap.next && !op->remap.skip_next) { + err = bind_op_prepare(vm, tile, pt_update_ops, + op->remap.next, false); + pt_update_ops->wait_vm_bookkeep = true; + } + break; + } + case DRM_GPUVA_OP_UNMAP: + { + struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); + + if (xe_vma_is_cpu_addr_mirror(vma)) + break; + + err = unbind_op_prepare(tile, pt_update_ops, vma); + break; + } + case DRM_GPUVA_OP_PREFETCH: + { + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + + if (xe_vma_is_cpu_addr_mirror(vma)) { + struct xe_svm_range *range; + unsigned long i; + + xa_for_each(&op->prefetch_range.range, i, range) { + err = bind_range_prepare(vm, tile, pt_update_ops, + vma, range); + if (err) + return err; + } + } else { + err = bind_op_prepare(vm, tile, pt_update_ops, vma, false); + pt_update_ops->wait_vm_kernel = true; + } + break; + } + case DRM_GPUVA_OP_DRIVER: + if (op->subop == XE_VMA_SUBOP_MAP_RANGE) { + xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma)); + + err = bind_range_prepare(vm, tile, pt_update_ops, + op->map_range.vma, + op->map_range.range); + } else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) { + err = unbind_range_prepare(vm, tile, pt_update_ops, + op->unmap_range.range); + } + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE"); + } + + return err; +} + +static void +xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) +{ + init_llist_head(&pt_update_ops->deferred); + pt_update_ops->current_op = 0; + pt_update_ops->start = ~0x0ull; + pt_update_ops->last = 0x0ull; + pt_update_ops->needs_svm_lock = false; + pt_update_ops->needs_invalidation = false; + xe_page_reclaim_list_init(&pt_update_ops->prl); +} + +/** + * xe_pt_update_ops_prepare() - Prepare PT update operations + * @tile: Tile of PT update operations + * @vops: VMA operationa + * + * Prepare PT update operations which includes updating internal PT state, + * allocate memory for page tables, populate page table being pruned in, and + * create PT update operations for leaf insertion / removal. + * + * Return: 0 on success, negative error code on error. + */ +int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) +{ + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[tile->id]; + struct xe_vma_op *op; + int shift = tile->media_gt ? 1 : 0; + int err; + + lockdep_assert_held(&vops->vm->lock); + xe_vm_assert_held(vops->vm); + + xe_pt_update_ops_init(pt_update_ops); + + err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), + tile_to_xe(tile)->info.tile_count << shift); + if (err) + return err; + + list_for_each_entry(op, &vops->list, link) { + err = op_prepare(vops->vm, tile, pt_update_ops, op); + + if (err) + return err; + } + + xe_tile_assert(tile, pt_update_ops->current_op <= + pt_update_ops->num_ops); + +#ifdef TEST_VM_OPS_ERROR + if (vops->inject_error && + vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) + return -ENOSPC; +#endif + + return 0; +} +ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO); + +static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma *vma, struct dma_fence *fence, + struct dma_fence *fence2, bool invalidate_on_bind) +{ + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + if (fence2) + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + } + /* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ + WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id)); + if (invalidate_on_bind) + WRITE_ONCE(vma->tile_invalidated, + vma->tile_invalidated | BIT(tile->id)); + else + WRITE_ONCE(vma->tile_invalidated, + vma->tile_invalidated & ~BIT(tile->id)); + vma->tile_staged &= ~BIT(tile->id); + if (xe_vma_is_userptr(vma)) { + xe_svm_assert_held_read_or_inject_write(vm); + to_userptr_vma(vma)->userptr.initial_bind = true; + } + + /* + * Kick rebind worker if this bind triggers preempt fences and not in + * the rebind worker + */ + if (pt_update_ops->wait_vm_bookkeep && + xe_vm_in_preempt_fence_mode(vm) && + !current->mm) + xe_vm_queue_rebind_worker(vm); +} + +static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma *vma, struct dma_fence *fence, + struct dma_fence *fence2) +{ + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + if (fence2) + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + } + vma->tile_present &= ~BIT(tile->id); + if (!vma->tile_present) { + list_del_init(&vma->combined_links.rebind); + if (xe_vma_is_userptr(vma)) { + xe_svm_assert_held_read_or_inject_write(vm); + + spin_lock(&vm->userptr.invalidated_lock); + list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); + spin_unlock(&vm->userptr.invalidated_lock); + } + } +} + +static void range_present_and_invalidated_tile(struct xe_vm *vm, + struct xe_svm_range *range, + u8 tile_id) +{ + /* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ + + lockdep_assert_held(&vm->svm.gpusvm.notifier_lock); + + WRITE_ONCE(range->tile_present, range->tile_present | BIT(tile_id)); + WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~BIT(tile_id)); +} + +static void op_commit(struct xe_vm *vm, + struct xe_tile *tile, + struct xe_vm_pgtable_update_ops *pt_update_ops, + struct xe_vma_op *op, struct dma_fence *fence, + struct dma_fence *fence2) +{ + xe_vm_assert_held(vm); + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) || + (op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) + break; + + bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, + fence2, op->map.invalidate_on_bind); + break; + case DRM_GPUVA_OP_REMAP: + { + struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); + + if (xe_vma_is_cpu_addr_mirror(old)) + break; + + unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); + + if (op->remap.prev && !op->remap.skip_prev) + bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, + fence, fence2, false); + if (op->remap.next && !op->remap.skip_next) + bind_op_commit(vm, tile, pt_update_ops, op->remap.next, + fence, fence2, false); + break; + } + case DRM_GPUVA_OP_UNMAP: + { + struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); + + if (!xe_vma_is_cpu_addr_mirror(vma)) + unbind_op_commit(vm, tile, pt_update_ops, vma, fence, + fence2); + break; + } + case DRM_GPUVA_OP_PREFETCH: + { + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + + if (xe_vma_is_cpu_addr_mirror(vma)) { + struct xe_svm_range *range = NULL; + unsigned long i; + + xa_for_each(&op->prefetch_range.range, i, range) + range_present_and_invalidated_tile(vm, range, tile->id); + } else { + bind_op_commit(vm, tile, pt_update_ops, vma, fence, + fence2, false); + } + break; + } + case DRM_GPUVA_OP_DRIVER: + { + /* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ + if (op->subop == XE_VMA_SUBOP_MAP_RANGE) + range_present_and_invalidated_tile(vm, op->map_range.range, tile->id); + else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) + WRITE_ONCE(op->unmap_range.range->tile_present, + op->unmap_range.range->tile_present & + ~BIT(tile->id)); + + break; + } + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE"); + } +} + +static const struct xe_migrate_pt_update_ops migrate_ops = { + .populate = xe_vm_populate_pgtable, + .clear = xe_migrate_clear_pgtable_callback, + .pre_commit = xe_pt_pre_commit, +}; + +#if IS_ENABLED(CONFIG_DRM_GPUSVM) +static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops = { + .populate = xe_vm_populate_pgtable, + .clear = xe_migrate_clear_pgtable_callback, + .pre_commit = xe_pt_svm_userptr_pre_commit, +}; +#else +static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops; +#endif + +static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q, + struct xe_gt *gt) +{ + if (xe_gt_is_media_type(gt)) + return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler; + + return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler; +} + +/** + * xe_pt_update_ops_run() - Run PT update operations + * @tile: Tile of PT update operations + * @vops: VMA operationa + * + * Run PT update operations which includes committing internal PT state changes, + * creating job for PT update operations for leaf insertion / removal, and + * installing job fence in various places. + * + * Return: fence on success, negative ERR_PTR on error. + */ +struct dma_fence * +xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) +{ + struct xe_vm *vm = vops->vm; + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[tile->id]; + struct xe_exec_queue *q = pt_update_ops->q; + struct dma_fence *fence, *ifence = NULL, *mfence = NULL; + struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL; + struct xe_range_fence *rfence; + struct xe_vma_op *op; + int err = 0, i; + struct xe_migrate_pt_update update = { + .ops = pt_update_ops->needs_svm_lock ? + &svm_userptr_migrate_ops : + &migrate_ops, + .vops = vops, + .tile_id = tile->id, + }; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + if (!pt_update_ops->current_op) { + xe_tile_assert(tile, xe_vm_in_fault_mode(vm)); + + return dma_fence_get_stub(); + } + +#ifdef TEST_VM_OPS_ERROR + if (vops->inject_error && + vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN) + return ERR_PTR(-ENOSPC); +#endif + + if (pt_update_ops->needs_invalidation) { + struct xe_dep_scheduler *dep_scheduler = + to_dep_scheduler(q, tile->primary_gt); + + ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval, + dep_scheduler, vm, + pt_update_ops->start, + pt_update_ops->last, + XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + if (IS_ERR(ijob)) { + err = PTR_ERR(ijob); + goto kill_vm_tile1; + } + update.ijob = ijob; + /* + * Only add page reclaim for the primary GT. Media GT does not have + * any PPC to flush, so enabling the PPC flush bit for media is + * effectively a NOP and provides no performance benefit nor + * interfere with primary GT. + */ + if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) { + xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl); + /* Release ref from alloc, job will now handle it */ + xe_page_reclaim_list_invalidate(&pt_update_ops->prl); + } + + if (tile->media_gt) { + dep_scheduler = to_dep_scheduler(q, tile->media_gt); + + mjob = xe_tlb_inval_job_create(q, + &tile->media_gt->tlb_inval, + dep_scheduler, vm, + pt_update_ops->start, + pt_update_ops->last, + XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT); + if (IS_ERR(mjob)) { + err = PTR_ERR(mjob); + goto free_ijob; + } + update.mjob = mjob; + } + } + + rfence = kzalloc_obj(*rfence); + if (!rfence) { + err = -ENOMEM; + goto free_ijob; + } + + fence = xe_migrate_update_pgtables(tile->migrate, &update); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto free_rfence; + } + + /* Point of no return - VM killed if failure after this */ + for (i = 0; i < pt_update_ops->current_op; ++i) { + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; + + xe_pt_commit(pt_op->vma, pt_op->entries, + pt_op->num_entries, &pt_update_ops->deferred); + pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ + } + + if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, + &xe_range_fence_kfree_ops, + pt_update_ops->start, + pt_update_ops->last, fence)) + dma_fence_wait(fence, false); + + if (ijob) + ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence); + if (mjob) + mfence = xe_tlb_inval_job_push(mjob, tile->migrate, fence); + + if (!mjob && !ijob) { + dma_resv_add_fence(xe_vm_resv(vm), fence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + list_for_each_entry(op, &vops->list, link) + op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); + } else if (ijob && !mjob) { + dma_resv_add_fence(xe_vm_resv(vm), ifence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + list_for_each_entry(op, &vops->list, link) + op_commit(vops->vm, tile, pt_update_ops, op, ifence, NULL); + } else { + dma_resv_add_fence(xe_vm_resv(vm), ifence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + dma_resv_add_fence(xe_vm_resv(vm), mfence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + list_for_each_entry(op, &vops->list, link) + op_commit(vops->vm, tile, pt_update_ops, op, ifence, + mfence); + } + + if (pt_update_ops->needs_svm_lock) + xe_pt_svm_userptr_notifier_unlock(vm); + + /* + * The last fence is only used for zero bind queue idling; migrate + * queues are not exposed to user space. + */ + if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE)) + xe_exec_queue_last_fence_set(q, vm, fence); + + xe_tlb_inval_job_put(mjob); + xe_tlb_inval_job_put(ijob); + dma_fence_put(ifence); + dma_fence_put(mfence); + + return fence; + +free_rfence: + kfree(rfence); +free_ijob: + xe_tlb_inval_job_put(mjob); + xe_tlb_inval_job_put(ijob); +kill_vm_tile1: + if (err != -EAGAIN && err != -ENODATA && tile->id) + xe_vm_kill(vops->vm, false); + + return ERR_PTR(err); +} +ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO); + +/** + * xe_pt_update_ops_fini() - Finish PT update operations + * @tile: Tile of PT update operations + * @vops: VMA operations + * + * Finish PT update operations by committing to destroy page table memory + */ +void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops) +{ + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[tile->id]; + int i; + + xe_page_reclaim_entries_put(pt_update_ops->prl.entries); + + lockdep_assert_held(&vops->vm->lock); + xe_vm_assert_held(vops->vm); + + for (i = 0; i < pt_update_ops->current_op; ++i) { + struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; + + xe_pt_free_bind(pt_op->entries, pt_op->num_entries); + } + xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); +} + +/** + * xe_pt_update_ops_abort() - Abort PT update operations + * @tile: Tile of PT update operations + * @vops: VMA operationa + * + * Abort PT update operations by unwinding internal PT state + */ +void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops) +{ + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[tile->id]; + int i; + + lockdep_assert_held(&vops->vm->lock); + xe_vm_assert_held(vops->vm); + + for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { + struct xe_vm_pgtable_update_op *pt_op = + &pt_update_ops->ops[i]; + + if (!pt_op->vma || i >= pt_update_ops->current_op) + continue; + + if (pt_op->bind) + xe_pt_abort_bind(pt_op->vma, pt_op->entries, + pt_op->num_entries, + pt_op->rebind); + else + xe_pt_abort_unbind(pt_op->vma, pt_op->entries, + pt_op->num_entries); + } + + xe_pt_update_ops_fini(tile, vops); +} diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h new file mode 100644 index 000000000..4daeebaab --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_PT_H_ +#define _XE_PT_H_ + +#include + +#include "xe_pt_types.h" + +struct dma_fence; +struct drm_exec; +struct xe_bo; +struct xe_device; +struct xe_exec_queue; +struct xe_svm_range; +struct xe_sync_entry; +struct xe_tile; +struct xe_vm; +struct xe_vma; +struct xe_vma_ops; + +/* Largest huge pte is currently 1GiB. May become device dependent. */ +#define MAX_HUGEPTE_LEVEL 2 + +#define xe_pt_write(xe, map, idx, data) \ + xe_map_wr(xe, map, (idx) * sizeof(u64), u64, data) + +unsigned int xe_pt_shift(unsigned int level); + +struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, + unsigned int level, struct drm_exec *exec); + +void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, + struct xe_pt *pt); + +void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred); + +void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt); + +int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops); +struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile, + struct xe_vma_ops *vops); +void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops); +void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops); + +bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma); +bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, + struct xe_svm_range *range); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h new file mode 100644 index 000000000..a7d1bb708 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PT_TYPES_H_ +#define _XE_PT_TYPES_H_ + +#include + +#include "xe_page_reclaim.h" +#include "xe_pt_walk.h" + +struct xe_bo; +struct xe_device; +struct xe_vma; + +enum xe_cache_level { + XE_CACHE_NONE, + XE_CACHE_WT, + XE_CACHE_WB, + XE_CACHE_NONE_COMPRESSION, /*UC + COH_NONE + COMPRESSION */ + XE_CACHE_WB_COMPRESSION, + __XE_CACHE_LEVEL_COUNT, +}; + +#define XE_VM_MAX_LEVEL 4 + +struct xe_pt { + struct xe_ptw base; + struct xe_bo *bo; + unsigned int level; + unsigned int num_live; + bool rebind; + bool is_compact; +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) + /** @addr: Virtual address start address of the PT. */ + u64 addr; +#endif +}; + +struct xe_pt_ops { + u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, + u16 pat_index, u32 pt_level); + u64 (*pte_encode_vma)(u64 pte, struct xe_vma *vma, + u16 pat_index, u32 pt_level); + u64 (*pte_encode_addr)(struct xe_device *xe, u64 addr, + u16 pat_index, + u32 pt_level, bool devmem, u64 flags); + u64 (*pde_encode_bo)(struct xe_bo *bo, u64 bo_offset); +}; + +struct xe_pt_entry { + struct xe_pt *pt; + u64 pte; +}; + +struct xe_vm_pgtable_update { + /** @bo: page table bo to write to */ + struct xe_bo *pt_bo; + + /** @ofs: offset inside this PTE to begin writing to (in qwords) */ + u32 ofs; + + /** @qwords: number of PTE's to write */ + u32 qwords; + + /** @pt: opaque pointer useful for the caller of xe_migrate_update_pgtables */ + struct xe_pt *pt; + + /** @pt_entries: Newly added pagetable entries */ + struct xe_pt_entry *pt_entries; + + /** @flags: Target flags */ + u32 flags; +}; + +/** struct xe_vm_pgtable_update_op - Page table update operation */ +struct xe_vm_pgtable_update_op { + /** @entries: entries to update for this operation */ + struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; + /** @vma: VMA for operation, operation not valid if NULL */ + struct xe_vma *vma; + /** @prl: Backing pointer to page reclaim list of pt_update_ops */ + struct xe_page_reclaim_list *prl; + /** @num_entries: number of entries for this update operation */ + u32 num_entries; + /** @bind: is a bind */ + bool bind; + /** @rebind: is a rebind */ + bool rebind; +}; + +/** struct xe_vm_pgtable_update_ops: page table update operations */ +struct xe_vm_pgtable_update_ops { + /** @ops: operations */ + struct xe_vm_pgtable_update_op *ops; + /** @deferred: deferred list to destroy PT entries */ + struct llist_head deferred; + /** @q: exec queue for PT operations */ + struct xe_exec_queue *q; + /** @prl: embedded page reclaim list */ + struct xe_page_reclaim_list prl; + /** @start: start address of ops */ + u64 start; + /** @last: last address of ops */ + u64 last; + /** @num_ops: number of operations */ + u32 num_ops; + /** @current_op: current operations */ + u32 current_op; + /** @needs_svm_lock: Needs SVM lock */ + bool needs_svm_lock; + /** @needs_invalidation: Needs invalidation */ + bool needs_invalidation; + /** + * @wait_vm_bookkeep: PT operations need to wait until VM is idle + * (bookkeep dma-resv slots are idle) and stage all future VM activity + * behind these operations (install PT operations into VM kernel + * dma-resv slot). + */ + bool wait_vm_bookkeep; + /** + * @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv + * slots are idle. + */ + bool wait_vm_kernel; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pt_walk.c b/drivers/gpu/drm/xe/xe_pt_walk.c new file mode 100644 index 000000000..be602a763 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt_walk.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2022 Intel Corporation + */ +#include "xe_pt_walk.h" + +/** + * DOC: GPU page-table tree walking. + * The utilities in this file are similar to the CPU page-table walk + * utilities in mm/pagewalk.c. The main difference is that we distinguish + * the various levels of a page-table tree with an unsigned integer rather + * than by name. 0 is the lowest level, and page-tables with level 0 can + * not be directories pointing to lower levels, whereas all other levels + * can. The user of the utilities determines the highest level. + * + * Nomenclature: + * Each struct xe_ptw, regardless of level is referred to as a page table, and + * multiple page tables typically form a page table tree with page tables at + * intermediate levels being page directories pointing at page tables at lower + * levels. A shared page table for a given address range is a page-table which + * is neither fully within nor fully outside the address range and that can + * thus be shared by two or more address ranges. + * + * Please keep this code generic so that it can used as a drm-wide page- + * table walker should other drivers find use for it. + */ +static u64 xe_pt_addr_end(u64 addr, u64 end, unsigned int level, + const struct xe_pt_walk *walk) +{ + u64 size = 1ull << walk->shifts[level]; + u64 tmp = round_up(addr + 1, size); + + return min_t(u64, tmp, end); +} + +static bool xe_pt_next(pgoff_t *offset, u64 *addr, u64 next, u64 end, + unsigned int level, const struct xe_pt_walk *walk) +{ + pgoff_t step = 1; + + /* Shared pt walk skips to the last pagetable */ + if (unlikely(walk->shared_pt_mode)) { + unsigned int shift = walk->shifts[level]; + u64 skip_to = round_down(end, 1ull << shift); + + if (skip_to > next) { + step += (skip_to - next) >> shift; + next = skip_to; + } + } + + *addr = next; + *offset += step; + + return next != end; +} + +/** + * xe_pt_walk_range() - Walk a range of a gpu page table tree with callbacks + * for each page-table entry in all levels. + * @parent: The root page table for walk start. + * @level: The root page table level. + * @addr: Virtual address start. + * @end: Virtual address end + 1. + * @walk: Walk info. + * + * Similar to the CPU page-table walker, this is a helper to walk + * a gpu page table and call a provided callback function for each entry. + * + * Return: 0 on success, negative error code on error. The error is + * propagated from the callback and on error the walk is terminated. + */ +int xe_pt_walk_range(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk) +{ + pgoff_t offset = xe_pt_offset(addr, level, walk); + struct xe_ptw **entries = walk->staging ? (parent->staging ?: NULL) : + (parent->children ?: NULL); + const struct xe_pt_walk_ops *ops = walk->ops; + enum page_walk_action action; + struct xe_ptw *child; + int err = 0; + u64 next; + + do { + next = xe_pt_addr_end(addr, end, level, walk); + if (walk->shared_pt_mode && xe_pt_covers(addr, next, level, + walk)) + continue; +again: + action = ACTION_SUBTREE; + child = entries ? entries[offset] : NULL; + err = ops->pt_entry(parent, offset, level, addr, next, + &child, &action, walk); + if (err) + break; + + /* Probably not needed yet for gpu pagetable walk. */ + if (unlikely(action == ACTION_AGAIN)) + goto again; + + if (likely(!level || !child || action == ACTION_CONTINUE)) + continue; + + err = xe_pt_walk_range(child, level - 1, addr, next, walk); + + if (!err && ops->pt_post_descend) + err = ops->pt_post_descend(parent, offset, level, addr, + next, &child, &action, walk); + if (err) + break; + + } while (xe_pt_next(&offset, &addr, next, end, level, walk)); + + return err; +} + +/** + * xe_pt_walk_shared() - Walk shared page tables of a page-table tree. + * @parent: Root page table directory. + * @level: Level of the root. + * @addr: Start address. + * @end: Last address + 1. + * @walk: Walk info. + * + * This function is similar to xe_pt_walk_range() but it skips page tables + * that are private to the range. Since the root (or @parent) page table is + * typically also a shared page table this function is different in that it + * calls the pt_entry callback and the post_descend callback also for the + * root. The root can be detected in the callbacks by checking whether + * parent == *child. + * Walking only the shared page tables is common for unbind-type operations + * where the page-table entries for an address range are cleared or detached + * from the main page-table tree. + * + * Return: 0 on success, negative error code on error: If a callback + * returns an error, the walk will be terminated and the error returned by + * this function. + */ +int xe_pt_walk_shared(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk) +{ + const struct xe_pt_walk_ops *ops = walk->ops; + enum page_walk_action action = ACTION_SUBTREE; + struct xe_ptw *child = parent; + int err; + + walk->shared_pt_mode = true; + err = walk->ops->pt_entry(parent, 0, level + 1, addr, end, + &child, &action, walk); + + if (err || action != ACTION_SUBTREE) + return err; + + err = xe_pt_walk_range(parent, level, addr, end, walk); + if (!err && ops->pt_post_descend) { + err = ops->pt_post_descend(parent, 0, level + 1, addr, end, + &child, &action, walk); + } + return err; +} diff --git a/drivers/gpu/drm/xe/xe_pt_walk.h b/drivers/gpu/drm/xe/xe_pt_walk.h new file mode 100644 index 000000000..e91995fa7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt_walk.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_PT_WALK_H_ +#define _XE_PT_WALK_H_ + +#include +#include + +/** + * struct xe_ptw - base class for driver pagetable subclassing. + * @children: Pointer to an array of children if any. + * @staging: Pointer to an array of staging if any. + * + * Drivers could subclass this, and if it's a page-directory, typically + * embed an array of xe_ptw pointers. + */ +struct xe_ptw { + struct xe_ptw **children; + struct xe_ptw **staging; +}; + +/** + * struct xe_pt_walk - Embeddable struct for walk parameters + */ +struct xe_pt_walk { + /** @ops: The walk ops used for the pagewalk */ + const struct xe_pt_walk_ops *ops; + /** + * @shifts: Array of page-table entry shifts used for the + * different levels, starting out with the leaf level 0 + * page-shift as the first entry. It's legal for this pointer to be + * changed during the walk. + */ + const u64 *shifts; + /** @max_level: Highest populated level in @shifts */ + unsigned int max_level; + /** + * @shared_pt_mode: Whether to skip all entries that are private + * to the address range and called only for entries that are + * shared with other address ranges. Such entries are referred to + * as shared pagetables. + */ + bool shared_pt_mode; + /** @staging: Walk staging PT structure */ + bool staging; +}; + +/** + * typedef xe_pt_entry_fn - gpu page-table-walk callback-function + * @parent: The parent page table. + * @offset: The offset (number of entries) into the page table. + * @level: The level of @parent. + * @addr: The virtual address. + * @next: The virtual address for the next call, or end address. + * @child: Pointer to pointer to child page-table at this @offset. The + * function may modify the value pointed to if, for example, allocating a + * child page table. + * @action: The walk action to take upon return. See . + * @walk: The walk parameters. + */ +typedef int (*xe_pt_entry_fn)(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk); + +/** + * struct xe_pt_walk_ops - Walk callbacks. + */ +struct xe_pt_walk_ops { + /** + * @pt_entry: Callback to be called for each page table entry prior + * to descending to the next level. The returned value of the action + * function parameter is honored. + */ + xe_pt_entry_fn pt_entry; + /** + * @pt_post_descend: Callback to be called for each page table entry + * after return from descending to the next level. The returned value + * of the action function parameter is ignored. + */ + xe_pt_entry_fn pt_post_descend; +}; + +int xe_pt_walk_range(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk); + +int xe_pt_walk_shared(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk); + +/** + * xe_pt_covers - Whether the address range covers an entire entry in @level + * @addr: Start of the range. + * @end: End of range + 1. + * @level: Page table level. + * @walk: Page table walk info. + * + * This function is a helper to aid in determining whether a leaf page table + * entry can be inserted at this @level. + * + * Return: Whether the range provided covers exactly an entry at this level. + */ +static inline bool xe_pt_covers(u64 addr, u64 end, unsigned int level, + const struct xe_pt_walk *walk) +{ + u64 pt_size = 1ull << walk->shifts[level]; + + return end - addr == pt_size && IS_ALIGNED(addr, pt_size); +} + +/** + * xe_pt_num_entries - Number of page-table entries of a given range at this + * level + * @addr: Start address. + * @end: End address. + * @level: Page table level. + * @walk: Walk info. + * + * Return: The number of page table entries at this level between @addr and + * @end. + */ +static inline pgoff_t +xe_pt_num_entries(u64 addr, u64 end, unsigned int level, + const struct xe_pt_walk *walk) +{ + u64 pt_size = 1ull << walk->shifts[level]; + + return (round_up(end, pt_size) - round_down(addr, pt_size)) >> + walk->shifts[level]; +} + +/** + * xe_pt_offset - Offset of the page-table entry for a given address. + * @addr: The address. + * @level: Page table level. + * @walk: Walk info. + * + * Return: The page table entry offset for the given address in a + * page table with size indicated by @level. + */ +static inline pgoff_t +xe_pt_offset(u64 addr, unsigned int level, const struct xe_pt_walk *walk) +{ + if (level < walk->max_level) + addr &= ((1ull << walk->shifts[level + 1]) - 1); + + return addr >> walk->shifts[level]; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c new file mode 100644 index 000000000..2fef274a5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -0,0 +1,1037 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2024 Intel Corporation. + */ + +#include "xe_pxp.h" + +#include +#include + +#include + +#include "xe_bo.h" +#include "xe_bo_types.h" +#include "xe_device_types.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_guc_submit.h" +#include "xe_gsc_proxy.h" +#include "xe_gt_types.h" +#include "xe_huc.h" +#include "xe_hw_engine.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_pxp_submit.h" +#include "xe_pxp_types.h" +#include "xe_uc_fw.h" +#include "regs/xe_irq_regs.h" +#include "regs/xe_pxp_regs.h" + +/** + * DOC: PXP + * + * PXP (Protected Xe Path) allows execution and flip to display of protected + * (i.e. encrypted) objects. This feature is currently only supported in + * integrated parts. + */ + +#define ARB_SESSION DRM_XE_PXP_HWDRM_DEFAULT_SESSION /* shorter define */ + +/* + * A submission to GSC can take up to 250ms to complete, so use a 300ms + * timeout for activation where only one of those is involved. Termination + * additionally requires a submission to VCS and an interaction with KCR, so + * bump the timeout to 500ms for that. + */ +#define PXP_ACTIVATION_TIMEOUT_MS 300 +#define PXP_TERMINATION_TIMEOUT_MS 500 + +bool xe_pxp_is_supported(const struct xe_device *xe) +{ + return xe->info.has_pxp && IS_ENABLED(CONFIG_INTEL_MEI_GSC_PROXY); +} + +bool xe_pxp_is_enabled(const struct xe_pxp *pxp) +{ + return pxp; +} + +static bool pxp_prerequisites_done(const struct xe_pxp *pxp) +{ + struct xe_gt *gt = pxp->gt; + bool huc_ok; + bool ready; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + + /* + * If force_wake fails we could falsely report the prerequisites as not + * done even if they are; the consequence of this would be that the + * callers won't go ahead with using PXP, but if force_wake doesn't work + * the GT is very likely in a bad state so not really a problem to abort + * PXP. Therefore, we can just log the force_wake error and not escalate + * it. + */ + XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)); + + /* + * PXP requires GSC proxy to be initialized. On platforms where the HuC + * is loaded by the kernel driver (i.e., pre media 35) PXP also requires + * the HuC to be authenticated by GSC. + */ + huc_ok = MEDIA_VER(gt_to_xe(gt)) >= 35 || + xe_huc_is_authenticated(>->uc.huc, XE_HUC_AUTH_VIA_GSC); + ready = huc_ok && xe_gsc_proxy_init_done(>->uc.gsc); + + return ready; +} + +/** + * xe_pxp_get_readiness_status - check whether PXP is ready for userspace use + * @pxp: the xe_pxp pointer (can be NULL if PXP is disabled) + * + * Returns: 0 if PXP is not ready yet, 1 if it is ready, a negative errno value + * if PXP is not supported/enabled or if something went wrong in the + * initialization of the prerequisites. Note that the return values of this + * function follow the uapi (see drm_xe_query_pxp_status), so they can be used + * directly in the query ioctl. + */ +int xe_pxp_get_readiness_status(struct xe_pxp *pxp) +{ + int ret = 0; + + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + /* If the GSC FW is in an error state, PXP will never work */ + if (xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status)) + return -EIO; + + /* Same for HuC FW, but only if the kernel owns HuC-loading (i.e. pre-NVL) */ + if (MEDIA_VER(gt_to_xe(pxp->gt)) < 35 && + xe_uc_fw_status_to_error(pxp->gt->uc.huc.fw.status)) + return -EIO; + + guard(xe_pm_runtime)(pxp->xe); + + /* PXP requires both HuC loaded and GSC proxy initialized */ + if (pxp_prerequisites_done(pxp)) + ret = 1; + + return ret; +} + +static bool pxp_session_is_in_play(struct xe_pxp *pxp, u32 id) +{ + struct xe_gt *gt = pxp->gt; + + return xe_mmio_read32(>->mmio, KCR_SIP) & BIT(id); +} + +static int pxp_wait_for_session_state(struct xe_pxp *pxp, u32 id, bool in_play) +{ + struct xe_gt *gt = pxp->gt; + u32 mask = BIT(id); + + return xe_mmio_wait32(>->mmio, KCR_SIP, mask, in_play ? mask : 0, + 250, NULL, false); +} + +static void pxp_invalidate_queues(struct xe_pxp *pxp); + +static int pxp_terminate_hw(struct xe_pxp *pxp) +{ + struct xe_gt *gt = pxp->gt; + int ret = 0; + + drm_dbg(&pxp->xe->drm, "Terminating PXP\n"); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -EIO; + + /* terminate the hw session */ + ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION); + if (ret) + return ret; + + ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false); + if (ret) + return ret; + + /* Trigger full HW cleanup */ + xe_mmio_write32(>->mmio, KCR_GLOBAL_TERMINATE, 1); + + /* now we can tell the GSC to clean up its own state */ + return xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION); +} + +static void mark_termination_in_progress(struct xe_pxp *pxp) +{ + lockdep_assert_held(&pxp->mutex); + + reinit_completion(&pxp->termination); + pxp->status = XE_PXP_TERMINATION_IN_PROGRESS; +} + +static bool pxp_prep_for_termination(struct xe_pxp *pxp) +{ + lockdep_assert_held(&pxp->mutex); + + if (pxp->status == XE_PXP_ACTIVE) + pxp->key_instance++; + + /* + * we'll mark the status as needing termination on resume, so no need to + * emit a termination now. + */ + if (pxp->status == XE_PXP_SUSPENDED) + return false; + + /* + * If we have a termination already in progress, we need to wait for + * it to complete before queueing another one. Once the first + * termination is completed we'll set the state back to + * NEEDS_TERMINATION and leave it to the pxp start code to issue it. + */ + if (pxp->status == XE_PXP_TERMINATION_IN_PROGRESS) { + pxp->status = XE_PXP_NEEDS_ADDITIONAL_TERMINATION; + return false; + } + + mark_termination_in_progress(pxp); + + return true; +} + +static void pxp_terminate(struct xe_pxp *pxp, bool hw_only) +{ + struct xe_device *xe = pxp->xe; + int ret = 0; + + if (!wait_for_completion_timeout(&pxp->activation, + msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) + drm_err(&xe->drm, "failed to wait for PXP start before termination\n"); + + if (!hw_only) { + bool prep_ok; + + mutex_lock(&pxp->mutex); + + prep_ok = pxp_prep_for_termination(pxp); + + mutex_unlock(&pxp->mutex); + + if (!prep_ok) + return; + + pxp_invalidate_queues(pxp); + } else { + /* + * The caller of the HW-only termination should have already + * called pxp_prep_for_termination and marked the termination as + * in progress. + */ + xe_assert(xe, !completion_done(&pxp->termination)); + } + + ret = pxp_terminate_hw(pxp); + if (ret) { + drm_err(&xe->drm, "PXP termination failed: %pe\n", ERR_PTR(ret)); + mutex_lock(&pxp->mutex); + pxp->status = XE_PXP_ERROR; + complete_all(&pxp->termination); + mutex_unlock(&pxp->mutex); + } +} + +static void pxp_terminate_complete(struct xe_pxp *pxp) +{ + /* + * We expect PXP to be in one of 3 states when we get here: + * - XE_PXP_TERMINATION_IN_PROGRESS: a single termination event was + * requested and it is now completing, so we're ready to start. + * - XE_PXP_NEEDS_ADDITIONAL_TERMINATION: a second termination was + * requested while the first one was still being processed. + * - XE_PXP_SUSPENDED: PXP is now suspended, so we defer everything to + * when we come back on resume. + */ + mutex_lock(&pxp->mutex); + + switch (pxp->status) { + case XE_PXP_TERMINATION_IN_PROGRESS: + pxp->status = XE_PXP_READY_TO_START; + break; + case XE_PXP_NEEDS_ADDITIONAL_TERMINATION: + pxp->status = XE_PXP_NEEDS_TERMINATION; + break; + case XE_PXP_SUSPENDED: + /* Nothing to do */ + break; + default: + drm_err(&pxp->xe->drm, + "PXP termination complete while status was %u\n", + pxp->status); + } + + complete_all(&pxp->termination); + + mutex_unlock(&pxp->mutex); +} + +static void pxp_events_work(struct work_struct *work) +{ + struct xe_pxp *pxp = container_of(work, typeof(*pxp), events.work); + struct xe_device *xe = pxp->xe; + bool hw_only = false; + u32 events = 0; + + events = atomic_xchg(&pxp->events.pending, 0); + + if (!events) + return; + + /* + * If the termination request comes from an irq while we're suspending, + * then we can defer it to the resume path instead of waking the device + * up. + * In the case of the termination on resume the pm reference is taken + * in xe_pxp_pm_resume() and released here. + * Note that we do not expect both events to be set at the same time, + * but if it does happen due to a spurious interrupt we want to behave + * as if the only request we got was the one from the resume path; this + * is because the termination prep has already been done in + * xe_pxp_pm_resume() and it is impossible for any PXP operations to + * occur between the prep and the termination completion, so there is no + * need for a new SW prep. + */ + if (events & PXP_TERMINATION_REQUEST_ON_RESUME) { + events &= ~PXP_TERMINATION_REQUEST_IRQ; + hw_only = true; + } + + if ((events & PXP_TERMINATION_REQUEST_IRQ) && !xe_pm_runtime_get_if_active(xe)) + return; + + if (events & PXP_TERMINATION_REQUEST) { + events &= ~PXP_TERMINATION_COMPLETE_IRQ; + pxp_terminate(pxp, hw_only); + } + + if (events & PXP_TERMINATION_COMPLETE_IRQ) + pxp_terminate_complete(pxp); + + if (events & PXP_TERMINATION_REQUEST) + xe_pm_runtime_put(xe); +} + +/** + * xe_pxp_irq_handler - Handles PXP interrupts. + * @xe: the xe_device structure + * @iir: interrupt vector + */ +void xe_pxp_irq_handler(struct xe_device *xe, u16 iir) +{ + struct xe_pxp *pxp = xe->pxp; + + if (!xe_pxp_is_enabled(pxp)) { + drm_err(&xe->drm, "PXP irq 0x%x received with PXP disabled!\n", iir); + return; + } + + if (unlikely(!iir)) + return; + + if (iir & (KCR_PXP_STATE_TERMINATED_INTERRUPT | + KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT)) + atomic_or(PXP_TERMINATION_REQUEST_IRQ, &pxp->events.pending); + + if (iir & KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT) + atomic_or(PXP_TERMINATION_COMPLETE_IRQ, &pxp->events.pending); + + if (atomic_read(&pxp->events.pending)) + queue_work(pxp->events.wq, &pxp->events.work); +} + +static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable) +{ + u32 val = enable ? REG_MASKED_FIELD_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) : + REG_MASKED_FIELD_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -EIO; + + xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val); + + return 0; +} + +static int kcr_pxp_enable(const struct xe_pxp *pxp) +{ + return kcr_pxp_set_status(pxp, true); +} + +static int kcr_pxp_disable(const struct xe_pxp *pxp) +{ + return kcr_pxp_set_status(pxp, false); +} + +static void pxp_fini(void *arg) +{ + struct xe_pxp *pxp = arg; + + destroy_workqueue(pxp->events.wq); + xe_pxp_destroy_execution_resources(pxp); + + /* no need to explicitly disable KCR since we're going to do an FLR */ +} + +/** + * xe_pxp_init - initialize PXP support + * @xe: the xe_device structure + * + * Initialize the HW state and allocate the objects required for PXP support. + * Note that some of the requirement for PXP support (GSC proxy init, HuC auth) + * are performed asynchronously as part of the GSC init. PXP can only be used + * after both this function and the async worker have completed. + * + * Returns 0 if PXP is not supported or if PXP initialization is successful, + * other errno value if there is an error during the init. + */ +int xe_pxp_init(struct xe_device *xe) +{ + struct xe_gt *gt = xe->tiles[0].media_gt; + bool gsc_ok, huc_ok; + struct xe_pxp *pxp; + int err; + + if (!xe_pxp_is_supported(xe)) + return 0; + + /* we only support PXP on single tile devices with a media GT */ + if (xe->info.tile_count > 1 || !gt) + return 0; + + /* The GSCCS is required for submissions to the GSC FW */ + if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0))) + return 0; + + /* PXP requires GSC FW to be available. Pre-NVL it also requires HuC FW */ + gsc_ok = xe_uc_fw_is_loadable(>->uc.gsc.fw); + huc_ok = MEDIA_VER(xe) >= 35 || xe_uc_fw_is_loadable(>->uc.huc.fw); + + if (!gsc_ok || !huc_ok) { + drm_info(&xe->drm, "Skipping PXP due to unsatisfied FW deps - GSC=%s, HuC=%s\n", + str_yes_no(gsc_ok), + MEDIA_VER(xe) >= 35 ? "not needed" : str_yes_no(huc_ok)); + return 0; + } + + /* + * On PTL, older GSC FWs have a bug that can cause them to crash during + * PXP invalidation events, which leads to a complete loss of power + * management on the media GT. Therefore, we can't use PXP on FWs that + * have this bug, which was fixed in PTL GSC build 1396. + */ + if (xe->info.platform == XE_PANTHERLAKE && + gt->uc.gsc.fw.versions.found[XE_UC_FW_VER_RELEASE].build < 1396) { + drm_info(&xe->drm, "PXP requires PTL GSC build 1396 or newer\n"); + return 0; + } + + pxp = drmm_kzalloc(&xe->drm, sizeof(struct xe_pxp), GFP_KERNEL); + if (!pxp) { + err = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&pxp->queues.list); + spin_lock_init(&pxp->queues.lock); + INIT_WORK(&pxp->events.work, pxp_events_work); + pxp->xe = xe; + pxp->gt = gt; + + pxp->key_instance = 1; + pxp->last_suspend_key_instance = 1; + + /* + * we'll use the completions to check if there is an action pending, + * so we start them as completed and we reinit it when an action is + * triggered. + */ + init_completion(&pxp->activation); + init_completion(&pxp->termination); + complete_all(&pxp->termination); + complete_all(&pxp->activation); + + mutex_init(&pxp->mutex); + + pxp->events.wq = alloc_ordered_workqueue("pxp-wq", 0); + if (!pxp->events.wq) { + err = -ENOMEM; + goto out_free; + } + + err = kcr_pxp_enable(pxp); + if (err) + goto out_wq; + + err = xe_pxp_allocate_execution_resources(pxp); + if (err) + goto out_kcr_disable; + + xe->pxp = pxp; + + return devm_add_action_or_reset(xe->drm.dev, pxp_fini, pxp); + +out_kcr_disable: + kcr_pxp_disable(pxp); +out_wq: + destroy_workqueue(pxp->events.wq); +out_free: + drmm_kfree(&xe->drm, pxp); +out: + drm_err(&xe->drm, "PXP initialization failed: %pe\n", ERR_PTR(err)); + return err; +} + +static int __pxp_start_arb_session(struct xe_pxp *pxp) +{ + int ret; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -EIO; + + if (pxp_session_is_in_play(pxp, ARB_SESSION)) + return -EEXIST; + + ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION); + if (ret) { + drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret)); + return ret; + } + + ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true); + if (ret) { + drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret)); + return ret; + } + + drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n"); + return 0; +} + +/** + * xe_pxp_exec_queue_set_type - Mark a queue as using PXP + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @q: the queue to mark as using PXP + * @type: the type of PXP session this queue will use + * + * Returns 0 if the selected PXP type is supported, -ENODEV otherwise. + */ +int xe_pxp_exec_queue_set_type(struct xe_pxp *pxp, struct xe_exec_queue *q, u8 type) +{ + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + /* we only support HWDRM sessions right now */ + xe_assert(pxp->xe, type == DRM_XE_PXP_TYPE_HWDRM); + + q->pxp.type = type; + + return 0; +} + +static int __exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q) +{ + int ret = 0; + + /* + * A queue can be added to the list only if the PXP is in active status, + * otherwise the termination might not handle it correctly. + */ + mutex_lock(&pxp->mutex); + + if (pxp->status == XE_PXP_ACTIVE) { + spin_lock_irq(&pxp->queues.lock); + list_add_tail(&q->pxp.link, &pxp->queues.list); + spin_unlock_irq(&pxp->queues.lock); + } else if (pxp->status == XE_PXP_ERROR || pxp->status == XE_PXP_SUSPENDED) { + ret = -EIO; + } else { + ret = -EBUSY; /* try again later */ + } + + mutex_unlock(&pxp->mutex); + + return ret; +} + +static int pxp_start(struct xe_pxp *pxp, u8 type) +{ + int ret = 0; + bool restart; + + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + /* we only support HWDRM sessions right now */ + xe_assert(pxp->xe, type == DRM_XE_PXP_TYPE_HWDRM); + + /* get_readiness_status() returns 0 for in-progress and 1 for done */ + ret = xe_pxp_get_readiness_status(pxp); + if (ret <= 0) + return ret ?: -EBUSY; + + ret = 0; + +wait_for_idle: + /* + * if there is an action in progress, wait for it. We need to wait + * outside the lock because the completion is done from within the lock. + * Note that the two actions should never be pending at the same time. + */ + if (!wait_for_completion_timeout(&pxp->termination, + msecs_to_jiffies(PXP_TERMINATION_TIMEOUT_MS))) + return -ETIMEDOUT; + + if (!wait_for_completion_timeout(&pxp->activation, + msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) + return -ETIMEDOUT; + + restart = false; + + mutex_lock(&pxp->mutex); + + /* If PXP is not already active, turn it on */ + switch (pxp->status) { + case XE_PXP_ERROR: + ret = -EIO; + goto out_unlock; + case XE_PXP_ACTIVE: + goto out_unlock; + case XE_PXP_READY_TO_START: + pxp->status = XE_PXP_START_IN_PROGRESS; + reinit_completion(&pxp->activation); + break; + case XE_PXP_START_IN_PROGRESS: + /* If a start is in progress then the completion must not be done */ + XE_WARN_ON(completion_done(&pxp->activation)); + restart = true; + goto out_unlock; + case XE_PXP_NEEDS_TERMINATION: + mark_termination_in_progress(pxp); + break; + case XE_PXP_TERMINATION_IN_PROGRESS: + case XE_PXP_NEEDS_ADDITIONAL_TERMINATION: + /* If a termination is in progress then the completion must not be done */ + XE_WARN_ON(completion_done(&pxp->termination)); + restart = true; + goto out_unlock; + case XE_PXP_SUSPENDED: + default: + drm_err(&pxp->xe->drm, "unexpected state during PXP start: %u\n", pxp->status); + ret = -EIO; + goto out_unlock; + } + + mutex_unlock(&pxp->mutex); + + if (!completion_done(&pxp->termination)) { + ret = pxp_terminate_hw(pxp); + if (ret) { + drm_err(&pxp->xe->drm, "PXP termination failed before start\n"); + mutex_lock(&pxp->mutex); + pxp->status = XE_PXP_ERROR; + complete_all(&pxp->termination); + + goto out_unlock; + } + + goto wait_for_idle; + } + + /* All the cases except for start should have exited earlier */ + XE_WARN_ON(completion_done(&pxp->activation)); + ret = __pxp_start_arb_session(pxp); + + mutex_lock(&pxp->mutex); + + complete_all(&pxp->activation); + + /* + * Any other process should wait until the state goes away from + * XE_PXP_START_IN_PROGRESS, so if the state is not that something went + * wrong. Mark the status as needing termination and try again. + */ + if (pxp->status != XE_PXP_START_IN_PROGRESS) { + drm_err(&pxp->xe->drm, "unexpected state after PXP start: %u\n", pxp->status); + pxp->status = XE_PXP_NEEDS_TERMINATION; + restart = true; + goto out_unlock; + } + + /* If everything went ok, update the status and add the queue to the list */ + if (!ret) + pxp->status = XE_PXP_ACTIVE; + else + pxp->status = XE_PXP_ERROR; + +out_unlock: + mutex_unlock(&pxp->mutex); + + if (restart) + goto wait_for_idle; + + return ret; +} + +/** + * xe_pxp_exec_queue_add - add a queue to the PXP list + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @q: the queue to add to the list + * + * If PXP is enabled and the prerequisites are done, start the PXP default + * session (if not already running) and add the queue to the PXP list. + * + * Returns 0 if the PXP session is running and the queue is in the list, + * -ENODEV if PXP is disabled, -EBUSY if the PXP prerequisites are not done, + * other errno value if something goes wrong during the session start. + */ +int xe_pxp_exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q) +{ + int ret; + + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + /* + * Runtime suspend kills PXP, so we take a reference to prevent it from + * happening while we have active queues that use PXP + */ + xe_pm_runtime_get(pxp->xe); + +start: + ret = pxp_start(pxp, q->pxp.type); + + if (!ret) { + ret = __exec_queue_add(pxp, q); + if (ret == -EBUSY) + goto start; + } + + /* + * in the successful case the PM ref is released from + * xe_pxp_exec_queue_remove + */ + if (ret) + xe_pm_runtime_put(pxp->xe); + + return ret; +} +ALLOW_ERROR_INJECTION(xe_pxp_exec_queue_add, ERRNO); + +static void __pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q, bool lock) +{ + bool need_pm_put = false; + + if (!xe_pxp_is_enabled(pxp)) + return; + + if (lock) + spin_lock_irq(&pxp->queues.lock); + + if (!list_empty(&q->pxp.link)) { + list_del_init(&q->pxp.link); + need_pm_put = true; + } + + q->pxp.type = DRM_XE_PXP_TYPE_NONE; + + if (lock) + spin_unlock_irq(&pxp->queues.lock); + + if (need_pm_put) + xe_pm_runtime_put(pxp->xe); +} + +/** + * xe_pxp_exec_queue_remove - remove a queue from the PXP list + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @q: the queue to remove from the list + * + * If PXP is enabled and the exec_queue is in the list, the queue will be + * removed from the list and its PM reference will be released. It is safe to + * call this function multiple times for the same queue. + */ +void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q) +{ + __pxp_exec_queue_remove(pxp, q, true); +} + +static void pxp_invalidate_queues(struct xe_pxp *pxp) +{ + struct xe_exec_queue *tmp, *q; + LIST_HEAD(to_clean); + + spin_lock_irq(&pxp->queues.lock); + + list_for_each_entry_safe(q, tmp, &pxp->queues.list, pxp.link) { + q = xe_exec_queue_get_unless_zero(q); + if (!q) + continue; + + list_move_tail(&q->pxp.link, &to_clean); + } + spin_unlock_irq(&pxp->queues.lock); + + list_for_each_entry_safe(q, tmp, &to_clean, pxp.link) { + drm_dbg(&pxp->xe->drm, + "Killing queue due to PXP termination: eclass=%s, guc_id=%d\n", + xe_hw_engine_class_to_str(q->class), q->guc->id); + + xe_exec_queue_kill(q); + + /* + * We hold a ref to the queue so there is no risk of racing with + * the calls to exec_queue_remove coming from exec_queue_destroy. + */ + __pxp_exec_queue_remove(pxp, q, false); + + xe_exec_queue_put(q); + } +} + +/** + * xe_pxp_key_assign - mark a BO as using the current PXP key iteration + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @bo: the BO to mark + * + * Returns: -ENODEV if PXP is disabled, 0 otherwise. + */ +int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo) +{ + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + xe_assert(pxp->xe, !bo->pxp_key_instance); + + /* + * Note that the PXP key handling is inherently racey, because the key + * can theoretically change at any time (although it's unlikely to do + * so without triggers), even right after we copy it. Taking a lock + * wouldn't help because the value might still change as soon as we + * release the lock. + * Userspace needs to handle the fact that their BOs can go invalid at + * any point. + */ + bo->pxp_key_instance = pxp->key_instance; + + return 0; +} + +/** + * xe_pxp_bo_key_check - check if the key used by a xe_bo is valid + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @bo: the BO we want to check + * + * Checks whether a BO was encrypted with the current key or an obsolete one. + * + * Returns: 0 if the key is valid, -ENODEV if PXP is disabled, -EINVAL if the + * BO is not using PXP, -ENOEXEC if the key is not valid. + */ +int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo) +{ + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + if (!xe_bo_is_protected(bo)) + return -EINVAL; + + xe_assert(pxp->xe, bo->pxp_key_instance); + + /* + * Note that the PXP key handling is inherently racey, because the key + * can theoretically change at any time (although it's unlikely to do + * so without triggers), even right after we check it. Taking a lock + * wouldn't help because the value might still change as soon as we + * release the lock. + * We mitigate the risk by checking the key at multiple points (on each + * submission involving the BO and right before flipping it on the + * display), but there is still a very small chance that we could + * operate on an invalid BO for a single submission or a single frame + * flip. This is a compromise made to protect the encrypted data (which + * is what the key termination is for). + */ + if (bo->pxp_key_instance != pxp->key_instance) + return -ENOEXEC; + + return 0; +} + +/** + * xe_pxp_obj_key_check - check if the key used by a drm_gem_obj is valid + * @obj: the drm_gem_obj we want to check + * + * Checks whether a drm_gem_obj was encrypted with the current key or an + * obsolete one. + * + * Returns: 0 if the key is valid, -ENODEV if PXP is disabled, -EINVAL if the + * obj is not using PXP, -ENOEXEC if the key is not valid. + */ +int xe_pxp_obj_key_check(struct drm_gem_object *obj) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + struct xe_device *xe = xe_bo_device(bo); + struct xe_pxp *pxp = xe->pxp; + + return xe_pxp_bo_key_check(pxp, bo); +} + +/** + * xe_pxp_pm_suspend - prepare PXP for HW suspend + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * + * Makes sure all PXP actions have completed and invalidates all PXP queues + * and objects before we go into a suspend state. + * + * Returns: 0 if successful, a negative errno value otherwise. + */ +int xe_pxp_pm_suspend(struct xe_pxp *pxp) +{ + bool needs_queue_inval = false; + int ret = 0; + + if (!xe_pxp_is_enabled(pxp)) + return 0; + +wait_for_activation: + if (!wait_for_completion_timeout(&pxp->activation, + msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) + ret = -ETIMEDOUT; + + mutex_lock(&pxp->mutex); + + switch (pxp->status) { + case XE_PXP_ERROR: + case XE_PXP_READY_TO_START: + case XE_PXP_SUSPENDED: + case XE_PXP_TERMINATION_IN_PROGRESS: + case XE_PXP_NEEDS_ADDITIONAL_TERMINATION: + /* + * If PXP is not running there is nothing to cleanup. If there + * is a termination pending then no need to issue another one. + */ + break; + case XE_PXP_START_IN_PROGRESS: + mutex_unlock(&pxp->mutex); + goto wait_for_activation; + case XE_PXP_NEEDS_TERMINATION: + /* If PXP was never used we can skip the cleanup */ + if (pxp->key_instance == pxp->last_suspend_key_instance) + break; + fallthrough; + case XE_PXP_ACTIVE: + pxp->key_instance++; + pxp->needs_termination_on_resume = true; + needs_queue_inval = true; + break; + } + + /* + * We set this even if we were in error state, hoping the suspend clears + * the error. Worse case we fail again and go in error state again. + */ + pxp->status = XE_PXP_SUSPENDED; + + mutex_unlock(&pxp->mutex); + + if (needs_queue_inval) + pxp_invalidate_queues(pxp); + + /* + * if there is a termination in progress, wait for it. + * We need to wait outside the lock because the completion is done from + * within the lock + */ + if (!wait_for_completion_timeout(&pxp->termination, + msecs_to_jiffies(PXP_TERMINATION_TIMEOUT_MS))) + ret = -ETIMEDOUT; + + pxp->last_suspend_key_instance = pxp->key_instance; + + return ret; +} + +/** + * xe_pxp_pm_resume - re-init PXP after HW suspend + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + */ +void xe_pxp_pm_resume(struct xe_pxp *pxp) +{ + bool has_pm = false; + int err; + + if (!xe_pxp_is_enabled(pxp)) + return; + + err = kcr_pxp_enable(pxp); + + /* + * We want to avoid the device runtime suspending before we're done with + * the termination queued below, so we need a runtime PM reference; we + * can't call the rpm functions from within the PXP lock, so we take the + * ref here. Note that we don't want the rpm resume code to actually run + * here as that would call back into this function, but as long as we + * don't enable DPM_FLAG_SMART_SUSPEND (which we currently do not) we're + * guaranteed to not be runtime suspended at this point, so we can + * safely use the get_noresume variant. + */ + if (pxp->needs_termination_on_resume) { + has_pm = true; + + xe_assert(pxp->xe, !dev_pm_smart_suspend(pxp->xe->drm.dev)); + xe_pm_runtime_get_noresume(pxp->xe); + } + + mutex_lock(&pxp->mutex); + + xe_assert(pxp->xe, pxp->status == XE_PXP_SUSPENDED); + + if (err) { + pxp->status = XE_PXP_ERROR; + } else { + pxp->status = XE_PXP_NEEDS_TERMINATION; + + if (pxp->needs_termination_on_resume) { + pxp->needs_termination_on_resume = false; + + /* + * We can't call pxp_terminate_hw directly from here + * because we're not allowed to do allocations within + * the rpm resume call, so we defer the termination to + * the worker that we use for the termination irqs. + * However, we do not want any PXP ops to go through + * between the suspend completing and the worker + * starting, so we need to do the termination prep + * immediately, which will mark the termination as in + * progress and stall PXP ops. + */ + if (pxp_prep_for_termination(pxp)) { + has_pm = false; /* move PM ref ownership to worker */ + + atomic_or(PXP_TERMINATION_REQUEST_ON_RESUME, &pxp->events.pending); + queue_work(pxp->events.wq, &pxp->events.work); + } + } + } + + mutex_unlock(&pxp->mutex); + + if (has_pm) + xe_pm_runtime_put(pxp->xe); +} diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h new file mode 100644 index 000000000..4fb6e0aff --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2024, Intel Corporation. All rights reserved. + */ + +#ifndef _XE_PXP_H_ +#define _XE_PXP_H_ + +#include + +struct drm_gem_object; +struct xe_bo; +struct xe_device; +struct xe_exec_queue; +struct xe_pxp; + +bool xe_pxp_is_supported(const struct xe_device *xe); +bool xe_pxp_is_enabled(const struct xe_pxp *pxp); +int xe_pxp_get_readiness_status(struct xe_pxp *pxp); + +int xe_pxp_init(struct xe_device *xe); +void xe_pxp_irq_handler(struct xe_device *xe, u16 iir); + +int xe_pxp_pm_suspend(struct xe_pxp *pxp); +void xe_pxp_pm_resume(struct xe_pxp *pxp); + +int xe_pxp_exec_queue_set_type(struct xe_pxp *pxp, struct xe_exec_queue *q, u8 type); +int xe_pxp_exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q); +void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q); + +int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo); +int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo); +int xe_pxp_obj_key_check(struct drm_gem_object *obj); + +#endif /* _XE_PXP_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_debugfs.c b/drivers/gpu/drm/xe/xe_pxp_debugfs.c new file mode 100644 index 000000000..d6e2e41bc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp_debugfs.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include "xe_pxp_debugfs.h" + +#include + +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_pxp.h" +#include "xe_pxp_types.h" +#include "regs/xe_irq_regs.h" + +static struct xe_pxp *node_to_pxp(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static const char *pxp_status_to_str(struct xe_pxp *pxp) +{ + lockdep_assert_held(&pxp->mutex); + + switch (pxp->status) { + case XE_PXP_ERROR: + return "error"; + case XE_PXP_NEEDS_TERMINATION: + return "needs termination"; + case XE_PXP_TERMINATION_IN_PROGRESS: + return "termination in progress"; + case XE_PXP_READY_TO_START: + return "ready to start"; + case XE_PXP_ACTIVE: + return "active"; + case XE_PXP_SUSPENDED: + return "suspended"; + default: + return "unknown"; + } +}; + +static int pxp_info(struct seq_file *m, void *data) +{ + struct xe_pxp *pxp = node_to_pxp(m->private); + struct drm_printer p = drm_seq_file_printer(m); + const char *status; + + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + mutex_lock(&pxp->mutex); + status = pxp_status_to_str(pxp); + + drm_printf(&p, "status: %s\n", status); + drm_printf(&p, "instance counter: %u\n", pxp->key_instance); + mutex_unlock(&pxp->mutex); + + return 0; +} + +static int pxp_terminate(struct seq_file *m, void *data) +{ + struct xe_pxp *pxp = node_to_pxp(m->private); + struct drm_printer p = drm_seq_file_printer(m); + int ready = xe_pxp_get_readiness_status(pxp); + + if (ready < 0) + return ready; /* disabled or error occurred */ + else if (!ready) + return -EBUSY; /* init still in progress */ + + /* no need for a termination if PXP is not active */ + if (pxp->status != XE_PXP_ACTIVE) { + drm_printf(&p, "PXP not active\n"); + return 0; + } + + /* simulate a termination interrupt */ + spin_lock_irq(&pxp->xe->irq.lock); + xe_pxp_irq_handler(pxp->xe, KCR_PXP_STATE_TERMINATED_INTERRUPT); + spin_unlock_irq(&pxp->xe->irq.lock); + + drm_printf(&p, "PXP termination queued\n"); + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"info", pxp_info, 0}, + {"terminate", pxp_terminate, 0}, +}; + +void xe_pxp_debugfs_register(struct xe_pxp *pxp) +{ + struct drm_minor *minor; + struct drm_info_list *local; + struct dentry *root; + int i; + + if (!xe_pxp_is_enabled(pxp)) + return; + + minor = pxp->xe->drm.primary; + if (!minor->debugfs_root) + return; + +#define DEBUGFS_SIZE (ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list)) + local = drmm_kmalloc(&pxp->xe->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) + return; + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = pxp; + + root = debugfs_create_dir("pxp", minor->debugfs_root); + if (IS_ERR(root)) + return; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + root, minor); +} diff --git a/drivers/gpu/drm/xe/xe_pxp_debugfs.h b/drivers/gpu/drm/xe/xe_pxp_debugfs.h new file mode 100644 index 000000000..2997de0c9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp_debugfs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_PXP_DEBUGFS_H_ +#define _XE_PXP_DEBUGFS_H_ + +struct xe_pxp; + +void xe_pxp_debugfs_register(struct xe_pxp *pxp); + +#endif /* _XE_PXP_DEBUGFS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.c b/drivers/gpu/drm/xe/xe_pxp_submit.c new file mode 100644 index 000000000..e60526e30 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp_submit.c @@ -0,0 +1,602 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2024 Intel Corporation. + */ + +#include "xe_pxp_submit.h" + +#include +#include + +#include "xe_device_types.h" +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_exec_queue.h" +#include "xe_gsc_submit.h" +#include "xe_gt.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_pxp.h" +#include "xe_pxp_types.h" +#include "xe_sched_job.h" +#include "xe_vm.h" +#include "abi/gsc_command_header_abi.h" +#include "abi/gsc_pxp_commands_abi.h" +#include "instructions/xe_gsc_commands.h" +#include "instructions/xe_mfx_commands.h" +#include "instructions/xe_mi_commands.h" + +/* + * The VCS is used for kernel-owned GGTT submissions to issue key termination. + * Terminations are serialized, so we only need a single queue and a single + * batch. + */ +static int allocate_vcs_execution_resources(struct xe_pxp *pxp) +{ + struct xe_gt *gt = pxp->gt; + struct xe_device *xe = pxp->xe; + struct xe_tile *tile = gt_to_tile(gt); + struct xe_hw_engine *hwe; + struct xe_exec_queue *q; + struct xe_bo *bo; + int err; + + hwe = xe_gt_hw_engine(gt, XE_ENGINE_CLASS_VIDEO_DECODE, 0, true); + if (!hwe) + return -ENODEV; + + q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1, hwe, + EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_PERMANENT, 0); + if (IS_ERR(q)) + return PTR_ERR(q); + + /* + * Each termination is 16 DWORDS, so 4K is enough to contain a + * termination for each sessions. + */ + bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED | XE_BO_FLAG_GGTT, + false); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + goto out_queue; + } + + pxp->vcs_exec.q = q; + pxp->vcs_exec.bo = bo; + + return 0; + +out_queue: + xe_exec_queue_put(q); + return err; +} + +static void destroy_vcs_execution_resources(struct xe_pxp *pxp) +{ + if (pxp->vcs_exec.bo) + xe_bo_unpin_map_no_vm(pxp->vcs_exec.bo); + + if (pxp->vcs_exec.q) + xe_exec_queue_put(pxp->vcs_exec.q); +} + +#define PXP_BB_SIZE XE_PAGE_SIZE +static int allocate_gsc_client_resources(struct xe_gt *gt, + struct xe_pxp_gsc_client_resources *gsc_res, + size_t inout_size) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = tile_to_xe(tile); + struct xe_validation_ctx ctx; + struct xe_hw_engine *hwe; + struct drm_exec exec; + struct xe_vm *vm; + struct xe_bo *bo; + struct xe_exec_queue *q; + struct dma_fence *fence; + long timeout; + int err = 0; + + hwe = xe_gt_hw_engine(gt, XE_ENGINE_CLASS_OTHER, 0, true); + + /* we shouldn't reach here if the GSC engine is not available */ + xe_assert(xe, hwe); + + /* PXP instructions must be issued from PPGTT */ + vm = xe_vm_create(xe, XE_VM_FLAG_GSC, NULL); + if (IS_ERR(vm)) + return PTR_ERR(vm); + + /* We allocate a single object for the batch and the in/out memory */ + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags){}, err) { + err = xe_vm_drm_exec_lock(vm, &exec); + drm_exec_retry_on_contention(&exec); + if (err) + break; + + bo = xe_bo_create_pin_map(xe, tile, vm, PXP_BB_SIZE + inout_size * 2, + ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED | + XE_BO_FLAG_NEEDS_UC, &exec); + drm_exec_retry_on_contention(&exec); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + } + if (err) + goto vm_out; + + fence = xe_vm_bind_kernel_bo(vm, bo, NULL, 0, XE_CACHE_WB); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto bo_out; + } + + timeout = dma_fence_wait_timeout(fence, false, HZ); + dma_fence_put(fence); + if (timeout <= 0) { + err = timeout ?: -ETIME; + goto bo_out; + } + + q = xe_exec_queue_create(xe, vm, BIT(hwe->logical_instance), 1, hwe, + EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_PERMANENT, 0); + if (IS_ERR(q)) { + err = PTR_ERR(q); + goto bo_out; + } + + gsc_res->vm = vm; + gsc_res->bo = bo; + gsc_res->inout_size = inout_size; + gsc_res->batch = IOSYS_MAP_INIT_OFFSET(&bo->vmap, 0); + gsc_res->msg_in = IOSYS_MAP_INIT_OFFSET(&bo->vmap, PXP_BB_SIZE); + gsc_res->msg_out = IOSYS_MAP_INIT_OFFSET(&bo->vmap, PXP_BB_SIZE + inout_size); + gsc_res->q = q; + + /* initialize host-session-handle (for all Xe-to-gsc-firmware PXP cmds) */ + gsc_res->host_session_handle = xe_gsc_create_host_session_id(); + + return 0; + +bo_out: + xe_bo_unpin_map_no_vm(bo); +vm_out: + xe_vm_close_and_put(vm); + + return err; +} + +static void destroy_gsc_client_resources(struct xe_pxp_gsc_client_resources *gsc_res) +{ + if (!gsc_res->q) + return; + + xe_exec_queue_put(gsc_res->q); + xe_bo_unpin_map_no_vm(gsc_res->bo); + xe_vm_close_and_put(gsc_res->vm); +} + +/** + * xe_pxp_allocate_execution_resources - Allocate PXP submission objects + * @pxp: the xe_pxp structure + * + * Allocates exec_queues objects for VCS and GSCCS submission. The GSCCS + * submissions are done via PPGTT, so this function allocates a VM for it and + * maps the object into it. + * + * Returns 0 if the allocation and mapping is successful, an errno value + * otherwise. + */ +int xe_pxp_allocate_execution_resources(struct xe_pxp *pxp) +{ + int err; + + err = allocate_vcs_execution_resources(pxp); + if (err) + return err; + + /* + * PXP commands can require a lot of BO space (see PXP_MAX_PACKET_SIZE), + * but we currently only support a subset of commands that are small + * (< 20 dwords), so a single page is enough for now. + */ + err = allocate_gsc_client_resources(pxp->gt, &pxp->gsc_res, XE_PAGE_SIZE); + if (err) + goto destroy_vcs_context; + + return 0; + +destroy_vcs_context: + destroy_vcs_execution_resources(pxp); + return err; +} + +void xe_pxp_destroy_execution_resources(struct xe_pxp *pxp) +{ + destroy_gsc_client_resources(&pxp->gsc_res); + destroy_vcs_execution_resources(pxp); +} + +#define emit_cmd(xe_, map_, offset_, val_) \ + xe_map_wr(xe_, map_, (offset_) * sizeof(u32), u32, val_) + +/* stall until prior PXP and MFX/HCP/HUC objects are completed */ +#define MFX_WAIT_PXP (MFX_WAIT | \ + MFX_WAIT_DW0_PXP_SYNC_CONTROL_FLAG | \ + MFX_WAIT_DW0_MFX_SYNC_CONTROL_FLAG) +static u32 pxp_emit_wait(struct xe_device *xe, struct iosys_map *batch, u32 offset) +{ + /* wait for cmds to go through */ + emit_cmd(xe, batch, offset++, MFX_WAIT_PXP); + emit_cmd(xe, batch, offset++, 0); + + return offset; +} + +static u32 pxp_emit_session_selection(struct xe_device *xe, struct iosys_map *batch, + u32 offset, u32 idx) +{ + offset = pxp_emit_wait(xe, batch, offset); + + /* pxp off */ + emit_cmd(xe, batch, offset++, MI_FLUSH_DW | MI_FLUSH_IMM_DW); + emit_cmd(xe, batch, offset++, 0); + emit_cmd(xe, batch, offset++, 0); + emit_cmd(xe, batch, offset++, 0); + + /* select session */ + emit_cmd(xe, batch, offset++, MI_SET_APPID | MI_SET_APPID_SESSION_ID(idx)); + emit_cmd(xe, batch, offset++, 0); + + offset = pxp_emit_wait(xe, batch, offset); + + /* pxp on */ + emit_cmd(xe, batch, offset++, MI_FLUSH_DW | + MI_FLUSH_DW_PROTECTED_MEM_EN | + MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX | + MI_FLUSH_IMM_DW); + emit_cmd(xe, batch, offset++, LRC_PPHWSP_PXP_INVAL_SCRATCH_ADDR | + MI_FLUSH_DW_USE_GTT); + emit_cmd(xe, batch, offset++, 0); + emit_cmd(xe, batch, offset++, 0); + + offset = pxp_emit_wait(xe, batch, offset); + + return offset; +} + +static u32 pxp_emit_inline_termination(struct xe_device *xe, + struct iosys_map *batch, u32 offset) +{ + /* session inline termination */ + emit_cmd(xe, batch, offset++, CRYPTO_KEY_EXCHANGE); + emit_cmd(xe, batch, offset++, 0); + + return offset; +} + +static u32 pxp_emit_session_termination(struct xe_device *xe, struct iosys_map *batch, + u32 offset, u32 idx) +{ + offset = pxp_emit_session_selection(xe, batch, offset, idx); + offset = pxp_emit_inline_termination(xe, batch, offset); + + return offset; +} + +/** + * xe_pxp_submit_session_termination - submits a PXP inline termination + * @pxp: the xe_pxp structure + * @id: the session to terminate + * + * Emit an inline termination via the VCS engine to terminate a session. + * + * Returns 0 if the submission is successful, an errno value otherwise. + */ +int xe_pxp_submit_session_termination(struct xe_pxp *pxp, u32 id) +{ + struct xe_sched_job *job; + struct dma_fence *fence; + long timeout; + u32 offset = 0; + u64 addr = xe_bo_ggtt_addr(pxp->vcs_exec.bo); + + offset = pxp_emit_session_termination(pxp->xe, &pxp->vcs_exec.bo->vmap, offset, id); + offset = pxp_emit_wait(pxp->xe, &pxp->vcs_exec.bo->vmap, offset); + emit_cmd(pxp->xe, &pxp->vcs_exec.bo->vmap, offset, MI_BATCH_BUFFER_END); + + job = xe_sched_job_create(pxp->vcs_exec.q, &addr); + if (IS_ERR(job)) + return PTR_ERR(job); + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, HZ); + + dma_fence_put(fence); + + if (!timeout) + return -ETIMEDOUT; + else if (timeout < 0) + return timeout; + + return 0; +} + +static bool +is_fw_err_platform_config(u32 type) +{ + switch (type) { + case PXP_STATUS_ERROR_API_VERSION: + case PXP_STATUS_PLATFCONFIG_KF1_NOVERIF: + case PXP_STATUS_PLATFCONFIG_KF1_BAD: + case PXP_STATUS_PLATFCONFIG_FIXED_KF1_NOT_SUPPORTED: + return true; + default: + break; + } + return false; +} + +static const char * +fw_err_to_string(u32 type) +{ + switch (type) { + case PXP_STATUS_ERROR_API_VERSION: + return "ERR_API_VERSION"; + case PXP_STATUS_NOT_READY: + return "ERR_NOT_READY"; + case PXP_STATUS_PLATFCONFIG_KF1_NOVERIF: + case PXP_STATUS_PLATFCONFIG_KF1_BAD: + case PXP_STATUS_PLATFCONFIG_FIXED_KF1_NOT_SUPPORTED: + return "ERR_PLATFORM_CONFIG"; + default: + break; + } + return NULL; +} + +static int pxp_pkt_submit(struct xe_exec_queue *q, u64 batch_addr) +{ + struct xe_gt *gt = q->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_sched_job *job; + struct dma_fence *fence; + long timeout; + + xe_assert(xe, q->hwe->engine_id == XE_HW_ENGINE_GSCCS0); + + job = xe_sched_job_create(q, &batch_addr); + if (IS_ERR(job)) + return PTR_ERR(job); + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, HZ); + dma_fence_put(fence); + if (timeout < 0) + return timeout; + else if (!timeout) + return -ETIME; + + return 0; +} + +static void emit_pxp_heci_cmd(struct xe_device *xe, struct iosys_map *batch, + u64 addr_in, u32 size_in, u64 addr_out, u32 size_out) +{ + u32 len = 0; + + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, GSC_HECI_CMD_PKT); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, lower_32_bits(addr_in)); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, upper_32_bits(addr_in)); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, size_in); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, lower_32_bits(addr_out)); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, upper_32_bits(addr_out)); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, size_out); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, 0); + xe_map_wr(xe, batch, len++ * sizeof(u32), u32, MI_BATCH_BUFFER_END); +} + +#define GSC_PENDING_RETRY_MAXCOUNT 40 +#define GSC_PENDING_RETRY_PAUSE_MS 50 +static int gsccs_send_message(struct xe_pxp_gsc_client_resources *gsc_res, + void *msg_in, size_t msg_in_size, + void *msg_out, size_t msg_out_size_max) +{ + struct xe_device *xe = gsc_res->vm->xe; + const size_t max_msg_size = gsc_res->inout_size - sizeof(struct intel_gsc_mtl_header); + u32 wr_offset; + u32 rd_offset; + u32 reply_size; + u32 min_reply_size = 0; + int ret; + int retry = GSC_PENDING_RETRY_MAXCOUNT; + + if (msg_in_size > max_msg_size || msg_out_size_max > max_msg_size) + return -ENOSPC; + + wr_offset = xe_gsc_emit_header(xe, &gsc_res->msg_in, 0, + HECI_MEADDRESS_PXP, + gsc_res->host_session_handle, + msg_in_size); + + /* NOTE: zero size packets are used for session-cleanups */ + if (msg_in && msg_in_size) { + xe_map_memcpy_to(xe, &gsc_res->msg_in, wr_offset, + msg_in, msg_in_size); + min_reply_size = sizeof(struct pxp_cmd_header); + } + + /* Make sure the reply header does not contain stale data */ + xe_gsc_poison_header(xe, &gsc_res->msg_out, 0); + + /* + * The BO is mapped at address 0 of the PPGTT, so no need to add its + * base offset when calculating the in/out addresses. + */ + emit_pxp_heci_cmd(xe, &gsc_res->batch, PXP_BB_SIZE, + wr_offset + msg_in_size, PXP_BB_SIZE + gsc_res->inout_size, + wr_offset + msg_out_size_max); + + xe_device_wmb(xe); + + /* + * If the GSC needs to communicate with CSME to complete our request, + * it'll set the "pending" flag in the return header. In this scenario + * we're expected to wait 50ms to give some time to the proxy code to + * handle the GSC<->CSME communication and then try again. Note that, + * although in most case the 50ms window is enough, the proxy flow is + * not actually guaranteed to complete within that time period, so we + * might have to try multiple times, up to a worst case of 2 seconds, + * after which the request is considered aborted. + */ + do { + ret = pxp_pkt_submit(gsc_res->q, 0); + if (ret) + break; + + if (xe_gsc_check_and_update_pending(xe, &gsc_res->msg_in, 0, + &gsc_res->msg_out, 0)) { + ret = -EAGAIN; + msleep(GSC_PENDING_RETRY_PAUSE_MS); + } + } while (--retry && ret == -EAGAIN); + + if (ret) { + drm_err(&xe->drm, "failed to submit GSC PXP message (%pe)\n", ERR_PTR(ret)); + return ret; + } + + ret = xe_gsc_read_out_header(xe, &gsc_res->msg_out, 0, + min_reply_size, &rd_offset); + if (ret) { + drm_err(&xe->drm, "invalid GSC reply for PXP (%pe)\n", ERR_PTR(ret)); + return ret; + } + + if (msg_out && min_reply_size) { + reply_size = xe_map_rd_field(xe, &gsc_res->msg_out, rd_offset, + struct pxp_cmd_header, buffer_len); + reply_size += sizeof(struct pxp_cmd_header); + + if (reply_size > msg_out_size_max) { + drm_warn(&xe->drm, "PXP reply size overflow: %u (%zu)\n", + reply_size, msg_out_size_max); + reply_size = msg_out_size_max; + } + + xe_map_memcpy_from(xe, msg_out, &gsc_res->msg_out, + rd_offset, reply_size); + } + + xe_gsc_poison_header(xe, &gsc_res->msg_in, 0); + + return ret; +} + +/** + * xe_pxp_submit_session_init - submits a PXP GSC session initialization + * @gsc_res: the pxp client resources + * @id: the session to initialize + * + * Submit a message to the GSC FW to initialize (i.e. start) a PXP session. + * + * Returns 0 if the submission is successful, an errno value otherwise. + */ +int xe_pxp_submit_session_init(struct xe_pxp_gsc_client_resources *gsc_res, u32 id) +{ + struct xe_device *xe = gsc_res->vm->xe; + struct pxp43_create_arb_in msg_in = {0}; + struct pxp43_create_arb_out msg_out = {0}; + int ret; + + msg_in.header.api_version = PXP_APIVER(4, 3); + msg_in.header.command_id = PXP43_CMDID_INIT_SESSION; + msg_in.header.stream_id = (FIELD_PREP(PXP43_INIT_SESSION_APPID, id) | + FIELD_PREP(PXP43_INIT_SESSION_VALID, 1) | + FIELD_PREP(PXP43_INIT_SESSION_APPTYPE, 0)); + msg_in.header.buffer_len = sizeof(msg_in) - sizeof(msg_in.header); + + if (id == DRM_XE_PXP_HWDRM_DEFAULT_SESSION) + msg_in.protection_mode = PXP43_INIT_SESSION_PROTECTION_ARB; + + ret = gsccs_send_message(gsc_res, &msg_in, sizeof(msg_in), + &msg_out, sizeof(msg_out)); + if (ret) { + drm_err(&xe->drm, "Failed to init PXP session %u (%pe)\n", id, ERR_PTR(ret)); + } else if (msg_out.header.status != 0) { + ret = -EIO; + + if (is_fw_err_platform_config(msg_out.header.status)) + drm_info_once(&xe->drm, + "Failed to init PXP session %u due to BIOS/SOC, s=0x%x(%s)\n", + id, msg_out.header.status, + fw_err_to_string(msg_out.header.status)); + else + drm_dbg(&xe->drm, "Failed to init PXP session %u, s=0x%x\n", + id, msg_out.header.status); + } + + return ret; +} + +/** + * xe_pxp_submit_session_invalidation - submits a PXP GSC invalidation + * @gsc_res: the pxp client resources + * @id: the session to invalidate + * + * Submit a message to the GSC FW to notify it that a session has been + * terminated and is therefore invalid. + * + * Returns 0 if the submission is successful, an errno value otherwise. + */ +int xe_pxp_submit_session_invalidation(struct xe_pxp_gsc_client_resources *gsc_res, u32 id) +{ + struct xe_device *xe = gsc_res->vm->xe; + struct pxp43_inv_stream_key_in msg_in = {0}; + struct pxp43_inv_stream_key_out msg_out = {0}; + int ret = 0; + + /* + * Stream key invalidation reuses the same version 4.2 input/output + * command format but firmware requires 4.3 API interaction + */ + msg_in.header.api_version = PXP_APIVER(4, 3); + msg_in.header.command_id = PXP43_CMDID_INVALIDATE_STREAM_KEY; + msg_in.header.buffer_len = sizeof(msg_in) - sizeof(msg_in.header); + + msg_in.header.stream_id = FIELD_PREP(PXP_CMDHDR_EXTDATA_SESSION_VALID, 1); + msg_in.header.stream_id |= FIELD_PREP(PXP_CMDHDR_EXTDATA_APP_TYPE, 0); + msg_in.header.stream_id |= FIELD_PREP(PXP_CMDHDR_EXTDATA_SESSION_ID, id); + + ret = gsccs_send_message(gsc_res, &msg_in, sizeof(msg_in), + &msg_out, sizeof(msg_out)); + if (ret) { + drm_err(&xe->drm, "Failed to invalidate PXP stream-key %u (%pe)\n", + id, ERR_PTR(ret)); + } else if (msg_out.header.status != 0) { + ret = -EIO; + + if (is_fw_err_platform_config(msg_out.header.status)) + drm_info_once(&xe->drm, + "Failed to invalidate PXP stream-key %u: BIOS/SOC 0x%08x(%s)\n", + id, msg_out.header.status, + fw_err_to_string(msg_out.header.status)); + else + drm_dbg(&xe->drm, "Failed to invalidate stream-key %u, s=0x%08x\n", + id, msg_out.header.status); + } + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.h b/drivers/gpu/drm/xe/xe_pxp_submit.h new file mode 100644 index 000000000..dbbbe6b92 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp_submit.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2024, Intel Corporation. All rights reserved. + */ + +#ifndef _XE_PXP_SUBMIT_H_ +#define _XE_PXP_SUBMIT_H_ + +#include + +struct xe_pxp; +struct xe_pxp_gsc_client_resources; + +int xe_pxp_allocate_execution_resources(struct xe_pxp *pxp); +void xe_pxp_destroy_execution_resources(struct xe_pxp *pxp); + +int xe_pxp_submit_session_init(struct xe_pxp_gsc_client_resources *gsc_res, u32 id); +int xe_pxp_submit_session_termination(struct xe_pxp *pxp, u32 id); +int xe_pxp_submit_session_invalidation(struct xe_pxp_gsc_client_resources *gsc_res, + u32 id); + +#endif /* _XE_PXP_SUBMIT_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_types.h b/drivers/gpu/drm/xe/xe_pxp_types.h new file mode 100644 index 000000000..8132a9750 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pxp_types.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2024, Intel Corporation. All rights reserved. + */ + +#ifndef _XE_PXP_TYPES_H_ +#define _XE_PXP_TYPES_H_ + +#include +#include +#include +#include +#include +#include + +struct xe_bo; +struct xe_exec_queue; +struct xe_device; +struct xe_gt; +struct xe_vm; + +enum xe_pxp_status { + XE_PXP_ERROR = -1, + XE_PXP_NEEDS_TERMINATION = 0, /* starting status */ + XE_PXP_NEEDS_ADDITIONAL_TERMINATION, + XE_PXP_TERMINATION_IN_PROGRESS, + XE_PXP_READY_TO_START, + XE_PXP_START_IN_PROGRESS, + XE_PXP_ACTIVE, + XE_PXP_SUSPENDED, +}; + +/** + * struct xe_pxp_gsc_client_resources - resources for GSC submission by a PXP + * client. The GSC FW supports multiple GSC client active at the same time. + */ +struct xe_pxp_gsc_client_resources { + /** + * @host_session_handle: handle used to identify the client in messages + * sent to the GSC firmware. + */ + u64 host_session_handle; + /** @vm: VM used for PXP submissions to the GSCCS */ + struct xe_vm *vm; + /** @q: GSCCS exec queue for PXP submissions */ + struct xe_exec_queue *q; + + /** + * @bo: BO used for submissions to the GSCCS and GSC FW. It includes + * space for the GSCCS batch and the input/output buffers read/written + * by the FW + */ + struct xe_bo *bo; + /** @inout_size: size of each of the msg_in/out sections individually */ + u32 inout_size; + /** @batch: iosys_map to the batch memory within the BO */ + struct iosys_map batch; + /** @msg_in: iosys_map to the input memory within the BO */ + struct iosys_map msg_in; + /** @msg_out: iosys_map to the output memory within the BO */ + struct iosys_map msg_out; +}; + +/** + * struct xe_pxp - pxp state + */ +struct xe_pxp { + /** @xe: Backpoiner to the xe_device struct */ + struct xe_device *xe; + + /** + * @gt: pointer to the gt that owns the submission-side of PXP + * (VDBOX, KCR and GSC) + */ + struct xe_gt *gt; + + /** @vcs_exec: kernel-owned objects for PXP submissions to the VCS */ + struct { + /** @vcs_exec.q: kernel-owned VCS exec queue used for PXP terminations */ + struct xe_exec_queue *q; + /** @vcs_exec.bo: BO used for submissions to the VCS */ + struct xe_bo *bo; + } vcs_exec; + + /** @gsc_res: kernel-owned objects for PXP submissions to the GSCCS */ + struct xe_pxp_gsc_client_resources gsc_res; + + /** @events: wrapper for the worker and queue used for PXP event handling */ + struct { + /** @events.work: worker that manages termination events. */ + struct work_struct work; + /** @events.wq: workqueue on which to queue the work. */ + struct workqueue_struct *wq; + /** @events.pending: pending events */ + atomic_t pending; +#define PXP_TERMINATION_REQUEST_IRQ BIT(0) +#define PXP_TERMINATION_REQUEST_ON_RESUME BIT(1) +#define PXP_TERMINATION_REQUEST (PXP_TERMINATION_REQUEST_IRQ | \ + PXP_TERMINATION_REQUEST_ON_RESUME) +#define PXP_TERMINATION_COMPLETE_IRQ BIT(2) + } events; + + /** @mutex: protects the pxp status and the queue list */ + struct mutex mutex; + /** @status: the current pxp status */ + enum xe_pxp_status status; + /** @activation: completion struct that tracks pxp start */ + struct completion activation; + /** @termination: completion struct that tracks terminations */ + struct completion termination; + + /** @queues: management of exec_queues that use PXP */ + struct { + /** @queues.lock: spinlock protecting the queue management */ + spinlock_t lock; + /** @queues.list: list of exec_queues that use PXP */ + struct list_head list; + } queues; + + /** + * @key_instance: keep track of the current iteration of the PXP key. + * Note that, due to the time needed for PXP termination and re-start + * to complete, the minimum time between 2 subsequent increases of this + * variable is 50ms, and even that only if there is a continuous attack; + * normal behavior is for this to increase much much slower than that. + * This means that we don't expect this to ever wrap and don't implement + * that case in the code. + */ + u32 key_instance; + /** + * @last_suspend_key_instance: value of key_instance at the last + * suspend. Used to check if any PXP session has been created between + * suspend cycles. + */ + u32 last_suspend_key_instance; + /** + * @needs_termination_on_resume: indicates if PXP termination is needed + * on resume. This is set if PXP was active when we suspend and it is + * cleared when we queue the termination on resume. Since the suspend + * and resume calls cannot execute at the same time, this variable does + * not need to be protected by the PXP lock. + */ + bool needs_termination_on_resume; +}; + +#endif /* _XE_PXP_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c new file mode 100644 index 000000000..dc975f595 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_query.c @@ -0,0 +1,824 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_query.h" + +#include +#include + +#include +#include +#include + +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_eu_stall.h" +#include "xe_exec_queue.h" +#include "xe_force_wake.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_topology.h" +#include "xe_guc_hwconfig.h" +#include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_oa.h" +#include "xe_pxp.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vram_types.h" +#include "xe_wa.h" + +static const u16 xe_to_user_engine_class[] = { + [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, + [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY, + [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE, + [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE, + [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, +}; + +static const enum xe_engine_class user_to_xe_engine_class[] = { + [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, + [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, + [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, + [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, + [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, +}; + +static size_t calc_hw_engine_info_size(struct xe_device *xe) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_gt *gt; + u8 gt_id; + int i = 0; + + for_each_gt(gt, xe, gt_id) + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + i++; + } + + return sizeof(struct drm_xe_query_engines) + + i * sizeof(struct drm_xe_engine); +} + +typedef u64 (*__ktime_func_t)(void); +static __ktime_func_t __clock_id_to_func(clockid_t clk_id) +{ + /* + * Use logic same as the perf subsystem to allow user to select the + * reference clock id to be used for timestamps. + */ + switch (clk_id) { + case CLOCK_MONOTONIC: + return &ktime_get_ns; + case CLOCK_MONOTONIC_RAW: + return &ktime_get_raw_ns; + case CLOCK_REALTIME: + return &ktime_get_real_ns; + case CLOCK_BOOTTIME: + return &ktime_get_boottime_ns; + case CLOCK_TAI: + return &ktime_get_clocktai_ns; + default: + return NULL; + } +} + +static void +hwe_read_timestamp(struct xe_hw_engine *hwe, u64 *engine_ts, u64 *cpu_ts, + u64 *cpu_delta, __ktime_func_t cpu_clock) +{ + struct xe_mmio *mmio = &hwe->gt->mmio; + u32 upper, lower, old_upper, loop = 0; + struct xe_reg upper_reg = RING_TIMESTAMP_UDW(hwe->mmio_base), + lower_reg = RING_TIMESTAMP(hwe->mmio_base); + + upper = xe_mmio_read32(mmio, upper_reg); + do { + *cpu_delta = local_clock(); + *cpu_ts = cpu_clock(); + lower = xe_mmio_read32(mmio, lower_reg); + *cpu_delta = local_clock() - *cpu_delta; + old_upper = upper; + upper = xe_mmio_read32(mmio, upper_reg); + } while (upper != old_upper && loop++ < 2); + + *engine_ts = (u64)upper << 32 | lower; +} + +static int +query_engine_cycles(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + struct drm_xe_query_engine_cycles __user *query_ptr; + struct drm_xe_engine_class_instance *eci; + struct drm_xe_query_engine_cycles resp; + size_t size = sizeof(resp); + enum xe_force_wake_domains fw_domain; + __ktime_func_t cpu_clock; + struct xe_hw_engine *hwe; + struct xe_gt *gt; + + if (IS_SRIOV_VF(xe)) + return -EOPNOTSUPP; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + query_ptr = u64_to_user_ptr(query->data); + if (copy_from_user(&resp, query_ptr, size)) + return -EFAULT; + + cpu_clock = __clock_id_to_func(resp.clockid); + if (!cpu_clock) + return -EINVAL; + + eci = &resp.eci; + gt = xe_device_get_gt(xe, eci->gt_id); + if (!gt) + return -EINVAL; + + if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) + return -EINVAL; + + hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class], + eci->engine_instance, true); + if (!hwe) + return -EINVAL; + + fw_domain = xe_hw_engine_to_fw_domain(hwe); + + xe_with_force_wake(fw_ref, gt_to_fw(gt), fw_domain) { + if (!xe_force_wake_ref_has_domain(fw_ref.domains, fw_domain)) + return -EIO; + + hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp, + &resp.cpu_delta, cpu_clock); + } + + if (GRAPHICS_VER(xe) >= 20) + resp.width = 64; + else + resp.width = 36; + + /* Only write to the output fields of user query */ + if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp) || + put_user(resp.cpu_delta, &query_ptr->cpu_delta) || + put_user(resp.engine_cycles, &query_ptr->engine_cycles) || + put_user(resp.width, &query_ptr->width)) + return -EFAULT; + + return 0; +} + +static int query_engines(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + size_t size = calc_hw_engine_info_size(xe); + struct drm_xe_query_engines __user *query_ptr = + u64_to_user_ptr(query->data); + struct drm_xe_query_engines *engines; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_gt *gt; + u8 gt_id; + int i = 0; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + engines = kzalloc(size, GFP_KERNEL); + if (!engines) + return -ENOMEM; + + for_each_gt(gt, xe, gt_id) + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + + engines->engines[i].instance.engine_class = + xe_to_user_engine_class[hwe->class]; + engines->engines[i].instance.engine_instance = + hwe->logical_instance; + engines->engines[i].instance.gt_id = gt->info.id; + + i++; + } + + engines->num_engines = i; + + if (copy_to_user(query_ptr, engines, size)) { + kfree(engines); + return -EFAULT; + } + kfree(engines); + + return 0; +} + +static size_t calc_mem_regions_size(struct xe_device *xe) +{ + u32 num_managers = 1; + int i; + + if (xe_device_is_admin_only(xe)) + return sizeof(struct drm_xe_query_mem_regions); + + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) + if (ttm_manager_type(&xe->ttm, i)) + num_managers++; + + return offsetof(struct drm_xe_query_mem_regions, mem_regions[num_managers]); +} + +static int query_mem_regions(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + size_t size = calc_mem_regions_size(xe); + struct drm_xe_query_mem_regions *mem_regions; + struct drm_xe_query_mem_regions __user *query_ptr = + u64_to_user_ptr(query->data); + struct ttm_resource_manager *man; + int ret, i; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + mem_regions = kzalloc(size, GFP_KERNEL); + if (XE_IOCTL_DBG(xe, !mem_regions)) + return -ENOMEM; + + if (xe_device_is_admin_only(xe)) + goto user_copy; + + man = ttm_manager_type(&xe->ttm, XE_PL_TT); + mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; + /* + * The instance needs to be a unique number that represents the index + * in the placement mask used at xe_gem_create_ioctl() for the + * xe_bo_create() placement. + */ + mem_regions->mem_regions[0].instance = 0; + mem_regions->mem_regions[0].min_page_size = PAGE_SIZE; + mem_regions->mem_regions[0].total_size = man->size << PAGE_SHIFT; + mem_regions->mem_regions[0].used = ttm_resource_manager_usage(man); + mem_regions->num_mem_regions = 1; + + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + mem_regions->mem_regions[mem_regions->num_mem_regions].mem_class = + DRM_XE_MEM_REGION_CLASS_VRAM; + mem_regions->mem_regions[mem_regions->num_mem_regions].instance = + mem_regions->num_mem_regions; + mem_regions->mem_regions[mem_regions->num_mem_regions].min_page_size = + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? + SZ_64K : PAGE_SIZE; + mem_regions->mem_regions[mem_regions->num_mem_regions].total_size = + man->size; + + xe_ttm_vram_get_used(man, + &mem_regions->mem_regions + [mem_regions->num_mem_regions].used, + &mem_regions->mem_regions + [mem_regions->num_mem_regions].cpu_visible_used); + + mem_regions->mem_regions[mem_regions->num_mem_regions].cpu_visible_size = + xe_ttm_vram_get_cpu_visible_size(man); + mem_regions->num_mem_regions++; + } + } + +user_copy: + if (!copy_to_user(query_ptr, mem_regions, size)) + ret = 0; + else + ret = -ENOSPC; + + kfree(mem_regions); + return ret; +} + +static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) +{ + const u32 num_params = DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; + size_t size = + sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); + struct drm_xe_query_config __user *query_ptr = + u64_to_user_ptr(query->data); + struct drm_xe_query_config *config; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + config = kzalloc(size, GFP_KERNEL); + if (!config) + return -ENOMEM; + + config->num_params = num_params; + config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = + xe->info.devid | (xe->info.revid << 16); + if (xe->mem.vram) + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; + if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_XE_GPUSVM)) + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR; + if (GRAPHICS_VER(xe) >= 20) + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT; + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY; + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_DISABLE_STATE_CACHE_PERF_FIX; + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_PURGING_SUPPORT; + config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; + config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; + config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = + xe_exec_queue_device_get_max_priority(xe); + + if (copy_to_user(query_ptr, config, size)) { + kfree(config); + return -EFAULT; + } + kfree(config); + + return 0; +} + +static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query) +{ + struct xe_gt *gt; + size_t size = sizeof(struct drm_xe_query_gt_list) + + xe->info.gt_count * sizeof(struct drm_xe_gt); + struct drm_xe_query_gt_list __user *query_ptr = + u64_to_user_ptr(query->data); + struct drm_xe_query_gt_list *gt_list; + int iter = 0; + u8 id; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + gt_list = kzalloc(size, GFP_KERNEL); + if (!gt_list) + return -ENOMEM; + + gt_list->num_gt = xe->info.gt_count; + + for_each_gt(gt, xe, id) { + if (xe_gt_is_media_type(gt)) + gt_list->gt_list[iter].type = DRM_XE_QUERY_GT_TYPE_MEDIA; + else + gt_list->gt_list[iter].type = DRM_XE_QUERY_GT_TYPE_MAIN; + gt_list->gt_list[iter].tile_id = gt_to_tile(gt)->id; + gt_list->gt_list[iter].gt_id = gt->info.id; + gt_list->gt_list[iter].reference_clock = gt->info.reference_clock; + /* + * The mem_regions indexes in the mask below need to + * directly identify the struct + * drm_xe_query_mem_regions' instance constructed at + * query_mem_regions() + * + * For our current platforms: + * Bit 0 -> System Memory + * Bit 1 -> VRAM0 on Tile0 + * Bit 2 -> VRAM1 on Tile1 + * However the uAPI is generic and it's userspace's + * responsibility to check the mem_class, without any + * assumption. + */ + if (!IS_DGFX(xe)) + gt_list->gt_list[iter].near_mem_regions = 0x1; + else + gt_list->gt_list[iter].near_mem_regions = + BIT(gt_to_tile(gt)->mem.vram->id) << 1; + gt_list->gt_list[iter].far_mem_regions = xe->info.mem_region_mask ^ + gt_list->gt_list[iter].near_mem_regions; + + gt_list->gt_list[iter].ip_ver_major = + REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid); + gt_list->gt_list[iter].ip_ver_minor = + REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid); + gt_list->gt_list[iter].ip_ver_rev = + REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid); + + iter++; + } + + if (copy_to_user(query_ptr, gt_list, size)) { + kfree(gt_list); + return -EFAULT; + } + kfree(gt_list); + + return 0; +} + +static int query_hwconfig(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + struct xe_gt *gt = xe_root_mmio_gt(xe); + size_t size = gt ? xe_guc_hwconfig_size(>->uc.guc) : 0; + void __user *query_ptr = u64_to_user_ptr(query->data); + void *hwconfig; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + hwconfig = kzalloc(size, GFP_KERNEL); + if (!hwconfig) + return -ENOMEM; + + xe_guc_hwconfig_copy(>->uc.guc, hwconfig); + + if (copy_to_user(query_ptr, hwconfig, size)) { + kfree(hwconfig); + return -EFAULT; + } + kfree(hwconfig); + + return 0; +} + +static size_t calc_topo_query_size(struct xe_device *xe) +{ + struct xe_gt *gt; + size_t query_size = 0; + int id; + + for_each_gt(gt, xe, id) { + query_size += 3 * sizeof(struct drm_xe_query_topology_mask) + + sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) + + sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) + + sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss); + + /* L3bank mask may not be available for some GTs */ + if (xe_gt_topology_report_l3(gt)) + query_size += sizeof(struct drm_xe_query_topology_mask) + + sizeof_field(struct xe_gt, fuse_topo.l3_bank_mask); + } + + return query_size; +} + +static int copy_mask(void __user **ptr, + struct drm_xe_query_topology_mask *topo, + void *mask, size_t mask_size) +{ + topo->num_bytes = mask_size; + + if (copy_to_user(*ptr, topo, sizeof(*topo))) + return -EFAULT; + *ptr += sizeof(*topo); + + if (copy_to_user(*ptr, mask, mask_size)) + return -EFAULT; + *ptr += mask_size; + + return 0; +} + +static int query_gt_topology(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + void __user *query_ptr = u64_to_user_ptr(query->data); + size_t size = calc_topo_query_size(xe); + struct drm_xe_query_topology_mask topo; + struct xe_gt *gt; + int id; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + for_each_gt(gt, xe, id) { + int err; + + topo.gt_id = id; + + topo.type = DRM_XE_TOPO_DSS_GEOMETRY; + err = copy_mask(&query_ptr, &topo, gt->fuse_topo.g_dss_mask, + sizeof(gt->fuse_topo.g_dss_mask)); + if (err) + return err; + + topo.type = DRM_XE_TOPO_DSS_COMPUTE; + err = copy_mask(&query_ptr, &topo, gt->fuse_topo.c_dss_mask, + sizeof(gt->fuse_topo.c_dss_mask)); + if (err) + return err; + + /* + * If the kernel doesn't have a way to obtain a correct L3bank + * mask, then it's better to omit L3 from the query rather than + * reporting bogus or zeroed information to userspace. + */ + if (xe_gt_topology_report_l3(gt)) { + topo.type = DRM_XE_TOPO_L3_BANK; + err = copy_mask(&query_ptr, &topo, gt->fuse_topo.l3_bank_mask, + sizeof(gt->fuse_topo.l3_bank_mask)); + if (err) + return err; + } + + topo.type = gt->fuse_topo.eu_type == XE_GT_EU_TYPE_SIMD16 ? + DRM_XE_TOPO_SIMD16_EU_PER_DSS : + DRM_XE_TOPO_EU_PER_DSS; + err = copy_mask(&query_ptr, &topo, + gt->fuse_topo.eu_mask_per_dss, + sizeof(gt->fuse_topo.eu_mask_per_dss)); + if (err) + return err; + } + + return 0; +} + +static int +query_uc_fw_version(struct xe_device *xe, struct drm_xe_device_query *query) +{ + struct drm_xe_query_uc_fw_version __user *query_ptr = u64_to_user_ptr(query->data); + size_t size = sizeof(struct drm_xe_query_uc_fw_version); + struct drm_xe_query_uc_fw_version resp; + struct xe_uc_fw_version *version = NULL; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + if (copy_from_user(&resp, query_ptr, size)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, resp.pad || resp.pad2 || resp.reserved)) + return -EINVAL; + + switch (resp.uc_type) { + case XE_QUERY_UC_TYPE_GUC_SUBMISSION: { + struct xe_guc *guc = &xe->tiles[0].primary_gt->uc.guc; + + version = &guc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]; + break; + } + case XE_QUERY_UC_TYPE_HUC: { + struct xe_gt *media_gt = NULL; + struct xe_huc *huc; + + if (MEDIA_VER(xe) >= 13) { + struct xe_tile *tile; + u8 gt_id; + + for_each_tile(tile, xe, gt_id) { + if (tile->media_gt) { + media_gt = tile->media_gt; + break; + } + } + } else { + media_gt = xe->tiles[0].primary_gt; + } + + if (!media_gt) + break; + + huc = &media_gt->uc.huc; + if (huc->fw.status == XE_UC_FIRMWARE_RUNNING) + version = &huc->fw.versions.found[XE_UC_FW_VER_RELEASE]; + break; + } + default: + return -EINVAL; + } + + if (version) { + resp.branch_ver = 0; + resp.major_ver = version->major; + resp.minor_ver = version->minor; + resp.patch_ver = version->patch; + } else { + return -ENODEV; + } + + if (copy_to_user(query_ptr, &resp, size)) + return -EFAULT; + + return 0; +} + +static size_t calc_oa_unit_query_size(struct xe_device *xe) +{ + size_t size = sizeof(struct drm_xe_query_oa_units); + struct xe_gt *gt; + int i, id; + + for_each_gt(gt, xe, id) { + for (i = 0; i < gt->oa.num_oa_units; i++) { + size += sizeof(struct drm_xe_oa_unit); + size += gt->oa.oa_unit[i].num_engines * + sizeof(struct drm_xe_engine_class_instance); + } + } + + return size; +} + +static int query_oa_units(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + void __user *query_ptr = u64_to_user_ptr(query->data); + size_t size = calc_oa_unit_query_size(xe); + struct drm_xe_query_oa_units *qoa; + enum xe_hw_engine_id hwe_id; + struct drm_xe_oa_unit *du; + struct xe_hw_engine *hwe; + struct xe_oa_unit *u; + int gt_id, i, j, ret; + struct xe_gt *gt; + u8 *pdu; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + qoa = kzalloc(size, GFP_KERNEL); + if (!qoa) + return -ENOMEM; + + pdu = (u8 *)&qoa->oa_units[0]; + for_each_gt(gt, xe, gt_id) { + for (i = 0; i < gt->oa.num_oa_units; i++) { + u = >->oa.oa_unit[i]; + du = (struct drm_xe_oa_unit *)pdu; + + du->oa_unit_id = u->oa_unit_id; + du->oa_unit_type = u->type; + du->oa_timestamp_freq = xe_oa_timestamp_frequency(gt); + du->capabilities = DRM_XE_OA_CAPS_BASE | DRM_XE_OA_CAPS_SYNCS | + DRM_XE_OA_CAPS_OA_BUFFER_SIZE | + DRM_XE_OA_CAPS_WAIT_NUM_REPORTS | + DRM_XE_OA_CAPS_OAM | + DRM_XE_OA_CAPS_OA_UNIT_GT_ID; + du->gt_id = u->gt->info.id; + j = 0; + for_each_hw_engine(hwe, gt, hwe_id) { + if (!xe_hw_engine_is_reserved(hwe) && + xe_oa_unit_id(hwe) == u->oa_unit_id) { + du->eci[j].engine_class = + xe_to_user_engine_class[hwe->class]; + du->eci[j].engine_instance = hwe->logical_instance; + du->eci[j].gt_id = gt->info.id; + j++; + } + } + du->num_engines = j; + pdu += sizeof(*du) + j * sizeof(du->eci[0]); + qoa->num_oa_units++; + } + } + + ret = copy_to_user(query_ptr, qoa, size); + kfree(qoa); + + return ret ? -EFAULT : 0; +} + +static int query_pxp_status(struct xe_device *xe, struct drm_xe_device_query *query) +{ + struct drm_xe_query_pxp_status __user *query_ptr = u64_to_user_ptr(query->data); + size_t size = sizeof(struct drm_xe_query_pxp_status); + struct drm_xe_query_pxp_status resp = { 0 }; + int ret; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + ret = xe_pxp_get_readiness_status(xe->pxp); + if (ret < 0) + return ret; + + resp.status = ret; + resp.supported_session_types = BIT(DRM_XE_PXP_TYPE_HWDRM); + + if (copy_to_user(query_ptr, &resp, size)) + return -EFAULT; + + return 0; +} + +static int query_eu_stall(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + void __user *query_ptr = u64_to_user_ptr(query->data); + struct drm_xe_query_eu_stall *info; + size_t size, array_size; + const u64 *rates; + u32 num_rates; + int ret; + + if (!xe_eu_stall_supported_on_platform(xe)) + return -ENODEV; + + array_size = xe_eu_stall_get_sampling_rates(&num_rates, &rates); + size = sizeof(struct drm_xe_query_eu_stall) + array_size; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + info = kzalloc(size, GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->num_sampling_rates = num_rates; + info->capabilities = DRM_XE_EU_STALL_CAPS_BASE; + info->record_size = xe_eu_stall_data_record_size(xe); + info->per_xecore_buf_size = xe_eu_stall_get_per_xecore_buf_size(); + memcpy(info->sampling_rates, rates, array_size); + + ret = copy_to_user(query_ptr, info, size); + kfree(info); + + return ret ? -EFAULT : 0; +} + +static int (* const xe_query_funcs[])(struct xe_device *xe, + struct drm_xe_device_query *query) = { + query_engines, + query_mem_regions, + query_config, + query_gt_list, + query_hwconfig, + query_gt_topology, + query_engine_cycles, + query_uc_fw_version, + query_oa_units, + query_pxp_status, + query_eu_stall, +}; + +int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct drm_xe_device_query *query = data; + u32 idx; + + if (XE_IOCTL_DBG(xe, query->extensions) || + XE_IOCTL_DBG(xe, query->reserved[0] || query->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, query->query >= ARRAY_SIZE(xe_query_funcs))) + return -EINVAL; + + idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs)); + if (XE_IOCTL_DBG(xe, !xe_query_funcs[idx])) + return -EINVAL; + + return xe_query_funcs[idx](xe, query); +} diff --git a/drivers/gpu/drm/xe/xe_query.h b/drivers/gpu/drm/xe/xe_query.h new file mode 100644 index 000000000..beeb7a819 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_query.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_QUERY_H_ +#define _XE_QUERY_H_ + +struct drm_device; +struct drm_file; + +int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_range_fence.c b/drivers/gpu/drm/xe/xe_range_fence.c new file mode 100644 index 000000000..3d8fa194a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_range_fence.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include "xe_macros.h" +#include "xe_range_fence.h" + +#define XE_RANGE_TREE_START(_node) ((_node)->start) +#define XE_RANGE_TREE_LAST(_node) ((_node)->last) + +INTERVAL_TREE_DEFINE(struct xe_range_fence, rb, u64, __subtree_last, + XE_RANGE_TREE_START, XE_RANGE_TREE_LAST, static, + xe_range_fence_tree); + +static void +xe_range_fence_signal_notify(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + struct xe_range_fence *rfence = container_of(cb, typeof(*rfence), cb); + struct xe_range_fence_tree *tree = rfence->tree; + + llist_add(&rfence->link, &tree->list); +} + +static bool __xe_range_fence_tree_cleanup(struct xe_range_fence_tree *tree) +{ + struct llist_node *node = llist_del_all(&tree->list); + struct xe_range_fence *rfence, *next; + + llist_for_each_entry_safe(rfence, next, node, link) { + xe_range_fence_tree_remove(rfence, &tree->root); + dma_fence_put(rfence->fence); + kfree(rfence); + } + + return !!node; +} + +/** + * xe_range_fence_insert() - range fence insert + * @tree: range fence tree to insert intoi + * @rfence: range fence + * @ops: range fence ops + * @start: start address of range fence + * @last: last address of range fence + * @fence: dma fence which signals range fence can be removed + freed + * + * Return: 0 on success, non-zero on failure + */ +int xe_range_fence_insert(struct xe_range_fence_tree *tree, + struct xe_range_fence *rfence, + const struct xe_range_fence_ops *ops, + u64 start, u64 last, struct dma_fence *fence) +{ + int err = 0; + + __xe_range_fence_tree_cleanup(tree); + + if (dma_fence_is_signaled(fence)) + goto free; + + rfence->ops = ops; + rfence->start = start; + rfence->last = last; + rfence->tree = tree; + rfence->fence = dma_fence_get(fence); + err = dma_fence_add_callback(fence, &rfence->cb, + xe_range_fence_signal_notify); + if (err == -ENOENT) { + dma_fence_put(fence); + err = 0; + goto free; + } else if (err == 0) { + xe_range_fence_tree_insert(rfence, &tree->root); + return 0; + } else { + dma_fence_put(fence); + } + +free: + if (ops->free) + ops->free(rfence); + + return err; +} + +static void xe_range_fence_tree_remove_all(struct xe_range_fence_tree *tree) +{ + struct xe_range_fence *rfence; + bool retry = true; + + rfence = xe_range_fence_tree_iter_first(&tree->root, 0, U64_MAX); + while (rfence) { + /* Should be ok with the minimalistic callback */ + if (dma_fence_remove_callback(rfence->fence, &rfence->cb)) + llist_add(&rfence->link, &tree->list); + rfence = xe_range_fence_tree_iter_next(rfence, 0, U64_MAX); + } + + while (retry) + retry = __xe_range_fence_tree_cleanup(tree); +} + +/** + * xe_range_fence_tree_init() - Init range fence tree + * @tree: range fence tree + */ +void xe_range_fence_tree_init(struct xe_range_fence_tree *tree) +{ + memset(tree, 0, sizeof(*tree)); +} + +/** + * xe_range_fence_tree_fini() - Fini range fence tree + * @tree: range fence tree + */ +void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree) +{ + xe_range_fence_tree_remove_all(tree); + XE_WARN_ON(!RB_EMPTY_ROOT(&tree->root.rb_root)); +} + +/** + * xe_range_fence_tree_first() - range fence tree iterator first + * @tree: range fence tree + * @start: start address of range fence + * @last: last address of range fence + * + * Return: first range fence found in range or NULL + */ +struct xe_range_fence * +xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start, + u64 last) +{ + return xe_range_fence_tree_iter_first(&tree->root, start, last); +} + +/** + * xe_range_fence_tree_next() - range fence tree iterator next + * @rfence: current range fence + * @start: start address of range fence + * @last: last address of range fence + * + * Return: next range fence found in range or NULL + */ +struct xe_range_fence * +xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last) +{ + return xe_range_fence_tree_iter_next(rfence, start, last); +} + +static void xe_range_fence_free(struct xe_range_fence *rfence) +{ + kfree(rfence); +} + +const struct xe_range_fence_ops xe_range_fence_kfree_ops = { + .free = xe_range_fence_free, +}; diff --git a/drivers/gpu/drm/xe/xe_range_fence.h b/drivers/gpu/drm/xe/xe_range_fence.h new file mode 100644 index 000000000..4934729dd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_range_fence.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_RANGE_FENCE_H_ +#define _XE_RANGE_FENCE_H_ + +#include +#include +#include + +struct xe_range_fence_tree; +struct xe_range_fence; + +/** struct xe_range_fence_ops - Xe range fence ops */ +struct xe_range_fence_ops { + /** @free: free range fence op */ + void (*free)(struct xe_range_fence *rfence); +}; + +/** struct xe_range_fence - Xe range fence (address conflict tracking) */ +struct xe_range_fence { + /** @rb: RB tree node inserted into interval tree */ + struct rb_node rb; + /** @start: start address of range fence is interval tree */ + u64 start; + /** @last: last address (inclusive) of range fence is interval tree */ + u64 last; + /** @__subtree_last: interval tree internal usage */ + u64 __subtree_last; + /** + * @fence: fence signals address in range fence no longer has conflict + */ + struct dma_fence *fence; + /** @tree: interval tree which range fence belongs to */ + struct xe_range_fence_tree *tree; + /** + * @cb: callback when fence signals to remove range fence free from interval tree + */ + struct dma_fence_cb cb; + /** @link: used to defer free of range fence to non-irq context */ + struct llist_node link; + /** @ops: range fence ops */ + const struct xe_range_fence_ops *ops; +}; + +/** struct xe_range_fence_tree - interval tree to store range fences */ +struct xe_range_fence_tree { + /** @root: interval tree root */ + struct rb_root_cached root; + /** @list: list of pending range fences to be freed */ + struct llist_head list; +}; + +extern const struct xe_range_fence_ops xe_range_fence_kfree_ops; + +struct xe_range_fence * +xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start, + u64 last); + +struct xe_range_fence * +xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last); + +void xe_range_fence_tree_init(struct xe_range_fence_tree *tree); + +void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree); + +int xe_range_fence_insert(struct xe_range_fence_tree *tree, + struct xe_range_fence *rfence, + const struct xe_range_fence_ops *ops, + u64 start, u64 end, + struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c new file mode 100644 index 000000000..d98ff9453 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_drm_ras.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_survivability_mode.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_event_types.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +#define CORE_COMPUTE_UNCORR_TYPE GENMASK(26, 25) +/* + * Uncorrectable error type for core compute errors. + * 0 - Correctable Error + * 1 - Local Uncorrectable Error + * 2 - Global Uncorrectable Error + * 3 - Informational Error + */ +#define GLOBAL_UNCORR_ERROR 2 + +/* Severity of detected errors */ +enum xe_ras_severity { + XE_RAS_SEV_NOT_SUPPORTED = 0, + XE_RAS_SEV_CORRECTABLE, + XE_RAS_SEV_UNCORRECTABLE, + XE_RAS_SEV_INFORMATIONAL, + XE_RAS_SEV_MAX +}; + +/* Major IP blocks/components where errors can originate */ +enum xe_ras_component { + XE_RAS_COMP_NOT_SUPPORTED = 0, + XE_RAS_COMP_DEVICE_MEMORY, + XE_RAS_COMP_CORE_COMPUTE, + XE_RAS_COMP_RESERVED, + XE_RAS_COMP_PCIE, + XE_RAS_COMP_FABRIC, + XE_RAS_COMP_SOC_INTERNAL, + XE_RAS_COMP_MAX +}; + +/* RAS response status codes */ +enum xe_ras_response_status { + XE_RAS_STATUS_SUCCESS = 0, + XE_RAS_STATUS_INVALID_PARAM, + XE_RAS_STATUS_OP_NOT_SUPPORTED, + XE_RAS_STATUS_TIMEOUT, + XE_RAS_STATUS_HARDWARE_FAILURE, + XE_RAS_STATUS_INSUFFICIENT_RESOURCES, + XE_RAS_STATUS_MAX +}; + +/* GPU health values */ +enum xe_ras_health { + XE_RAS_HEALTH_OK = 0, + XE_RAS_HEALTH_WARNING, + XE_RAS_HEALTH_CRITICAL, + XE_RAS_HEALTH_MAX +}; + +static const char *const xe_ras_severities[] = { + [XE_RAS_SEV_NOT_SUPPORTED] = "Not Supported", + [XE_RAS_SEV_CORRECTABLE] = "Correctable Error", + [XE_RAS_SEV_UNCORRECTABLE] = "Uncorrectable Error", + [XE_RAS_SEV_INFORMATIONAL] = "Informational Error", +}; +static_assert(ARRAY_SIZE(xe_ras_severities) == XE_RAS_SEV_MAX); + +static const char *const xe_ras_components[] = { + [XE_RAS_COMP_NOT_SUPPORTED] = "Not Supported", + [XE_RAS_COMP_DEVICE_MEMORY] = "Device Memory", + [XE_RAS_COMP_CORE_COMPUTE] = "Core Compute", + [XE_RAS_COMP_RESERVED] = "Reserved", + [XE_RAS_COMP_PCIE] = "PCIe", + [XE_RAS_COMP_FABRIC] = "Fabric", + [XE_RAS_COMP_SOC_INTERNAL] = "SoC Internal", +}; +static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX); + +static const char * const gpu_health_states[] = { + [XE_RAS_HEALTH_OK] = "ok", + [XE_RAS_HEALTH_WARNING] = "warning", + [XE_RAS_HEALTH_CRITICAL] = "critical", +}; +static_assert(ARRAY_SIZE(gpu_health_states) == XE_RAS_HEALTH_MAX); + +static u8 drm_to_xe_ras_severity(u8 severity) +{ + switch (severity) { + case DRM_XE_RAS_ERR_SEV_CORRECTABLE: + return XE_RAS_SEV_CORRECTABLE; + case DRM_XE_RAS_ERR_SEV_UNCORRECTABLE: + return XE_RAS_SEV_UNCORRECTABLE; + default: + return XE_RAS_SEV_NOT_SUPPORTED; + } +} + +static u8 drm_to_xe_ras_component(u8 component) +{ + switch (component) { + case DRM_XE_RAS_ERR_COMP_CORE_COMPUTE: + return XE_RAS_COMP_CORE_COMPUTE; + case DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: + return XE_RAS_COMP_SOC_INTERNAL; + case DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: + return XE_RAS_COMP_DEVICE_MEMORY; + case DRM_XE_RAS_ERR_COMP_PCIE: + return XE_RAS_COMP_PCIE; + case DRM_XE_RAS_ERR_COMP_FABRIC: + return XE_RAS_COMP_FABRIC; + default: + return XE_RAS_COMP_NOT_SUPPORTED; + } +} + +static int ras_status_to_errno(u32 status) +{ + switch (status) { + case XE_RAS_STATUS_SUCCESS: + return 0; + case XE_RAS_STATUS_INVALID_PARAM: + return -EINVAL; + case XE_RAS_STATUS_OP_NOT_SUPPORTED: + return -EOPNOTSUPP; + case XE_RAS_STATUS_TIMEOUT: + return -ETIMEDOUT; + case XE_RAS_STATUS_HARDWARE_FAILURE: + return -EIO; + case XE_RAS_STATUS_INSUFFICIENT_RESOURCES: + return -ENOSPC; + default: + return -EPROTO; + } +} + +static inline const char *sev_to_str(u8 severity) +{ + if (severity >= XE_RAS_SEV_MAX) + severity = XE_RAS_SEV_NOT_SUPPORTED; + + return xe_ras_severities[severity]; +} + +static inline const char *comp_to_str(u8 component) +{ + if (component >= XE_RAS_COMP_MAX) + component = XE_RAS_COMP_NOT_SUPPORTED; + + return xe_ras_components[component]; +} + +static struct pci_dev *find_usp_dev(struct pci_dev *pdev) +{ + struct pci_dev *vsp; + + /* + * Device Hierarchy: + * + * Upstream Switch Port (USP) --> Virtual Switch Port (VSP) --> SGunit (GPU endpoint) + */ + vsp = pci_upstream_bridge(pdev); + if (!vsp) + return NULL; + + return pci_upstream_bridge(vsp); +} + +static void ras_usp_aer_init(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct pci_dev *usp; + u16 aer_cap; + u32 status; + + usp = find_usp_dev(pdev); + if (!usp) + return; + + aer_cap = pci_find_ext_capability(usp, PCI_EXT_CAP_ID_ERR); + if (!aer_cap) { + dev_warn(&usp->dev, "AER capability unavailable\n"); + return; + } + + /* + * Clear any stale Uncorrectable Internal Error Status event in Uncorrectable Error + * Status Register. + */ + pci_read_config_dword(usp, aer_cap + PCI_ERR_UNCOR_STATUS, &status); + if (status & PCI_ERR_UNC_INTN) + pci_write_config_dword(usp, aer_cap + PCI_ERR_UNCOR_STATUS, PCI_ERR_UNC_INTN); + + /* + * All errors are steered to USP which is a PCIe AER Compliant device. + * Downgrade all the errors to non-fatal to prevent PCIe bus driver + * from triggering a Secondary Bus Reset (SBR). This allows error + * detection, containment and recovery in the driver. + * + * The Uncorrectable Error Severity Register has the 'Uncorrectable + * Internal Error Severity' set to fatal by default. Set this to + * non-fatal and unmask the error. + */ + + /* Downgrade Uncorrectable Internal Error to non-fatal */ + pci_clear_and_set_config_dword(usp, aer_cap + PCI_ERR_UNCOR_SEVER, PCI_ERR_UNC_INTN, 0); + + /* Unmask Uncorrectable Internal Error */ + pci_clear_and_set_config_dword(usp, aer_cap + PCI_ERR_UNCOR_MASK, PCI_ERR_UNC_INTN, 0); + + pci_save_state(usp); + dev_dbg(&usp->dev, "Uncorrectable Internal Errors downgraded and unmasked\n"); +} + +static u8 handle_core_compute_errors(struct xe_ras_error_array *arr) +{ + struct xe_ras_compute_error *error_info = (void *)arr->details; + u8 uncorr_type; + + uncorr_type = FIELD_GET(CORE_COMPUTE_UNCORR_TYPE, error_info->log_header); + + /* Request a reset if error is global */ + if (uncorr_type == GLOBAL_UNCORR_ERROR) + return XE_RAS_RECOVERY_ACTION_RESET; + + /* + * No action needed for other errors. + * Local errors are recovered using an engine reset by GuC. + */ + return XE_RAS_RECOVERY_ACTION_RECOVERED; +} + +static u8 handle_soc_internal_errors(struct xe_device *xe, struct xe_ras_error_array *arr) +{ + struct xe_ras_soc_error *info = (void *)arr->details; + struct xe_ras_soc_error_source *source = &info->source; + struct xe_ras_error_class *counter = &arr->counter; + + if (source->csc) { + struct xe_ras_csc_error *csc_error = (void *)info->details; + + /* + * CSC uncorrectable errors are classified as hardware errors and firmware errors. + * CSC firmware errors are critical errors that can be recovered only by firmware + * update via SPI driver. On a CSC firmware error, PCODE enables FDO mode and sets + * the bit in the capability register. On receiving this error, the driver enables + * runtime survivability mode which notifies userspace that a firmware update + * is required. + */ + if (csc_error->hec_fw_error) { + xe_err(xe, "[RAS]: CSC %s detected: 0x%x\n", + sev_to_str(counter->common.severity), + csc_error->hec_fw_error); + xe_survivability_mode_runtime_enable(xe); + return XE_RAS_RECOVERY_ACTION_DISCONNECT; + } + } else if (source->ieh) { + struct xe_ras_ieh_error *ieh_error = (void *)info->details; + + if (ieh_error->global_error_status & XE_RAS_SOC_IEH_PUNIT) { + xe_err(xe, "[RAS]: PUNIT %s detected: 0x%x\n", + sev_to_str(counter->common.severity), + ieh_error->global_error_status); + /* TODO: Add PUNIT error handling */ + return XE_RAS_RECOVERY_ACTION_DISCONNECT; + } + } + + /* For other SoC internal errors, request a reset as recovery mechanism */ + return XE_RAS_RECOVERY_ACTION_RESET; +} + +static u8 handle_device_memory_errors(struct xe_device *xe, struct xe_ras_error_array *arr) +{ + struct xe_ras_memory_error *info = (void *)arr->details; + + /* + * For memory errors, the recovery action depends on the error category + * + * TODO: Double-bit ECC errors: Page offlining + * Poison and data parity errors: Log only + * For any other memory errors, request a reset as recovery mechanism + */ + switch (info->category) { + case XE_RAS_MEMORY_POISON: + xe_info(xe, "[RAS]: Poison error detected\n"); + break; + case XE_RAS_MEMORY_DATA_PARITY: + xe_info(xe, "[RAS]: Data parity error detected\n"); + break; + case XE_RAS_MEMORY_DB_ECC: + xe_info(xe, "[RAS]: Double-bit ECC error detected at sw address 0x%llx\n", + info->sw_address); + /* TODO: Add page offlining for Double-bit ECC error */ + fallthrough; + default: + return XE_RAS_RECOVERY_ACTION_RESET; + } + + return XE_RAS_RECOVERY_ACTION_RECOVERED; +} + +void xe_ras_counter_threshold_crossed(struct xe_device *xe, + struct xe_sysctrl_event_response *response) +{ + struct xe_ras_threshold_crossed *pending = (void *)&response->data; + struct xe_ras_error_class *errors = pending->counters; + u32 id, ncounters = pending->ncounters; + + BUILD_BUG_ON(sizeof(response->data) < sizeof(*pending)); + xe_device_assert_mem_access(xe); + + if (!ncounters || ncounters > XE_RAS_NUM_COUNTERS) + xe_err(xe, "sysctrl: unexpected counter threshold crossed %u\n", ncounters); + else + xe_warn(xe, "[RAS]: counter threshold crossed, %u new errors\n", ncounters); + + for (id = 0; id < ncounters && id < XE_RAS_NUM_COUNTERS; id++) { + u8 severity, component; + + severity = errors[id].common.severity; + component = errors[id].common.component; + + xe_warn(xe, "[RAS]: %s %s detected\n", + comp_to_str(component), sev_to_str(severity)); + } +} + +static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value) +{ + struct xe_ras_get_counter_response response = {0}; + struct xe_ras_get_counter_request request = {0}; + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_error_common *common; + size_t rlen; + int ret; + + request.counter = *counter; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_COUNTER, + &request, sizeof(request), &response, sizeof(response)); + + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to get counter %d\n", ret); + return ret; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected get counter response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EIO; + } + + common = &response.counter.common; + *value = response.value; + + xe_dbg(xe, "[RAS]: get counter %u for %s %s\n", *value, comp_to_str(common->component), + sev_to_str(common->severity)); + + return 0; +} + +/** + * xe_ras_process_errors() - Process and contain hardware errors + * @xe: xe device instance + * + * Get error details from system controller and return recovery + * method. + * + * Returns: recovery action to be taken + */ +enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe) +{ + struct xe_sysctrl_mailbox_command command = {0}; + enum xe_ras_recovery_action final_action; + u32 remaining = XE_SYSCTRL_FLOOD_LIMIT; + struct xe_ras_get_soc_error response; + size_t rlen; + int ret; + + if (!xe->info.has_sysctrl) + return XE_RAS_RECOVERY_ACTION_RESET; + + /* Default action */ + final_action = XE_RAS_RECOVERY_ACTION_RECOVERED; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_SOC_ERROR, + NULL, 0, &response, sizeof(response)); + + do { + memset(&response, 0, sizeof(response)); + + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to get soc error %d\n", ret); + goto err; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected get soc error response length %zu (expected %zu)\n", + rlen, sizeof(response)); + goto err; + } + + /* Report if number of errors exceeds the maximum errors supported */ + if (response.num_errors > XE_RAS_NUM_ERROR_ARR) + xe_err(xe, "sysctrl: number of errors received %d out of bound (%d)\n", + response.num_errors, XE_RAS_NUM_ERROR_ARR); + + for (int i = 0; i < response.num_errors && i < XE_RAS_NUM_ERROR_ARR; i++) { + struct xe_ras_error_array *arr = &response.arr[i]; + enum xe_ras_recovery_action action; + u8 component, severity; + + component = arr->counter.common.component; + severity = arr->counter.common.severity; + + xe_info(xe, "[RAS]: %s %s detected\n", comp_to_str(component), + sev_to_str(severity)); + + switch (component) { + case XE_RAS_COMP_CORE_COMPUTE: + action = handle_core_compute_errors(arr); + break; + case XE_RAS_COMP_SOC_INTERNAL: + action = handle_soc_internal_errors(xe, arr); + break; + case XE_RAS_COMP_DEVICE_MEMORY: + action = handle_device_memory_errors(xe, arr); + break; + default: + /* For any other component, reset */ + action = XE_RAS_RECOVERY_ACTION_RESET; + break; + } + + /* Process and log all errors and then trigger highest recovery action */ + if (action > final_action) + final_action = action; + } + + /* Treat flooding as a system controller error */ + if (!--remaining) { + xe_err(xe, "[RAS]: sysctrl: get soc error response flooding\n"); + goto err; + } + + } while (response.additional_errors); + + return final_action; + +err: + return XE_RAS_RECOVERY_ACTION_RESET; +} + +/** + * xe_ras_get_counter() - Get error counter value + * @xe: Xe device instance + * @severity: Error severity to be queried (&enum drm_xe_ras_error_severity) + * @component: Error component to be queried (&enum drm_xe_ras_error_component) + * @value: Counter value + * + * This function retrieves the value of a specific error counter based on + * the error severity and component. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value) +{ + struct xe_ras_error_class counter = {0}; + + counter.common.severity = drm_to_xe_ras_severity(severity); + counter.common.component = drm_to_xe_ras_component(component); + + guard(xe_pm_runtime)(xe); + return get_counter(xe, &counter, value); +} + +/** + * xe_ras_clear_counter() - Clear error counter value + * @xe: Xe device instance + * @severity: Error severity to be cleared (&enum drm_xe_ras_error_severity) + * @component: Error component to be cleared (&enum drm_xe_ras_error_component) + * + * This function clears the value of a specific error counter based on + * the error severity and component. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_ras_clear_counter(struct xe_device *xe, u8 severity, u8 component) +{ + struct xe_ras_clear_counter_response response = {0}; + struct xe_ras_clear_counter_request request = {0}; + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_error_class *counter; + size_t rlen; + int ret; + + counter = &request.counter; + counter->common.severity = drm_to_xe_ras_severity(severity); + counter->common.component = drm_to_xe_ras_component(component); + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_CLEAR_COUNTER, + &request, sizeof(request), &response, sizeof(response)); + + guard(xe_pm_runtime)(xe); + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to clear counter %d\n", ret); + return ret; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected clear counter response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EIO; + } + + ret = ras_status_to_errno(response.status); + if (ret) { + xe_err(xe, "sysctrl: clear counter command failed with status %#x\n", + response.status); + return ret; + } + + counter = &response.counter; + + xe_dbg(xe, "[RAS]: clear counter for %s %s\n", comp_to_str(counter->common.component), + sev_to_str(counter->common.severity)); + + return 0; +} + +static ssize_t gpu_health_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct xe_ras_get_health_response response = {0}; + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_get_health_request request = {0}; + struct xe_device *xe = kdev_to_xe_device(dev); + const char *health; + size_t rlen; + int ret; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_HEALTH, + &request, sizeof(request), &response, sizeof(response)); + guard(xe_pm_runtime)(xe); + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to get health %d\n", ret); + return ret; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected get health response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EIO; + } + if (response.health >= XE_RAS_HEALTH_MAX) { + xe_err(xe, "sysctrl: invalid health state %u\n", + response.health); + return -EIO; + } + + health = gpu_health_states[response.health]; + + xe_dbg(xe, "[RAS]: get health: %s\n", health); + + return sysfs_emit(buf, "%s\n", health); +} + +static ssize_t gpu_health_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct xe_ras_set_health_response response = {0}; + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_set_health_request request = {0}; + struct xe_device *xe = kdev_to_xe_device(dev); + const char *health; + size_t rlen; + int state; + int ret; + + state = sysfs_match_string(gpu_health_states, buf); + if (state < 0) + return -EINVAL; + + request.health = state; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_SET_HEALTH, + &request, sizeof(request), &response, sizeof(response)); + guard(xe_pm_runtime)(xe); + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to set health %d\n", ret); + return ret; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected set health response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EIO; + } + + ret = ras_status_to_errno(response.status); + if (ret) { + xe_err(xe, "sysctrl: set health command failed with status %#x\n", + response.status); + return ret; + } + + if (response.health >= XE_RAS_HEALTH_MAX) { + xe_err(xe, "sysctrl: invalid health state %u\n", + response.health); + return -EIO; + } + + health = gpu_health_states[response.health]; + + xe_dbg(xe, "[RAS]: set health: %s\n", health); + + return count; +} +static DEVICE_ATTR_RW(gpu_health); + +static struct attribute *gpu_health_attrs[] = { + &dev_attr_gpu_health.attr, + NULL +}; + +/** + * DOC: GPU Health Indicator + * + * On Intel Xe platforms that support the gpu health indicator interface, + * the driver exposes this sysfs attribute for in-band access to the gpu + * health state:: + * + * /sys/bus/pci/devices//gpu_health + * + * Reading the attribute is available to all users and returns a single + * line containing the current gpu health state, whereas writing is + * restricted to administrative users and updates the state to one of the + * valid values. + * + * Management tools and administrators use this interface to query the + * current gpu health state (e.g. for telemetry/monitoring) and to + * update it - for example, to mark the gpu as ``warning`` or ``critical`` + * after diagnostics, or reset it back to ``ok`` once remediated. + * + * The valid values for the gpu health state are: + * + * - ``ok`` + * The gpu is healthy and operating within normal parameters. + * + * - ``warning`` + * The gpu is experiencing minor issues but remains operational. + * + * - ``critical`` + * The gpu is in a critical state and may not be operational. + * + * See Documentation/ABI/testing/sysfs-driver-intel-xe-ras for the ABI + * specification. + */ +static const struct attribute_group gpu_health_group = { + .attrs = gpu_health_attrs, +}; + +/** + * xe_ras_init - Initialize Xe RAS + * @xe: xe device instance + * + * Initialize Xe RAS + */ +void xe_ras_init(struct xe_device *xe) +{ + int ret; + + xe_drm_ras_init(xe); + + if (!xe->info.has_sysctrl) + return; + + if (IS_ENABLED(CONFIG_PCIEAER)) + ras_usp_aer_init(xe); + + ret = devm_device_add_group(xe->drm.dev, &gpu_health_group); + if (ret) + xe_err(xe, "Failed to create GPU health sysfs, err=%d\n", ret); +} diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h new file mode 100644 index 000000000..618364734 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RAS_H_ +#define _XE_RAS_H_ + +#include +#include "xe_ras_types.h" + +struct xe_device; +struct xe_sysctrl_event_response; + +void xe_ras_counter_threshold_crossed(struct xe_device *xe, + struct xe_sysctrl_event_response *response); +int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value); +int xe_ras_clear_counter(struct xe_device *xe, u8 severity, u8 component); +void xe_ras_init(struct xe_device *xe); +enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h new file mode 100644 index 000000000..99b2466e2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras_types.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RAS_TYPES_H_ +#define _XE_RAS_TYPES_H_ + +#include + +#define XE_RAS_NUM_COUNTERS 16 +#define XE_RAS_NUM_ERROR_ARR 3 +/* Error bits in IEH global error status register */ +#define XE_RAS_SOC_IEH_PUNIT BIT(1) +/* Device memory error categories */ +#define XE_RAS_MEMORY_DB_ECC BIT(1) +#define XE_RAS_MEMORY_POISON BIT(2) +#define XE_RAS_MEMORY_DATA_PARITY BIT(5) + +/** + * enum xe_ras_recovery_action - RAS recovery actions + * + * @XE_RAS_RECOVERY_ACTION_RECOVERED: Error recovered + * @XE_RAS_RECOVERY_ACTION_RESET: Requires reset + * @XE_RAS_RECOVERY_ACTION_DISCONNECT: Requires disconnect + * @XE_RAS_RECOVERY_ACTION_MAX: Max action value + * + * This enum defines the possible recovery actions that can be taken in response + * to RAS errors. + */ +enum xe_ras_recovery_action { + XE_RAS_RECOVERY_ACTION_RECOVERED = 0, + XE_RAS_RECOVERY_ACTION_RESET, + XE_RAS_RECOVERY_ACTION_DISCONNECT, + XE_RAS_RECOVERY_ACTION_MAX +}; + +/** + * struct xe_ras_error_common - Error fields that are common across all products + */ +struct xe_ras_error_common { + /** @severity: Error severity */ + u8 severity; + /** @component: IP block where error originated */ + u8 component; +} __packed; + +/** + * struct xe_ras_error_unit - Error unit information + */ +struct xe_ras_error_unit { + /** @tile: Tile identifier */ + u8 tile; + /** @instance: Instance identifier specific to IP */ + u32 instance; +} __packed; + +/** + * struct xe_ras_error_cause - Error cause information + */ +struct xe_ras_error_cause { + /** @cause: Cause/checker */ + u32 cause; + /** @reserved: For future use */ + u8 reserved; +} __packed; + +/** + * struct xe_ras_error_product - Error fields that are specific to the product + */ +struct xe_ras_error_product { + /** @unit: Unit within IP block */ + struct xe_ras_error_unit unit; + /** @cause: Cause/checker */ + struct xe_ras_error_cause cause; +} __packed; + +/** + * struct xe_ras_error_class - Combines common and product-specific parts + */ +struct xe_ras_error_class { + /** @common: Common error type and component */ + struct xe_ras_error_common common; + /** @product: Product-specific unit and cause */ + struct xe_ras_error_product product; +} __packed; + +/** + * struct xe_ras_threshold_crossed - Data for threshold crossed event + */ +struct xe_ras_threshold_crossed { + /** @ncounters: Number of error counters that crossed thresholds */ + u32 ncounters; + /** @counters: Array of error counters that crossed threshold */ + struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS]; +} __packed; + +/** + * struct xe_ras_get_counter_request - Request structure for get counter + */ +struct xe_ras_get_counter_request { + /** @counter: Error counter to be queried */ + struct xe_ras_error_class counter; + /** @reserved: Reserved for future use */ + u32 reserved; +} __packed; + +/** + * struct xe_ras_get_counter_response - Response structure for get counter + */ +struct xe_ras_get_counter_response { + /** @counter: Error counter that was queried */ + struct xe_ras_error_class counter; + /** @value: Current counter value */ + u32 value; + /** @timestamp: Timestamp when counter was last updated */ + u64 timestamp; + /** @threshold: Threshold value for the counter */ + u32 threshold; + /** @reserved: Reserved */ + u32 reserved[57]; +} __packed; + +/** + * struct xe_ras_clear_counter_request - Request structure for clear counter + */ +struct xe_ras_clear_counter_request { + /** @counter: Counter class to be cleared */ + struct xe_ras_error_class counter; + /** @reserved: Reserved for future use */ + u32 reserved; +} __packed; + +/** + * struct xe_ras_clear_counter_response - Response structure for clear counter + */ +struct xe_ras_clear_counter_response { + /** @counter: Counter class that was cleared */ + struct xe_ras_error_class counter; + /** @reserved: Reserved */ + u32 reserved; + /** @timestamp: Timestamp when the counter was cleared */ + u64 timestamp; + /** @status: Status of the clear operation */ + u32 status; + /** @reserved1: Reserved for future use */ + u32 reserved1[3]; +} __packed; + +/** + * struct xe_ras_error_array - Details of the error types + */ +struct xe_ras_error_array { + /** @value: Counter value of the detailed error */ + u32 value; + /** @counter: Error counter */ + struct xe_ras_error_class counter; + /** @timestamp: Timestamp */ + u64 timestamp; + /** @details: Error details specific to the counter */ + u32 details[XE_RAS_NUM_COUNTERS]; +} __packed; + +/** + * struct xe_ras_get_soc_error - Response from get soc error command + */ +struct xe_ras_get_soc_error { + /** @num_errors: Number of errors reported in this response */ + u8 num_errors; + /** @additional_errors: Indicates if the errors are pending */ + u8 additional_errors; + /** @arr: Array of up to 3 errors */ + struct xe_ras_error_array arr[XE_RAS_NUM_ERROR_ARR]; +} __packed; + +/** + * struct xe_ras_compute_error - Error details of Core Compute error + */ +struct xe_ras_compute_error { + /** @log_header: Error Source and type */ + u32 log_header; + /** @reserved: Reserved */ + u32 reserved[15]; +} __packed; + +/** + * struct xe_ras_soc_error_source - Source of SoC error + */ +struct xe_ras_soc_error_source { + /** @csc: CSC */ + u32 csc:1; + /** @ieh: IEH (Integrated Error Handler) */ + u32 ieh:1; + /** @reserved: Reserved for future use */ + u32 reserved:30; +} __packed; + +/** + * struct xe_ras_soc_error - Error details of SoC internal error + */ +struct xe_ras_soc_error { + /** @source: Error source */ + struct xe_ras_soc_error_source source; + /** @details: Error details specific to the error source */ + u32 details[15]; +} __packed; + +/** + * struct xe_ras_csc_error - CSC error details + */ +struct xe_ras_csc_error { + /** @reserved: Reserved for future use */ + u32 reserved; + /** @hec_fw_error: CSC firmware error */ + u32 hec_fw_error; +} __packed; + +/** + * struct xe_ras_ieh_error - IEH (Integrated Error Handler) error details + */ +struct xe_ras_ieh_error { + /** @reserved: Reserved for future use */ + u32 reserved; + /** @global_error_status: Global error status */ + u32 global_error_status; + /** @reserved1: Reserved for future use */ + u32 reserved1[2]; + /** @info: Additional information */ + u32 info[10]; +} __packed; + +/** + * struct xe_ras_memory_error - Device memory error details + */ +struct xe_ras_memory_error { + /** @category: Device memory error category */ + u8 category; + /** @reserved: Reserved for future use */ + u8 reserved[7]; + /** @reserved1: Reserved for future use */ + u64 reserved1; + /** @sw_address: Software address where error occurred */ + u64 sw_address; + /** @reserved2: Reserved for future use */ + u32 reserved2[10]; +} __packed; + +/** + * struct xe_ras_get_health_request - Request structure for obtaining gpu health + */ +struct xe_ras_get_health_request { + /** @reserved: Reserved for future use. */ + u32 reserved[2]; +} __packed; + +/** + * struct xe_ras_get_health_response - Response structure for obtaining gpu health + */ +struct xe_ras_get_health_response { + /** @health: gpu health value */ + u8 health; + /** @reserved: Reserved for future use */ + u8 reserved[3]; +} __packed; + +/** + * struct xe_ras_set_health_request - Request structure for setting gpu health + */ +struct xe_ras_set_health_request { + /** @health: gpu health value */ + u8 health; + /** @reserved: Reserved for future use */ + u8 reserved[3]; +} __packed; + +/** + * struct xe_ras_set_health_response - Response structure for setting gpu health + */ +struct xe_ras_set_health_response { + /** @status: Status of set health operation */ + u32 status; + /** @health: Resulting gpu health value */ + u8 health; + /** @reserved: Reserved for future use */ + u8 reserved[3]; + /** @reserved1: Reserved for future use */ + u32 reserved1[2]; +} __packed; +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c new file mode 100644 index 000000000..e328f5072 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_reg_sr.h" + +#include +#include +#include +#include + +#include +#include + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gt_mcr.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_hw_engine_types.h" +#include "xe_lrc.h" +#include "xe_mmio.h" +#include "xe_rtp_types.h" + +static void reg_sr_fini(struct drm_device *drm, void *arg) +{ + struct xe_reg_sr *sr = arg; + struct xe_reg_sr_entry *entry; + unsigned long reg; + + xa_for_each(&sr->xa, reg, entry) + kfree(entry); + + xa_destroy(&sr->xa); +} + +int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) +{ + xa_init(&sr->xa); + sr->name = name; + + return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); +} +EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); + +static bool compatible_entries(const struct xe_reg_sr_entry *e1, + const struct xe_reg_sr_entry *e2) +{ + /* + * Don't allow overwriting values: clr_bits/set_bits should be disjoint + * when operating in the same register + */ + if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || + e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) + return false; + + if (e1->reg.raw != e2->reg.raw) + return false; + + return true; +} + +static void reg_sr_inc_error(struct xe_reg_sr *sr) +{ +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + sr->errors++; +#endif +} + +static struct xe_reg sanitize_mcr(struct xe_reg_sr *sr, + const struct xe_reg_sr_entry *e, + struct xe_gt *gt) +{ + struct xe_reg reg = e->reg; + bool is_mcr; + + /* + * We need the gt structure to check MCR ranges. + */ + if (!gt) + return reg; + + is_mcr = xe_gt_mcr_check_reg(gt, reg); + + if (is_mcr && !reg.mcr) { + reg.mcr = 1; + xe_gt_notice(gt, "xe_reg_sr_entry using non-MCR register for address 0x%x, forcing MCR\n", + reg.addr); + reg_sr_inc_error(sr); + } + + if (!is_mcr && reg.mcr) { + reg.mcr = 0; + xe_gt_notice(gt, "xe_reg_sr_entry using MCR register for address 0x%x, forcing non-MCR\n", + reg.addr); + reg_sr_inc_error(sr); + } + + return reg; +} + +int xe_reg_sr_add(struct xe_reg_sr *sr, + const struct xe_reg_sr_entry *e, + struct xe_gt *gt) +{ + unsigned long idx = e->reg.addr; + struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); + struct xe_reg reg; + int ret; + + reg = sanitize_mcr(sr, e, gt); + + if (pentry) { + if (!compatible_entries(pentry, e)) { + ret = -EINVAL; + goto fail; + } + + pentry->clr_bits |= e->clr_bits; + pentry->set_bits |= e->set_bits; + pentry->read_mask |= e->read_mask; + + return 0; + } + + pentry = kmalloc_obj(*pentry); + if (!pentry) { + ret = -ENOMEM; + goto fail; + } + + *pentry = *e; + pentry->reg = reg; + ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); + if (ret) + goto fail_free; + + return 0; + +fail_free: + kfree(pentry); +fail: + xe_gt_err(gt, + "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", + idx, e->clr_bits, e->set_bits, + str_yes_no(e->reg.masked), + str_yes_no(e->reg.mcr), + ret); + reg_sr_inc_error(sr); + + return ret; +} + +/* + * Convert back from encoded value to type-safe, only to be used when reg.mcr + * is true + */ +static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) +{ + return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; +} + +static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) +{ + struct xe_reg reg = entry->reg; + struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); + u32 val; + + /* + * If this is a masked register, need to set the upper 16 bits. + * Set them to clr_bits since that is always a superset of the bits + * being modified. + * + * When it's not masked, we have to read it from hardware, unless we are + * supposed to set all bits. + */ + if (reg.masked) + val = entry->clr_bits << 16; + else if (entry->clr_bits + 1) + val = (reg.mcr ? + xe_gt_mcr_unicast_read_any(gt, reg_mcr) : + xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); + else + val = 0; + + /* + * TODO: add selftest to validate all tables, regardless of platform: + * - Masked registers can't have set_bits with upper bits set + * - set_bits must be contained in clr_bits + */ + val |= entry->set_bits; + + xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); + + if (entry->reg.mcr) + xe_gt_mcr_multicast_write(gt, reg_mcr, val); + else + xe_mmio_write32(>->mmio, reg, val); +} + +void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) +{ + struct xe_reg_sr_entry *entry; + unsigned long reg; + + if (xa_empty(&sr->xa)) + return; + + /* + * We don't process non-LRC reg_sr lists in VF, so they should have + * been empty in the check above. + */ + xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); + + xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); + return; + } + + xa_for_each(&sr->xa, reg, entry) + apply_one_mmio(gt, entry); +} + +/** + * xe_reg_sr_dump - print all save/restore entries + * @sr: Save/restore entries + * @p: DRM printer + */ +void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) +{ + struct xe_reg_sr_entry *entry; + unsigned long reg; + + if (!sr->name || xa_empty(&sr->xa)) + return; + + drm_printf(p, "%s\n", sr->name); + xa_for_each(&sr->xa, reg, entry) + drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", + reg, entry->clr_bits, entry->set_bits, + str_yes_no(entry->reg.masked), + str_yes_no(entry->reg.mcr)); +} + +static u32 readback_reg(struct xe_gt *gt, struct xe_reg reg) +{ + struct xe_reg_mcr mcr_reg = to_xe_reg_mcr(reg); + + if (reg.mcr) + return xe_gt_mcr_unicast_read_any(gt, mcr_reg); + else + return xe_mmio_read32(>->mmio, reg); +} + +/** + * xe_reg_sr_readback_check() - Readback registers referenced in save/restore + * entries and check whether the programming is in place. + * @sr: Save/restore entries + * @gt: GT to read register from + * @p: DRM printer to report discrepancies on + */ +void xe_reg_sr_readback_check(struct xe_reg_sr *sr, + struct xe_gt *gt, + struct drm_printer *p) +{ + struct xe_reg_sr_entry *entry; + unsigned long offset; + + xa_for_each(&sr->xa, offset, entry) { + u32 val = readback_reg(gt, entry->reg); + u32 mask = entry->clr_bits | entry->set_bits; + + if ((val & mask) != entry->set_bits) + drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n", + offset, mask, entry->set_bits, val & mask); + } +} + +/** + * xe_reg_sr_lrc_check() - Check LRC for registers referenced in save/restore + * entries and check whether the programming is in place. + * @sr: Save/restore entries + * @gt: GT to read register from + * @hwe: Hardware engine type to check LRC for + * @p: DRM printer to report discrepancies on + */ +void xe_reg_sr_lrc_check(struct xe_reg_sr *sr, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + struct drm_printer *p) +{ + struct xe_reg_sr_entry *entry; + unsigned long offset; + + xa_for_each(&sr->xa, offset, entry) { + u32 val; + int ret = xe_lrc_lookup_default_reg_value(gt, hwe->class, offset, &val); + u32 mask = entry->clr_bits | entry->set_bits; + + if (ret == -ENOENT) + drm_printf(p, "%#8lx :: not found in LRC for %s\n", offset, hwe->name); + else if ((val & mask) != entry->set_bits) + drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n", + offset, mask, entry->set_bits, val & mask); + } +} diff --git a/drivers/gpu/drm/xe/xe_reg_sr.h b/drivers/gpu/drm/xe/xe_reg_sr.h new file mode 100644 index 000000000..d26cf4713 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_sr.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_REG_SR_H_ +#define _XE_REG_SR_H_ + +/* + * Reg save/restore bookkeeping + */ + +struct xe_device; +struct xe_gt; +struct xe_hw_engine; +struct xe_reg_sr; +struct xe_reg_sr_entry; +struct drm_printer; + +int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe); +void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p); +void xe_reg_sr_readback_check(struct xe_reg_sr *sr, + struct xe_gt *gt, + struct drm_printer *p); +void xe_reg_sr_lrc_check(struct xe_reg_sr *sr, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + struct drm_printer *p); + +int xe_reg_sr_add(struct xe_reg_sr *sr, const struct xe_reg_sr_entry *e, + struct xe_gt *gt); +void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt); +void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_sr_types.h b/drivers/gpu/drm/xe/xe_reg_sr_types.h new file mode 100644 index 000000000..0a6695db2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_sr_types.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_REG_SR_TYPES_H_ +#define _XE_REG_SR_TYPES_H_ + +#include +#include + +#include "regs/xe_reg_defs.h" + +struct xe_reg_sr_entry { + struct xe_reg reg; + u32 clr_bits; + u32 set_bits; + /* Mask for bits to consider when reading value back */ + u32 read_mask; +}; + +struct xe_reg_sr { + struct xarray xa; + const char *name; + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + unsigned int errors; +#endif +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c new file mode 100644 index 000000000..cab1b578c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_reg_whitelist.h" + +#include + +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_oa_regs.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_types.h" +#include "xe_gt_printk.h" +#include "xe_platform_types.h" +#include "xe_reg_sr.h" +#include "xe_rtp.h" +#include "xe_step.h" + +#undef XE_REG_MCR +#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) + +static bool match_not_render(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return hwe->class != XE_ENGINE_CLASS_RENDER; +} + +static bool match_has_mert(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_device_has_mert((struct xe_device *)xe); +} + +static bool match_multi_queue_class(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_gt_supports_multi_queue(gt, hwe->class); +} + +VISIBLE_IF_KUNIT const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR( + { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST(PS_INVOCATION_COUNT, + RING_FORCE_TO_NONPRIV_ACCESS_RD | + RING_FORCE_TO_NONPRIV_RANGE_4)) + }, + { XE_RTP_NAME("1508744258, 14012131227, 1808121037"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST(COMMON_SLICE_CHICKEN1, 0)) + }, + { XE_RTP_NAME("1806527549"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST(HIZ_CHICKEN, 0)) + }, + { XE_RTP_NAME("allow_read_ctx_timestamp"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260), FUNC(match_not_render)), + XE_RTP_ACTIONS(WHITELIST(RING_CTX_TIMESTAMP(0), + RING_FORCE_TO_NONPRIV_ACCESS_RD, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("allow_read_queue_timestamp"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)), + XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0), + RING_FORCE_TO_NONPRIV_ACCESS_RD, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("16014440446"), + XE_RTP_RULES(PLATFORM(PVC)), + XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400), + RING_FORCE_TO_NONPRIV_DENY | + RING_FORCE_TO_NONPRIV_RANGE_64), + WHITELIST(XE_REG(0x4500), + RING_FORCE_TO_NONPRIV_DENY | + RING_FORCE_TO_NONPRIV_RANGE_64)) + }, + { XE_RTP_NAME("16017236439"), + XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COPY)), + XE_RTP_ACTIONS(WHITELIST(BCS_SWCTRL(0), + RING_FORCE_TO_NONPRIV_DENY, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("16020183090"), + XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0)) + }, + { XE_RTP_NAME("14024997852"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST(FF_MODE, + RING_FORCE_TO_NONPRIV_ACCESS_RW), + WHITELIST(VFLSKPD, + RING_FORCE_TO_NONPRIV_ACCESS_RW)) + }, + { XE_RTP_NAME("14024997852"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST(FF_MODE, + RING_FORCE_TO_NONPRIV_ACCESS_RW), + WHITELIST(VFLSKPD, + RING_FORCE_TO_NONPRIV_ACCESS_RW)) + }, +); +EXPORT_SYMBOL_IF_KUNIT(register_whitelist); + +static const struct xe_rtp_table_sr oa_whitelist = XE_RTP_TABLE_SR( + +#define WHITELIST_DENY(r, f) WHITELIST(r, (f) | RING_FORCE_TO_NONPRIV_DENY) + +#define WHITELIST_OA_MMIO_TRG(trg, status, head) \ + WHITELIST_DENY(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \ + WHITELIST_DENY(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \ + WHITELIST_DENY(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4) + +#define WHITELIST_OAG_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR) + +#define WHITELIST_OAM_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \ + OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \ + OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \ + WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \ + OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \ + OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \ + WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \ + OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \ + OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ)) + +#define WHITELIST_OA_MERT_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_TAIL_POINTER) + + { XE_RTP_NAME("oag_mmio_trg_rcs"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) + }, + { XE_RTP_NAME("oag_mmio_trg_ccs"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(COMPUTE)), + XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) + }, + { XE_RTP_NAME("oam_mmio_trg_vcs"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) + }, + { XE_RTP_NAME("oam_mmio_trg_vecs"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(VIDEO_ENHANCE)), + XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) + }, + { XE_RTP_NAME("oa_mert_mmio_trg_ccs"), + XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)), + XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) + }, + { XE_RTP_NAME("oa_mert_mmio_trg_bcs"), + XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)), + XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) + }, +); + +static int whitelist_apply_to_hwe(struct xe_hw_engine *hwe, struct xe_reg_sr *in, + struct xe_reg_sr *out, int first_slot) +{ + struct xe_reg_sr *sr = in; + struct xe_reg_sr_entry *entry; + struct drm_printer p; + unsigned long reg; + unsigned int slot; + + xe_gt_dbg(hwe->gt, "Add %s whitelist to engine\n", sr->name); + p = xe_gt_dbg_printer(hwe->gt); + + slot = first_slot; + xa_for_each(&sr->xa, reg, entry) { + struct xe_reg_sr_entry hwe_entry = { + .reg = RING_FORCE_TO_NONPRIV(hwe->mmio_base, slot), + .set_bits = entry->reg.addr | entry->set_bits, + .clr_bits = ~0u, + .read_mask = entry->read_mask, + }; + + if (slot == RING_MAX_NONPRIV_SLOTS) { + xe_gt_err(hwe->gt, + "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n", + hwe->name, RING_MAX_NONPRIV_SLOTS); + break; + } + + xe_reg_whitelist_print_entry(&p, 0, reg, entry); + xe_reg_sr_add(out, &hwe_entry, hwe->gt); + + slot++; + } + + return slot; +} + +/** + * xe_reg_whitelist_process_engine - process table of registers to whitelist + * @hwe: engine instance to process whitelist for + * + * Process wwhitelist table for this platform, saving in @hwe all the + * registers that need to be whitelisted by the hardware so they can be accessed + * by userspace. + */ +void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + int first_oa_slot; + + xe_rtp_process_to_sr(&ctx, ®ister_whitelist, &hwe->reg_whitelist, false); + first_oa_slot = whitelist_apply_to_hwe(hwe, &hwe->reg_whitelist, &hwe->reg_sr, 0); + + xe_rtp_process_to_sr(&ctx, &oa_whitelist, &hwe->oa_whitelist, false); + + /* + * Save oa nonpriv registers to hwe->oa_sr, from which oa registers are whitelisted + * or de-whitelisted, by toggling the 'deny' bit on oa stream open/close + */ + whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->oa_sr, first_oa_slot); + + /* + * Also save oa nonpriv registers to hwe->reg_sr, to ensure oa registers are not + * whitelisted by default after probe, gt reset, resume and engine reset + */ + whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->reg_sr, first_oa_slot); +} + +static void __whitelist_oa_regs(struct xe_hw_engine *hwe, bool whitelist) +{ + struct xe_reg_sr_entry *entry; + unsigned long reg; + + xa_for_each(&hwe->oa_sr.xa, reg, entry) { + if (whitelist) + entry->set_bits &= ~RING_FORCE_TO_NONPRIV_DENY; + else + entry->set_bits |= RING_FORCE_TO_NONPRIV_DENY; + } + + xe_reg_sr_apply_mmio(&hwe->oa_sr, hwe->gt); +} + +/** + * xe_reg_whitelist_oa_regs - whitelist oa registers for gt + * @gt: gt to whitelist oa registers for + * + * Whitelist OA registers by resetting RING_FORCE_TO_NONPRIV_DENY + */ +void xe_reg_whitelist_oa_regs(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + lockdep_assert_held(>->oa.gt_lock); + if (gt->oa.whitelist_count++) + return; + + for_each_hw_engine(hwe, gt, id) + __whitelist_oa_regs(hwe, true); +} + +/** + * xe_reg_dewhitelist_oa_regs - dewhitelist oa registers for gt + * @gt: gt to dewhitelist oa registers for + * + * Dewhitelist OA registers by setting RING_FORCE_TO_NONPRIV_DENY + */ +void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + lockdep_assert_held(>->oa.gt_lock); + xe_assert(gt_to_xe(gt), gt->oa.whitelist_count); + if (--gt->oa.whitelist_count) + return; + + for_each_hw_engine(hwe, gt, id) + __whitelist_oa_regs(hwe, false); +} + +/** + * xe_reg_whitelist_print_entry - print one whitelist entry + * @p: DRM printer + * @indent: indent level + * @reg: register allowed/denied + * @entry: save-restore entry + * + * Print details about the entry added to allow/deny access + */ +void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, + u32 reg, struct xe_reg_sr_entry *entry) +{ + u32 val = entry->set_bits; + const char *access_str = "(invalid)"; + unsigned int range_bit = 2; + u32 range_start, range_end; + bool deny; + + deny = val & RING_FORCE_TO_NONPRIV_DENY; + + switch (val & RING_FORCE_TO_NONPRIV_RANGE_MASK) { + case RING_FORCE_TO_NONPRIV_RANGE_4: + range_bit = 4; + break; + case RING_FORCE_TO_NONPRIV_RANGE_16: + range_bit = 6; + break; + case RING_FORCE_TO_NONPRIV_RANGE_64: + range_bit = 8; + break; + } + + range_start = reg & REG_GENMASK(25, range_bit); + range_end = range_start | REG_GENMASK(range_bit - 1, 0); + + switch (val & RING_FORCE_TO_NONPRIV_ACCESS_MASK) { + case RING_FORCE_TO_NONPRIV_ACCESS_RW: + access_str = "rw"; + break; + case RING_FORCE_TO_NONPRIV_ACCESS_RD: + access_str = "read"; + break; + case RING_FORCE_TO_NONPRIV_ACCESS_WR: + access_str = "write"; + break; + } + + drm_printf_indent(p, indent, "REG[0x%x-0x%x]: %s %s access\n", + range_start, range_end, + deny ? "deny" : "allow", + access_str); +} + +/** + * xe_reg_whitelist_dump - print all whitelist entries + * @sr: Save/restore entries + * @p: DRM printer + */ +void xe_reg_whitelist_dump(struct xe_reg_sr *sr, struct drm_printer *p) +{ + struct xe_reg_sr_entry *entry; + unsigned long reg; + + if (!sr->name || xa_empty(&sr->xa)) + return; + + drm_printf(p, "%s\n", sr->name); + xa_for_each(&sr->xa, reg, entry) + xe_reg_whitelist_print_entry(p, 1, reg, entry); +} diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h new file mode 100644 index 000000000..c0248063d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_REG_WHITELIST_H_ +#define _XE_REG_WHITELIST_H_ + +#include + +struct drm_printer; +struct xe_gt; +struct xe_hw_engine; +struct xe_reg_sr; +struct xe_reg_sr_entry; + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +extern const struct xe_rtp_table_sr register_whitelist; +#endif + +void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe); + +void xe_reg_whitelist_oa_regs(struct xe_gt *gt); +void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt); + +void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, + u32 reg, struct xe_reg_sr_entry *entry); + +void xe_reg_whitelist_dump(struct xe_reg_sr *sr, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h new file mode 100644 index 000000000..0522caafd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -0,0 +1,368 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _XE_RES_CURSOR_H_ +#define _XE_RES_CURSOR_H_ + +#include + +#include +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_macros.h" +#include "xe_svm.h" +#include "xe_ttm_vram_mgr.h" + +/** + * struct xe_res_cursor - state for walking over dma mapping, vram_mgr, + * stolen_mgr, and gtt_mgr allocations + */ +struct xe_res_cursor { + /** @start: Start of cursor */ + u64 start; + /** @size: Size of the current segment. */ + u64 size; + /** @remaining: Remaining bytes in cursor */ + u64 remaining; + /** @node: Opaque point current node cursor */ + void *node; + /** @mem_type: Memory type */ + u32 mem_type; + /** @sgl: Scatterlist for cursor */ + struct scatterlist *sgl; + /** @dma_addr: Current element in a struct drm_pagemap_addr array */ + const struct drm_pagemap_addr *dma_addr; + /** @mm: Buddy allocator for VRAM cursor */ + struct gpu_buddy *mm; + /** + * @dma_start: DMA start address for the current segment. + * This may be different to @dma_addr.addr since elements in + * the array may be coalesced to a single segment. + */ + u64 dma_start; + /** @dma_seg_size: Size of the current DMA segment. */ + u64 dma_seg_size; +}; + +static struct gpu_buddy *xe_res_get_buddy(struct ttm_resource *res) +{ + struct ttm_resource_manager *mgr; + + mgr = ttm_manager_type(res->bo->bdev, res->mem_type); + return &to_xe_ttm_vram_mgr(mgr)->mm; +} + +/** + * xe_res_first - initialize a xe_res_cursor + * + * @res: TTM resource object to walk + * @start: Start of the range + * @size: Size of the range + * @cur: cursor object to initialize + * + * Start walking over the range of allocations between @start and @size. + */ +static inline void xe_res_first(struct ttm_resource *res, + u64 start, u64 size, + struct xe_res_cursor *cur) +{ + cur->sgl = NULL; + cur->dma_addr = NULL; + if (!res) + goto fallback; + + XE_WARN_ON(start + size > res->size); + + cur->mem_type = res->mem_type; + + switch (cur->mem_type) { + case XE_PL_STOLEN: { + /* res->start is in pages (ttm_range_manager). */ + cur->start = (res->start << PAGE_SHIFT) + start; + cur->size = size; + cur->remaining = size; + cur->node = NULL; + cur->mm = NULL; + break; + } + case XE_PL_VRAM0: + case XE_PL_VRAM1: { + struct gpu_buddy_block *block; + struct list_head *head, *next; + struct gpu_buddy *mm = xe_res_get_buddy(res); + + head = &to_xe_ttm_vram_mgr_resource(res)->blocks; + + block = list_first_entry_or_null(head, + struct gpu_buddy_block, + link); + if (!block) + goto fallback; + + while (start >= gpu_buddy_block_size(mm, block)) { + start -= gpu_buddy_block_size(mm, block); + + next = block->link.next; + if (next != head) + block = list_entry(next, struct gpu_buddy_block, + link); + } + + cur->mm = mm; + cur->start = gpu_buddy_block_offset(block) + start; + cur->size = min(gpu_buddy_block_size(mm, block) - start, + size); + cur->remaining = size; + cur->node = block; + break; + } + default: + goto fallback; + } + + return; + +fallback: + cur->start = start; + cur->size = size; + cur->remaining = size; + cur->node = NULL; + cur->mem_type = XE_PL_TT; + XE_WARN_ON(res && start + size > res->size); +} + +static inline void __xe_res_sg_next(struct xe_res_cursor *cur) +{ + struct scatterlist *sgl = cur->sgl; + u64 start = cur->start; + + while (start >= sg_dma_len(sgl)) { + start -= sg_dma_len(sgl); + sgl = sg_next(sgl); + XE_WARN_ON(!sgl); + } + + cur->start = start; + cur->size = sg_dma_len(sgl) - start; + cur->sgl = sgl; +} + +/** + * __xe_res_dma_next() - Advance the cursor when end-of-segment is reached + * @cur: The cursor + */ +static inline void __xe_res_dma_next(struct xe_res_cursor *cur) +{ + const struct drm_pagemap_addr *addr = cur->dma_addr; + u64 start = cur->start; + + while (start >= cur->dma_seg_size) { + start -= cur->dma_seg_size; + addr++; + cur->dma_seg_size = PAGE_SIZE << addr->order; + } + cur->dma_start = addr->addr; + + /* Coalesce array_elements */ + while (cur->dma_seg_size - start < cur->remaining) { + if (cur->dma_start + cur->dma_seg_size != addr[1].addr || + addr->proto != addr[1].proto) + break; + addr++; + cur->dma_seg_size += PAGE_SIZE << addr->order; + } + + cur->dma_addr = addr; + cur->start = start; + cur->size = cur->dma_seg_size - start; +} + +/** + * xe_res_first_sg - initialize a xe_res_cursor with a scatter gather table + * + * @sg: scatter gather table to walk + * @start: Start of the range + * @size: Size of the range + * @cur: cursor object to initialize + * + * Start walking over the range of allocations between @start and @size. + */ +static inline void xe_res_first_sg(const struct sg_table *sg, + u64 start, u64 size, + struct xe_res_cursor *cur) +{ + XE_WARN_ON(!sg); + cur->node = NULL; + cur->start = start; + cur->remaining = size; + cur->size = 0; + cur->dma_addr = NULL; + cur->sgl = sg->sgl; + cur->mem_type = XE_PL_TT; + __xe_res_sg_next(cur); +} + +/** + * xe_res_first_dma - initialize a xe_res_cursor with dma_addr array + * + * @dma_addr: struct drm_pagemap_addr array to walk + * @start: Start of the range + * @size: Size of the range + * @cur: cursor object to initialize + * + * Start walking over the range of allocations between @start and @size. + */ +static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr, + u64 start, u64 size, + struct xe_res_cursor *cur) +{ + XE_WARN_ON(!dma_addr); + XE_WARN_ON(!IS_ALIGNED(start, PAGE_SIZE) || + !IS_ALIGNED(size, PAGE_SIZE)); + + cur->node = NULL; + cur->start = start; + cur->remaining = size; + cur->dma_seg_size = PAGE_SIZE << dma_addr->order; + cur->dma_start = 0; + cur->size = 0; + cur->dma_addr = dma_addr; + __xe_res_dma_next(cur); + cur->sgl = NULL; + cur->mem_type = XE_PL_TT; +} + +/** + * xe_res_next - advance the cursor + * + * @cur: the cursor to advance + * @size: number of bytes to move forward + * + * Move the cursor @size bytes forwrad, walking to the next node if necessary. + */ +static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) +{ + struct gpu_buddy_block *block; + struct list_head *next; + u64 start; + + XE_WARN_ON(size > cur->remaining); + + cur->remaining -= size; + if (!cur->remaining) + return; + + if (cur->size > size) { + cur->size -= size; + cur->start += size; + return; + } + + if (cur->dma_addr) { + cur->start += size; + __xe_res_dma_next(cur); + return; + } + + if (cur->sgl) { + cur->start += size; + __xe_res_sg_next(cur); + return; + } + + switch (cur->mem_type) { + case XE_PL_STOLEN: + /* Just advance within the contiguous region. */ + cur->start += size; + cur->size = cur->remaining; + break; + case XE_PL_VRAM0: + case XE_PL_VRAM1: + start = size - cur->size; + block = cur->node; + + next = block->link.next; + block = list_entry(next, struct gpu_buddy_block, link); + + + while (start >= gpu_buddy_block_size(cur->mm, block)) { + start -= gpu_buddy_block_size(cur->mm, block); + + next = block->link.next; + block = list_entry(next, struct gpu_buddy_block, link); + } + + cur->start = gpu_buddy_block_offset(block) + start; + cur->size = min(gpu_buddy_block_size(cur->mm, block) - start, + cur->remaining); + cur->node = block; + break; + default: + return; + } +} + +/** + * xe_res_dma - return dma address of cursor at current position + * + * @cur: the cursor to return the dma address from + */ +static inline u64 xe_res_dma(const struct xe_res_cursor *cur) +{ + if (cur->dma_addr) + return cur->dma_start + cur->start; + else if (cur->sgl) + return sg_dma_address(cur->sgl) + cur->start; + else + return cur->start; +} + +/** + * xe_res_is_vram() - Whether the cursor current dma address points to + * same-device VRAM + * @cur: The cursor. + * + * Return: true iff the address returned by xe_res_dma() points to internal vram. + */ +static inline bool xe_res_is_vram(const struct xe_res_cursor *cur) +{ + if (cur->dma_addr) + return cur->dma_addr->proto == XE_INTERCONNECT_VRAM; + + switch (cur->mem_type) { + case XE_PL_STOLEN: + case XE_PL_VRAM0: + case XE_PL_VRAM1: + return true; + default: + break; + } + + return false; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c new file mode 100644 index 000000000..39a670e91 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -0,0 +1,652 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_ring_ops.h" + +#include + +#include "instructions/xe_gpu_commands.h" +#include "instructions/xe_mi_commands.h" +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "xe_exec_queue.h" +#include "xe_gt_types.h" +#include "xe_lrc.h" +#include "xe_sched_job.h" +#include "xe_sriov.h" +#include "xe_vm_types.h" +#include "xe_vm.h" +#include "xe_wa.h" + +/* + * 3D-related flags that can't be set on _engines_ that lack access to the 3D + * pipeline (i.e., CCS engines). + */ +#define PIPE_CONTROL_3D_ENGINE_FLAGS (\ + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ + PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ + PIPE_CONTROL_TILE_CACHE_FLUSH | \ + PIPE_CONTROL_DEPTH_STALL | \ + PIPE_CONTROL_STALL_AT_SCOREBOARD | \ + PIPE_CONTROL_PSD_SYNC | \ + PIPE_CONTROL_AMFS_FLUSH | \ + PIPE_CONTROL_VF_CACHE_INVALIDATE | \ + PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) + +/* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ +#define PIPE_CONTROL_3D_ARCH_FLAGS ( \ + PIPE_CONTROL_3D_ENGINE_FLAGS | \ + PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ + PIPE_CONTROL_FLUSH_ENABLE | \ + PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ + PIPE_CONTROL_DC_FLUSH_ENABLE) + +static u32 preparser_disable(bool state) +{ + return MI_ARB_CHECK | BIT(8) | state; +} + +static u32 * +__emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset) +{ + *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | + MI_LRI_MMIO_REMAP_EN; + *cmd++ = reg.addr + adj_offset; + *cmd++ = AUX_INV; + *cmd++ = MI_SEMAPHORE_WAIT_TOKEN | MI_SEMAPHORE_REGISTER_POLL | + MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_EQ_SDD; + *cmd++ = 0; + *cmd++ = reg.addr + adj_offset; + *cmd++ = 0; + *cmd++ = 0; + + return cmd; +} + +static u32 *emit_aux_table_inv_render_compute(struct xe_gt *gt, u32 *cmd) +{ + return __emit_aux_table_inv(cmd, CCS_AUX_INV, gt->mmio.adj_offset); +} + +static u32 *emit_aux_table_inv_video_decode(struct xe_gt *gt, u32 *cmd) +{ + return __emit_aux_table_inv(cmd, VD0_AUX_INV, gt->mmio.adj_offset); +} + +static u32 *emit_aux_table_inv_video_enhance(struct xe_gt *gt, u32 *cmd) +{ + return __emit_aux_table_inv(cmd, VE0_AUX_INV, gt->mmio.adj_offset); +} + +static int emit_aux_table_inv(struct xe_hw_engine *hwe, u32 *dw, int i) +{ + struct xe_gt *gt = hwe->gt; + u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = + gt->ring_ops[hwe->class]->emit_aux_table_inv; + + if (emit) + return emit(gt, dw + i) - dw; + else + return i; +} + +static int emit_user_interrupt(u32 *dw, int i) +{ + dw[i++] = MI_USER_INTERRUPT; + dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; + dw[i++] = MI_ARB_CHECK; + + return i; +} + +static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) +{ + dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); + dw[i++] = addr; + dw[i++] = 0; + dw[i++] = value; + + return i; +} + +static int emit_flush_dw(u32 *dw, int i) +{ + dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW; + dw[i++] = 0; + dw[i++] = 0; + dw[i++] = 0; + + return i; +} + +static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i) +{ + dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | + flags; + dw[i++] = addr | MI_FLUSH_DW_USE_GTT; + dw[i++] = 0; + dw[i++] = value; + + return i; +} + +static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) +{ + dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); + dw[i++] = lower_32_bits(batch_addr); + dw[i++] = upper_32_bits(batch_addr); + + return i; +} + +static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i) +{ + dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | + MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0; + + dw[i++] = addr | MI_FLUSH_DW_USE_GTT; + dw[i++] = 0; + dw[i++] = val; + + return i; +} + +static int +emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value) +{ + dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; + dw[i++] = bit_group_1; + dw[i++] = offset; + dw[i++] = 0; + dw[i++] = value; + dw[i++] = 0; + + return i; +} + +static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags, + bool invalidate_tlb, u32 *dw, int i) +{ + u32 flags0 = 0; + u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | + PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | + PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | + PIPE_CONTROL_VF_CACHE_INVALIDATE | + PIPE_CONTROL_CONST_CACHE_INVALIDATE | + PIPE_CONTROL_STATE_CACHE_INVALIDATE | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_STORE_DATA_INDEX; + + if (invalidate_tlb) + flags1 |= PIPE_CONTROL_TLB_INVALIDATE; + + if (xe_exec_queue_is_multi_queue(q)) + flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; + else + flags1 |= PIPE_CONTROL_CS_STALL; + + flags1 &= ~mask_flags; + + if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE) + flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE; + + return emit_pipe_control(dw, i, flags0, flags1, + LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); +} + +static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, + u32 *dw, int i) +{ + dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); + dw[i++] = lower_32_bits(addr); + dw[i++] = upper_32_bits(addr); + dw[i++] = lower_32_bits(value); + dw[i++] = upper_32_bits(value); + + return i; +} + +static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) +{ + struct xe_exec_queue *q = job->q; + struct xe_gt *gt = q->gt; + bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); + u32 flags0, flags1; + + if (XE_GT_WA(gt, 14016712196)) + i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, + LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); + + flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; + flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_DC_FLUSH_ENABLE | + PIPE_CONTROL_FLUSH_ENABLE); + + if (XE_GT_WA(gt, 1409600907)) + flags1 |= PIPE_CONTROL_DEPTH_STALL; + + if (lacks_render) + flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS; + else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) + flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; + + if (xe_exec_queue_is_multi_queue(q)) + flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; + else + flags1 |= PIPE_CONTROL_CS_STALL; + + return emit_pipe_control(dw, i, flags0, flags1, 0, 0); +} + +static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value, + bool stall_only, u32 *dw, int i) +{ + u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; + + if (!stall_only) + flags1 |= PIPE_CONTROL_FLUSH_ENABLE; + + if (xe_exec_queue_is_multi_queue(q)) + flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; + else + flags1 |= PIPE_CONTROL_CS_STALL; + + return emit_pipe_control(dw, i, flags0, flags1, addr, value); +} + +static u32 get_ppgtt_flag(struct xe_sched_job *job) +{ + if (job->q->vm && !job->ggtt) + return BIT(8); + + return 0; +} + +static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, + u32 *dw, int i) +{ + const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ? + RING_QUEUE_TIMESTAMP(0) : + RING_CTX_TIMESTAMP(0); + + dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; + dw[i++] = reg.addr; + dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); + dw[i++] = 0; + + /* + * Ensure CTX timestamp >= Job timestamp during VF sampling to avoid + * arithmetic wraparound in TDR. + */ + if (IS_SRIOV_VF(xe)) { + dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | + MI_SRM_ADD_CS_OFFSET; + dw[i++] = reg.addr; + dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); + dw[i++] = 0; + } + + return i; +} + +static int emit_fake_watchdog(struct xe_lrc *lrc, u32 *dw, int i) +{ + /* + * Setup a watchdog with impossible condition to always trigger an + * hardware interrupt that would force the GuC to reset the engine. + */ + + dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) | MI_LRI_LRM_CS_MMIO; + dw[i++] = PR_CTR_THRSH(0).addr; + dw[i++] = 2; /* small threshold */ + dw[i++] = PR_CTR_CTRL(0).addr; + dw[i++] = CTR_LOGIC_OP(START); + + dw[i++] = MI_SEMAPHORE_WAIT | MI_SEMW_GGTT | MI_SEMW_POLL | MI_SEMW_COMPARE(SAD_EQ_SDD); + dw[i++] = 0xdead; /* this should never be seen */ + dw[i++] = lower_32_bits(xe_lrc_ggtt_addr(lrc)); + dw[i++] = upper_32_bits(xe_lrc_ggtt_addr(lrc)); + dw[i++] = 0; /* unused token */ + + dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_LRM_CS_MMIO; + dw[i++] = PR_CTR_CTRL(0).addr; + dw[i++] = CTR_LOGIC_OP(STOP); + + return i; +} + +/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ +static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, + u64 batch_addr, u32 *head, u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + u32 ppgtt_flag = get_ppgtt_flag(job); + struct xe_gt *gt = job->q->gt; + + *head = lrc->ring.tail; + + if (job->ring_ops_force_reset) + i = emit_fake_watchdog(lrc, dw, i); + + i = emit_copy_timestamp(gt_to_xe(gt), lrc, dw, i); + + if (job->ring_ops_flush_tlb) { + dw[i++] = preparser_disable(true); + i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, MI_INVALIDATE_TLB, dw, i); + dw[i++] = preparser_disable(false); + } else { + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + } + + i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + + if (job->user_fence.used) { + i = emit_flush_dw(dw, i); + i = emit_store_imm_ppgtt_posted(job->user_fence.addr, + job->user_fence.value, + dw, i); + } + + i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); + + i = emit_user_interrupt(dw, i); + + xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static bool has_aux_ccs(struct xe_device *xe) +{ + /* + * PVC is a special case that has no compression of either type + * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 + * onward, so any future platforms with no FlatCCS will not have + * AuxCCS, and we explicitly do not want to support it on MTL. + */ + if (GRAPHICS_VERx100(xe) >= 1270 || xe->info.platform == XE_PVC) + return false; + + return !xe->info.has_flat_ccs; +} + +static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, + u64 batch_addr, u32 *head, u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + u32 ppgtt_flag = get_ppgtt_flag(job); + struct xe_gt *gt = job->q->gt; + struct xe_device *xe = gt_to_xe(gt); + + *head = lrc->ring.tail; + + if (job->ring_ops_force_reset) + i = emit_fake_watchdog(lrc, dw, i); + + i = emit_copy_timestamp(xe, lrc, dw, i); + + dw[i++] = preparser_disable(true); + + /* hsdes: 1809175790 */ + i = emit_aux_table_inv(job->q->hwe, dw, i); + + if (job->ring_ops_flush_tlb) + i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, MI_INVALIDATE_TLB, dw, i); + + dw[i++] = preparser_disable(false); + + if (!job->ring_ops_flush_tlb) + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + + i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + + if (job->user_fence.used) { + i = emit_flush_dw(dw, i); + i = emit_store_imm_ppgtt_posted(job->user_fence.addr, + job->user_fence.value, + dw, i); + } + + i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); + + i = emit_user_interrupt(dw, i); + + xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static void __emit_job_gen12_render_compute(struct xe_sched_job *job, + struct xe_lrc *lrc, + u64 batch_addr, u32 *head, + u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + u32 ppgtt_flag = get_ppgtt_flag(job); + struct xe_gt *gt = job->q->gt; + struct xe_device *xe = gt_to_xe(gt); + bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); + u32 mask_flags = 0; + + *head = lrc->ring.tail; + + if (job->ring_ops_force_reset) + i = emit_fake_watchdog(lrc, dw, i); + + i = emit_copy_timestamp(xe, lrc, dw, i); + + /* + * On AuxCCS platforms the invalidation of the Aux table requires + * quiescing the memory traffic beforehand. + */ + if (has_aux_ccs(xe)) + i = emit_render_cache_flush(job, dw, i); + + dw[i++] = preparser_disable(true); + if (lacks_render) + mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; + else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) + mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; + + /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ + i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i); + + /* hsdes: 1809175790 */ + i = emit_aux_table_inv(job->q->hwe, dw, i); + + dw[i++] = preparser_disable(false); + + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + + i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + + i = emit_render_cache_flush(job, dw, i); + + if (job->user_fence.used) + i = emit_store_imm_ppgtt_posted(job->user_fence.addr, + job->user_fence.value, + dw, i); + + i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); + + i = emit_user_interrupt(dw, i); + + xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static void emit_migration_job_gen12(struct xe_sched_job *job, + struct xe_lrc *lrc, u32 *head, + u32 seqno) +{ + struct xe_gt *gt = job->q->gt; + struct xe_device *xe = gt_to_xe(gt); + u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc); + u32 dw[MAX_JOB_SIZE_DW], i = 0; + + *head = lrc->ring.tail; + + xe_gt_assert(gt, !job->ring_ops_force_reset); + + i = emit_copy_timestamp(xe, lrc, dw, i); + + i = emit_store_imm_ggtt(saddr, seqno, dw, i); + + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ + + i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); + + dw[i++] = preparser_disable(true); + i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i); + dw[i++] = preparser_disable(false); + + i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); + + i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, + job->migrate_flush_flags, + dw, i); + + i = emit_user_interrupt(dw, i); + + xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static void emit_job_gen12_gsc(struct xe_sched_job *job) +{ + struct xe_gt *gt = job->q->gt; + + xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ + + __emit_job_gen12_simple(job, job->q->lrc[0], + job->ptrs[0].batch_addr, + &job->ptrs[0].head, + xe_sched_job_lrc_seqno(job)); +} + +static void emit_job_gen12_copy(struct xe_sched_job *job) +{ + int i; + + if (xe_sched_job_is_migration(job->q)) { + emit_migration_job_gen12(job, job->q->lrc[0], + &job->ptrs[0].head, + xe_sched_job_lrc_seqno(job)); + return; + } + + for (i = 0; i < job->q->width; ++i) + __emit_job_gen12_simple(job, job->q->lrc[i], + job->ptrs[i].batch_addr, + &job->ptrs[i].head, + xe_sched_job_lrc_seqno(job)); +} + +static void emit_job_gen12_video(struct xe_sched_job *job) +{ + int i; + + /* FIXME: Not doing parallel handshake for now */ + for (i = 0; i < job->q->width; ++i) + __emit_job_gen12_video(job, job->q->lrc[i], + job->ptrs[i].batch_addr, + &job->ptrs[i].head, + xe_sched_job_lrc_seqno(job)); +} + +static void emit_job_gen12_render_compute(struct xe_sched_job *job) +{ + int i; + + for (i = 0; i < job->q->width; ++i) + __emit_job_gen12_render_compute(job, job->q->lrc[i], + job->ptrs[i].batch_addr, + &job->ptrs[i].head, + xe_sched_job_lrc_seqno(job)); +} + +static const struct xe_ring_ops ring_ops_gen12_gsc = { + .emit_job = emit_job_gen12_gsc, +}; + +static const struct xe_ring_ops ring_ops_gen12_copy = { + .emit_job = emit_job_gen12_copy, +}; + +static const struct xe_ring_ops ring_ops_gen12_video_decode = { + .emit_job = emit_job_gen12_video, +}; + +static const struct xe_ring_ops ring_ops_gen12_video_enhance = { + .emit_job = emit_job_gen12_video, +}; + +static const struct xe_ring_ops ring_ops_gen12_render_compute = { + .emit_job = emit_job_gen12_render_compute, +}; + +static const struct xe_ring_ops auxccs_ring_ops_gen12_video_decode = { + .emit_job = emit_job_gen12_video, + .emit_aux_table_inv = emit_aux_table_inv_video_decode, +}; + +static const struct xe_ring_ops auxccs_ring_ops_gen12_video_enhance = { + .emit_job = emit_job_gen12_video, + .emit_aux_table_inv = emit_aux_table_inv_video_enhance, +}; + +static const struct xe_ring_ops auxccs_ring_ops_gen12_render_compute = { + .emit_job = emit_job_gen12_render_compute, + .emit_aux_table_inv = emit_aux_table_inv_render_compute, +}; + +const struct xe_ring_ops * +xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) +{ + struct xe_device *xe = gt_to_xe(gt); + + switch (class) { + case XE_ENGINE_CLASS_OTHER: + return &ring_ops_gen12_gsc; + case XE_ENGINE_CLASS_COPY: + return &ring_ops_gen12_copy; + case XE_ENGINE_CLASS_VIDEO_DECODE: + if (has_aux_ccs(xe)) + return &auxccs_ring_ops_gen12_video_decode; + else + return &ring_ops_gen12_video_decode; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + if (has_aux_ccs(xe)) + return &auxccs_ring_ops_gen12_video_enhance; + else + return &ring_ops_gen12_video_enhance; + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + if (has_aux_ccs(xe)) + return &auxccs_ring_ops_gen12_render_compute; + else + return &ring_ops_gen12_render_compute; + default: + return NULL; + } +} diff --git a/drivers/gpu/drm/xe/xe_ring_ops.h b/drivers/gpu/drm/xe/xe_ring_ops.h new file mode 100644 index 000000000..e942735d7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ring_ops.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RING_OPS_H_ +#define _XE_RING_OPS_H_ + +#include "xe_hw_engine_types.h" +#include "xe_ring_ops_types.h" + +struct xe_gt; + +const struct xe_ring_ops * +xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops_types.h b/drivers/gpu/drm/xe/xe_ring_ops_types.h new file mode 100644 index 000000000..52ff96bc4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ring_ops_types.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RING_OPS_TYPES_H_ +#define _XE_RING_OPS_TYPES_H_ + +#include + +struct xe_gt; +struct xe_sched_job; + +#define MAX_JOB_SIZE_DW 74 +#define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4) + +/** + * struct xe_ring_ops - Ring operations + */ +struct xe_ring_ops { + /** @emit_job: Write job to ring */ + void (*emit_job)(struct xe_sched_job *job); + + /** @emit_aux_table_inv: Emit aux table invalidation to the ring */ + u32 *(*emit_aux_table_inv)(struct xe_gt *gt, u32 *cmd); +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c new file mode 100644 index 000000000..6a8d6ea68 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_rtp.h" + +#include + +#include + +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_topology.h" +#include "xe_reg_sr.h" +#include "xe_sriov.h" + +/** + * DOC: Register Table Processing + * + * Internal infrastructure to define how registers should be updated based on + * rules and actions. This can be used to define tables with multiple entries + * (one per register) that will be walked over at some point in time to apply + * the values to the registers that have matching rules. + */ + +static bool has_samedia(const struct xe_device *xe) +{ + return xe->info.media_verx100 >= 1300; +} + +struct rule_match_ctx { + const struct xe_device *xe; + struct xe_gt *gt; + struct xe_hw_engine *hwe; + const struct xe_rtp_rule *rules; + const unsigned int n_rules; + unsigned int head; + int err; +}; + +static bool rule_is_item(const struct xe_rtp_rule *r) +{ + return r->match_type != XE_RTP_MATCH_OR; +} + +static bool rule_match_item(struct rule_match_ctx *match_ctx) +{ + const struct xe_device *xe = match_ctx->xe; + struct xe_gt *gt = match_ctx->gt; + struct xe_hw_engine *hwe = match_ctx->hwe; + const struct xe_rtp_rule *r = &match_ctx->rules[match_ctx->head]; + + switch (r->match_type) { + case XE_RTP_MATCH_PLATFORM: + return xe->info.platform == r->platform; + case XE_RTP_MATCH_SUBPLATFORM: + return xe->info.platform == r->platform && + xe->info.subplatform == r->subplatform; + case XE_RTP_MATCH_PLATFORM_STEP: + if (drm_WARN_ON(&xe->drm, xe->info.step.platform == STEP_NONE)) + return false; + + return xe->info.step.platform >= r->step_start && + xe->info.step.platform < r->step_end; + case XE_RTP_MATCH_GRAPHICS_VERSION: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.graphics_verx100 == r->ver_start && + (!has_samedia(xe) || !xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_GRAPHICS_VERSION_RANGE: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.graphics_verx100 >= r->ver_start && + xe->info.graphics_verx100 <= r->ver_end && + (!has_samedia(xe) || !xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.graphics_verx100 == r->ver_start; + case XE_RTP_MATCH_GRAPHICS_STEP: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.step.graphics >= r->step_start && + xe->info.step.graphics < r->step_end && + (!has_samedia(xe) || !xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_VERSION: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.media_verx100 == r->ver_start && + (!has_samedia(xe) || xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_VERSION_RANGE: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.media_verx100 >= r->ver_start && + xe->info.media_verx100 <= r->ver_end && + (!has_samedia(xe) || xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_STEP: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.step.media >= r->step_start && + xe->info.step.media < r->step_end && + (!has_samedia(xe) || xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_VERSION_ANY_GT: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.media_verx100 == r->ver_start; + case XE_RTP_MATCH_INTEGRATED: + return !xe->info.is_dgfx; + case XE_RTP_MATCH_DISCRETE: + return xe->info.is_dgfx; + case XE_RTP_MATCH_ENGINE_CLASS: + if (drm_WARN_ON(&xe->drm, !hwe)) + return false; + + return hwe->class == r->engine_class; + case XE_RTP_MATCH_NOT_ENGINE_CLASS: + if (drm_WARN_ON(&xe->drm, !hwe)) + return false; + + return hwe->class != r->engine_class; + case XE_RTP_MATCH_FUNC: + return r->match_func(xe, gt, hwe); + default: + drm_warn(&xe->drm, "Invalid RTP match %u\n", + r->match_type); + return false; + } +} + +/* + * Match a conjunctive set of rules (rules joined by an implicit "AND"). + * + * Once one item evaluates to false, the remaining items are not evaluated + * anymore. Nevetheless, all rules are consumed to allow detecting syntax + * errors. + */ +static bool rule_match_and(struct rule_match_ctx *match_ctx, bool parse_only) +{ + bool match = true; + unsigned int count = 0; + + while (match_ctx->head < match_ctx->n_rules && + rule_is_item(&match_ctx->rules[match_ctx->head])) { + if (!parse_only) + match = rule_match_item(match_ctx); + + if (!match) + parse_only = true; + + match_ctx->head++; + count++; + } + + if (drm_WARN_ON(&match_ctx->xe->drm, !count)) { + match_ctx->err = -EINVAL; + + if (!parse_only) + match = false; + } + + return match; +} + +/* + * Match a disjunctive set of rules (subset of rules joined by + * "XE_RTP_MATCH_OR"). + * + * Once one subset evaluates to true, the remaining items are not evaluated + * anymore. Nevetheless, all rules are consumed to allow detecting syntax + * errors. + */ +static bool rule_match_or(struct rule_match_ctx *match_ctx) +{ + bool match = rule_match_and(match_ctx, false); + + while (match_ctx->head < match_ctx->n_rules && + match_ctx->rules[match_ctx->head].match_type == XE_RTP_MATCH_OR) { + /* Consume XE_RTP_MATCH_OR. */ + match_ctx->head++; + + match = rule_match_and(match_ctx, match); + } + + return match; +} + +static bool rule_matches_with_err(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules, + int *err) +{ + struct rule_match_ctx match_ctx = { + .xe = xe, + .gt = gt, + .hwe = hwe, + .rules = rules, + .n_rules = n_rules, + }; + bool match = rule_match_or(&match_ctx); + + if (err) + *err = match_ctx.err; + + return match; +} + +static bool rule_matches(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules) +{ + return rule_matches_with_err(xe, gt, hwe, rules, n_rules, NULL); +} + +static void rtp_add_sr_entry(const struct xe_rtp_action *action, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + u32 mmio_base, + struct xe_reg_sr *sr) +{ + struct xe_reg_sr_entry sr_entry = { + .reg = action->reg, + .clr_bits = action->clr_bits, + .read_mask = action->read_mask, + }; + + if (action->use_func) + sr_entry.set_bits = action->set_func(gt, hwe); + else + sr_entry.set_bits = action->set_bits; + + sr_entry.reg.addr += mmio_base; + + xe_reg_sr_add(sr, &sr_entry, gt); +} + +static bool rtp_process_one_sr(const struct xe_rtp_entry_sr *entry, + struct xe_device *xe, struct xe_gt *gt, + struct xe_hw_engine *hwe, struct xe_reg_sr *sr) +{ + const struct xe_rtp_action *action; + u32 mmio_base; + unsigned int i; + + if (!rule_matches(xe, gt, hwe, entry->rules, entry->n_rules)) + return false; + + for (i = 0, action = &entry->actions[0]; i < entry->n_actions; action++, i++) { + if ((entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE) || + (action->flags & XE_RTP_ACTION_FLAG_ENGINE_BASE)) + mmio_base = hwe->mmio_base; + else + mmio_base = 0; + + rtp_add_sr_entry(action, gt, hwe, mmio_base, sr); + } + + return true; +} + +static void rtp_get_context(struct xe_rtp_process_ctx *ctx, + struct xe_hw_engine **hwe, + struct xe_gt **gt, + struct xe_device **xe) +{ + switch (ctx->type) { + case XE_RTP_PROCESS_TYPE_DEVICE: + *hwe = NULL; + *gt = NULL; + *xe = ctx->xe; + break; + case XE_RTP_PROCESS_TYPE_GT: + *hwe = NULL; + *gt = ctx->gt; + *xe = gt_to_xe(*gt); + break; + case XE_RTP_PROCESS_TYPE_ENGINE: + *hwe = ctx->hwe; + *gt = (*hwe)->gt; + *xe = gt_to_xe(*gt); + break; + } +} + +/** + * xe_rtp_process_ctx_enable_active_tracking - Enable tracking of active entries + * + * Set additional metadata to track what entries are considered "active", i.e. + * their rules match the condition. Bits are never cleared: entries with + * matching rules set the corresponding bit in the bitmap. + * + * @ctx: The context for processing the table + * @active_entries: bitmap to store the active entries + * @n_entries: number of entries to be processed + */ +void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx, + unsigned long *active_entries, + size_t n_entries) +{ + ctx->active_entries = active_entries; + ctx->n_entries = n_entries; +} +EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_ctx_enable_active_tracking); + +static void rtp_mark_active(struct xe_device *xe, + struct xe_rtp_process_ctx *ctx, + unsigned int idx) +{ + if (!ctx->active_entries) + return; + + if (drm_WARN_ON(&xe->drm, idx >= ctx->n_entries)) + return; + + bitmap_set(ctx->active_entries, idx, 1); +} + +/** + * xe_rtp_process_to_sr - Process all rtp @entries, adding the matching ones to + * the save-restore argument. + * @ctx: The context for processing the table, with one of device, gt or hwe + * @table: Table with RTP definitions + * @sr: Save-restore struct where matching rules execute the action. This can be + * viewed as the "coalesced view" of multiple the tables. The bits for each + * register set are expected not to collide with previously added entries + * @process_in_vf: Whether this RTP table should get processed for SR-IOV VF + * devices. Should generally only be 'true' for LRC tables. + * + * Walk the table pointed by @entries (with an empty sentinel) and add all + * entries with matching rules to @sr. If @hwe is not NULL, its mmio_base is + * used to calculate the right register offset + */ +void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, + const struct xe_rtp_table_sr *table, + struct xe_reg_sr *sr, + bool process_in_vf) +{ + struct xe_hw_engine *hwe = NULL; + struct xe_gt *gt = NULL; + struct xe_device *xe = NULL; + + rtp_get_context(ctx, &hwe, >, &xe); + + if (!process_in_vf && IS_SRIOV_VF(xe)) + return; + + xe_assert(xe, table->entries); + + for (size_t i = 0; i < table->n_entries; i++) { + const struct xe_rtp_entry_sr *entry = &table->entries[i]; + bool match = false; + + if (entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE) { + struct xe_hw_engine *each_hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(each_hwe, gt, id) + match |= rtp_process_one_sr(entry, xe, gt, + each_hwe, sr); + } else { + match = rtp_process_one_sr(entry, xe, gt, hwe, sr); + } + + if (match) + rtp_mark_active(xe, ctx, i); + } +} +EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_to_sr); + +/** + * xe_rtp_process - Process all entries in rtp @table, without running any action + * @ctx: The context for processing the table, with one of device, gt or hwe + * @table: Table with RTP definitions + * + * Walk the table pointed by @table, executing the + * rules. One difference from xe_rtp_process_to_sr(): there is no action + * associated with each entry since this uses struct xe_rtp_entry. Its main use + * is for marking active workarounds via + * xe_rtp_process_ctx_enable_active_tracking(). + */ +void xe_rtp_process(struct xe_rtp_process_ctx *ctx, + const struct xe_rtp_table *table) +{ + struct xe_hw_engine *hwe; + struct xe_gt *gt; + struct xe_device *xe; + + rtp_get_context(ctx, &hwe, >, &xe); + + xe_assert(xe, table->entries); + + for (size_t i = 0; i < table->n_entries; i++) { + const struct xe_rtp_entry *entry = &table->entries[i]; + + if (!rule_matches(xe, gt, hwe, entry->rules, entry->n_rules)) + continue; + + rtp_mark_active(xe, ctx, i); + } +} +EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process); + +bool xe_rtp_match_always(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return true; +} + +bool xe_rtp_match_even_instance(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return hwe->instance % 2 == 0; +} + +bool xe_rtp_match_first_render_or_compute(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + u64 render_compute_mask = gt->info.engine_mask & + (XE_HW_ENGINE_CCS_MASK | XE_HW_ENGINE_RCS_MASK); + + return render_compute_mask && + hwe->engine_id == __ffs(render_compute_mask); +} + +bool xe_rtp_match_not_sriov_vf(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return !IS_SRIOV_VF(xe); +} + +bool xe_rtp_match_psmi_enabled(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev)); +} + +bool xe_rtp_match_gt_has_discontiguous_dss_groups(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_gt_has_discontiguous_dss_groups(gt); +} + +bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe->info.has_flat_ccs; +} + +bool xe_rtp_match_has_msix(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_device_has_msix(xe); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_rtp.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h new file mode 100644 index 000000000..0032f68ea --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp.h @@ -0,0 +1,609 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RTP_H_ +#define _XE_RTP_H_ + +#include +#include + +#define _XE_RTP_INCLUDE_PRIVATE_HELPERS + +#include "xe_rtp_helpers.h" +#include "xe_rtp_types.h" + +#undef _XE_RTP_INCLUDE_PRIVATE_HELPERS + +/* + * Register table poke infrastructure + */ + +struct xe_hw_engine; +struct xe_gt; +struct xe_reg_sr; + +/* + * Macros to encode rules to match against platform, IP version, stepping, etc. + * Shouldn't be used directly - see XE_RTP_RULES() + */ +#define _XE_RTP_RULE_PLATFORM(plat__) \ + { .match_type = XE_RTP_MATCH_PLATFORM, .platform = plat__ } + +#define _XE_RTP_RULE_SUBPLATFORM(plat__, sub__) \ + { .match_type = XE_RTP_MATCH_SUBPLATFORM, \ + .platform = plat__, .subplatform = sub__ } + +#define _XE_RTP_RULE_PLATFORM_STEP(start__, end__) \ + { .match_type = XE_RTP_MATCH_PLATFORM_STEP, \ + .step_start = start__, .step_end = end__ } + +#define _XE_RTP_RULE_GRAPHICS_STEP(start__, end__) \ + { .match_type = XE_RTP_MATCH_GRAPHICS_STEP, \ + .step_start = start__, .step_end = end__ } + +#define _XE_RTP_RULE_MEDIA_STEP(start__, end__) \ + { .match_type = XE_RTP_MATCH_MEDIA_STEP, \ + .step_start = start__, .step_end = end__ } + +#define _XE_RTP_RULE_ENGINE_CLASS(cls__) \ + { .match_type = XE_RTP_MATCH_ENGINE_CLASS, \ + .engine_class = (cls__) } + +/** + * XE_RTP_RULE_PLATFORM - Create rule matching platform + * @plat_: platform to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_PLATFORM(plat_) \ + _XE_RTP_RULE_PLATFORM(XE_##plat_) + +/** + * XE_RTP_RULE_SUBPLATFORM - Create rule matching platform and sub-platform + * @plat_: platform to match + * @sub_: sub-platform to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_SUBPLATFORM(plat_, sub_) \ + _XE_RTP_RULE_SUBPLATFORM(XE_##plat_, XE_SUBPLATFORM_##plat_##_##sub_) + +/** + * XE_RTP_RULE_PLATFORM_STEP - Create rule matching platform-level stepping + * @start_: First stepping matching the rule + * @end_: First stepping that does not match the rule + * + * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive + * on the left, exclusive on the right. + * + * You need to make sure that proper support for reading platform-level stepping + * information is present for the target platform before using this rule. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_PLATFORM_STEP(start_, end_) \ + _XE_RTP_RULE_PLATFORM_STEP(STEP_##start_, STEP_##end_) + +/** + * XE_RTP_RULE_GRAPHICS_STEP - Create rule matching graphics stepping + * @start_: First stepping matching the rule + * @end_: First stepping that does not match the rule + * + * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive + * on the left, exclusive on the right. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_GRAPHICS_STEP(start_, end_) \ + _XE_RTP_RULE_GRAPHICS_STEP(STEP_##start_, STEP_##end_) + +/** + * XE_RTP_RULE_MEDIA_STEP - Create rule matching media stepping + * @start_: First stepping matching the rule + * @end_: First stepping that does not match the rule + * + * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive + * on the left, exclusive on the right. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_MEDIA_STEP(start_, end_) \ + _XE_RTP_RULE_MEDIA_STEP(STEP_##start_, STEP_##end_) + +/** + * XE_RTP_RULE_ENGINE_CLASS - Create rule matching an engine class + * @cls_: Engine class to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_ENGINE_CLASS(cls_) \ + _XE_RTP_RULE_ENGINE_CLASS(XE_ENGINE_CLASS_##cls_) + +/** + * XE_RTP_RULE_FUNC - Create rule using callback function for match + * @func__: Function to call to decide if rule matches + * + * This allows more complex checks to be performed. The ``XE_RTP`` + * infrastructure will simply call the function @func_ passed to decide if this + * rule matches the device. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_FUNC(func__) \ + { .match_type = XE_RTP_MATCH_FUNC, \ + .match_func = (func__) } + +/** + * XE_RTP_RULE_GRAPHICS_VERSION - Create rule matching graphics version + * @ver__: Graphics IP version to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_GRAPHICS_VERSION(ver__) \ + { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION, \ + .ver_start = ver__, } + +/** + * XE_RTP_RULE_GRAPHICS_VERSION_RANGE - Create rule matching a range of graphics version + * @ver_start__: First graphics IP version to match + * @ver_end__: Last graphics IP version to match + * + * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e. + * inclusive on both sides + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_GRAPHICS_VERSION_RANGE(ver_start__, ver_end__) \ + { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_RANGE, \ + .ver_start = ver_start__, .ver_end = ver_end__, } + +/** + * XE_RTP_RULE_GRAPHICS_VERSION_ANY_GT - Create rule matching graphics version on any GT + * @ver__: Graphics IP version to match + * + * Like XE_RTP_RULE_GRAPHICS_VERSION, but it matches even if the current GT + * being checked is not of the graphics type. It allows to add RTP entries to + * another GT when the device contains a Graphics IP with that version. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_GRAPHICS_VERSION_ANY_GT(ver__) \ + { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT, \ + .ver_start = ver__, } + +/** + * XE_RTP_RULE_MEDIA_VERSION - Create rule matching media version + * @ver__: Media IP version to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_MEDIA_VERSION(ver__) \ + { .match_type = XE_RTP_MATCH_MEDIA_VERSION, \ + .ver_start = ver__, } + +/** + * XE_RTP_RULE_MEDIA_VERSION_RANGE - Create rule matching a range of media version + * @ver_start__: First media IP version to match + * @ver_end__: Last media IP version to match + * + * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e. + * inclusive on both sides + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_MEDIA_VERSION_RANGE(ver_start__, ver_end__) \ + { .match_type = XE_RTP_MATCH_MEDIA_VERSION_RANGE, \ + .ver_start = ver_start__, .ver_end = ver_end__, } + +/** + * XE_RTP_RULE_MEDIA_VERSION_ANY_GT - Create rule matching media version on any GT + * @ver__: Media IP version to match + * + * Like XE_RTP_RULE_MEDIA_VERSION, but it matches even if the current GT being + * checked is not of the media type. It allows to add RTP entries to another + * GT when the device contains a Media IP with that version. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_MEDIA_VERSION_ANY_GT(ver__) \ + { .match_type = XE_RTP_MATCH_MEDIA_VERSION_ANY_GT, \ + .ver_start = ver__, } + +/** + * XE_RTP_RULE_IS_INTEGRATED - Create a rule matching integrated graphics devices + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_IS_INTEGRATED \ + { .match_type = XE_RTP_MATCH_INTEGRATED } + +/** + * XE_RTP_RULE_IS_DISCRETE - Create a rule matching discrete graphics devices + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_IS_DISCRETE \ + { .match_type = XE_RTP_MATCH_DISCRETE } + +/** + * XE_RTP_RULE_OR - Create an OR condition for rtp rules + * + * RTP rules are AND'ed when evaluated and all of them need to match. + * XE_RTP_RULE_OR allows to create set of rules where any of them matching is + * sufficient for the action to trigger. Example: + * + * .. code-block:: c + * + * const struct xe_rtp_entry_sr entries[] = { + * ... + * { XE_RTP_NAME("test-entry"), + * XE_RTP_RULES(PLATFORM(DG2), OR, PLATFORM(TIGERLAKE)), + * ... + * }, + * ... + * }; + */ +#define XE_RTP_RULE_OR \ + { .match_type = XE_RTP_MATCH_OR } + +/** + * XE_RTP_ACTION_WR - Helper to write a value to the register, overriding all + * the bits + * @reg_: Register + * @val_: Value to set + * @...: Additional fields to override in the struct xe_rtp_action entry + * + * The correspondent notation in bspec is: + * + * REGNAME = VALUE + */ +#define XE_RTP_ACTION_WR(reg_, val_, ...) \ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .clr_bits = ~0u, .set_bits = (val_), \ + .read_mask = (~0u), ##__VA_ARGS__ } + +/** + * XE_RTP_ACTION_SET - Set bits from @val_ in the register. + * @reg_: Register + * @val_: Bits to set in the register + * @...: Additional fields to override in the struct xe_rtp_action entry + * + * For masked registers this translates to a single write, while for other + * registers it's a RMW. The correspondent bspec notation is (example for bits 2 + * and 5, but could be any): + * + * REGNAME[2] = 1 + * REGNAME[5] = 1 + */ +#define XE_RTP_ACTION_SET(reg_, val_, ...) \ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .clr_bits = val_, .set_bits = val_, \ + .read_mask = val_, ##__VA_ARGS__ } + +/** + * XE_RTP_ACTION_CLR: Clear bits from @val_ in the register. + * @reg_: Register + * @val_: Bits to clear in the register + * @...: Additional fields to override in the struct xe_rtp_action entry + * + * For masked registers this translates to a single write, while for other + * registers it's a RMW. The correspondent bspec notation is (example for bits 2 + * and 5, but could be any): + * + * REGNAME[2] = 0 + * REGNAME[5] = 0 + */ +#define XE_RTP_ACTION_CLR(reg_, val_, ...) \ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .clr_bits = val_, .set_bits = 0, \ + .read_mask = val_, ##__VA_ARGS__ } + +/** + * XE_RTP_ACTION_FIELD_SET: Set a bit range + * @reg_: Register + * @mask_bits_: Mask of bits to be changed in the register, forming a field + * @val_: Value to set in the field denoted by @mask_bits_ + * @...: Additional fields to override in the struct xe_rtp_action entry + * + * For masked registers this translates to a single write, while for other + * registers it's a RMW. The correspondent bspec notation is: + * + * REGNAME[:] = VALUE + */ +#define XE_RTP_ACTION_FIELD_SET(reg_, mask_bits_, val_, ...) \ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .clr_bits = mask_bits_, .set_bits = val_, \ + .read_mask = mask_bits_, ##__VA_ARGS__ } + +#define XE_RTP_ACTION_FIELD_SET_NO_READ_MASK(reg_, mask_bits_, val_, ...) \ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .clr_bits = (mask_bits_), .set_bits = (val_), \ + .read_mask = 0, ##__VA_ARGS__ } + +/** + * XE_RTP_ACTION_FIELD_SET_FUNC: Set a bit range to the value returned by a function + * @reg_: Register + * @mask_bits_: Mask of bits to be changed in the register, forming a field + * @func_: Function that returns value to set in the field denoted by @mask_bits_ + * @...: Additional fields to override in the struct xe_rtp_action entry + * + * This macro works like XE_RTP_ACTION_FIELD_SET(), except that the + * field value is evaluated at the time the RTP table is processed. + * + * @func_ will only be called a single time, when the RTP table is being + * processed. After processing, the value in the reg_sr entry is fixed and + * will not be re-evaluated. + */ +#define XE_RTP_ACTION_FIELD_SET_FUNC(reg_, mask_bits_, func_, ...) \ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .clr_bits = mask_bits_, .set_func = func_, .use_func = 1, \ + .read_mask = mask_bits_, ##__VA_ARGS__ } + +/** + * XE_RTP_ACTION_WHITELIST - Add register to userspace whitelist + * @reg_: Register + * @val_: Whitelist-specific flags to set + * @...: Additional fields to override in the struct xe_rtp_action entry + * + * Add a register to the whitelist, allowing userspace to modify the ster with + * regular user privileges. + */ +#define XE_RTP_ACTION_WHITELIST(reg_, val_, ...) \ + /* TODO fail build if ((flags) & ~(RING_FORCE_TO_NONPRIV_MASK_VALID)) */\ + { .reg = XE_RTP_DROP_CAST(reg_), \ + .set_bits = val_, \ + .clr_bits = RING_FORCE_TO_NONPRIV_MASK_VALID, \ + ##__VA_ARGS__ } + +/** + * XE_RTP_NAME - Helper to set the name in xe_rtp_entry + * @s_: Name describing this rule, often a HW-specific number + * + * TODO: maybe move this behind a debug config? + */ +#define XE_RTP_NAME(s_) .name = (s_) + +/** + * XE_RTP_ENTRY_FLAG - Helper to add multiple flags to a struct xe_rtp_entry_sr + * @...: Entry flags, without the ``XE_RTP_ENTRY_FLAG_`` prefix + * + * Helper to automatically add a ``XE_RTP_ENTRY_FLAG_`` prefix to the flags + * when defining struct xe_rtp_entry entries. Example: + * + * .. code-block:: c + * + * const struct xe_rtp_entry_sr wa_entries[] = { + * ... + * { XE_RTP_NAME("test-entry"), + * ... + * XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + * ... + * }, + * ... + * }; + */ +#define XE_RTP_ENTRY_FLAG(...) \ + .flags = (XE_RTP_PASTE_FOREACH(ENTRY_FLAG_, BITWISE_OR, (__VA_ARGS__))) + +/** + * XE_RTP_ACTION_FLAG - Helper to add multiple flags to a struct xe_rtp_action + * @...: Action flags, without the ``XE_RTP_ACTION_FLAG_`` prefix + * + * Helper to automatically add a ``XE_RTP_ACTION_FLAG_`` prefix to the flags + * when defining struct xe_rtp_action entries. Example: + * + * .. code-block:: c + * + * const struct xe_rtp_entry_sr wa_entries[] = { + * ... + * { XE_RTP_NAME("test-entry"), + * ... + * XE_RTP_ACTION_SET(..., XE_RTP_ACTION_FLAG(FOREACH_ENGINE)), + * ... + * }, + * ... + * }; + */ +#define XE_RTP_ACTION_FLAG(...) \ + .flags = (XE_RTP_PASTE_FOREACH(ACTION_FLAG_, BITWISE_OR, (__VA_ARGS__))) + +/** + * XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry_sr entry + * @...: Rules + * + * When an RTP table is being processed, the rules of each entry are evaluated + * to check if they match the target entity (platform, gt or hwe, depending on + * the specific RTP table). + * + * The sequence of arguments of this macro must follow the following eBNF + * grammar:: + * + * rules = disjunction; + * disjunction = conjunction, { "OR", conjunction }; + * conjunction = single_rule, { single_rule }; + * (* the AND operator is implicit *) + * single_rule = ? GRAPHICS_VERSION(...), MEDIA_VERSION(...), + * FUNC(...), etc ? + * + * Examples: + * + * .. code-block:: c + * + * const struct xe_rtp_entry_sr wa_entries[] = { + * ... + * { XE_RTP_NAME("entry-a"), + * // Match DG2-G10 with graphics steppings A0 up-to B0 + * // (exclusive). + * XE_RTP_RULES(SUBPLATFORM(DG2, G10), GRAPHICS_STEP(A0, B0)), + * ... + * }, + * { XE_RTP_NAME("entry-b"), + * // Match graphics version 20 (all steppings) or graphics + * // version 30 steppings A0 up-to B0 (exclusive). + * XE_RTP_RULES(GRAPHICS_VERSION(2000), OR, + * GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0)) + * ... + * }, + * ... + * }; + */ +#define XE_RTP_RULES(...) \ + .n_rules = COUNT_ARGS(__VA_ARGS__), \ + .rules = (const struct xe_rtp_rule[]) { \ + XE_RTP_PASTE_FOREACH(RULE_, COMMA, (__VA_ARGS__)) \ + } + +/** + * XE_RTP_ACTIONS - Helper to set multiple actions to a struct xe_rtp_entry_sr + * @...: Actions to be taken + * + * At least one action is needed and up to 12 are supported. See XE_RTP_ACTION_* + * for the possible actions. Example: + * + * .. code-block:: c + * + * const struct xe_rtp_entry_sr wa_entries[] = { + * ... + * { XE_RTP_NAME("test-entry"), + * XE_RTP_RULES(...), + * XE_RTP_ACTIONS(SET(..), SET(...), CLR(...)), + * ... + * }, + * ... + * }; + */ +#define XE_RTP_ACTIONS(...) \ + .n_actions = COUNT_ARGS(__VA_ARGS__), \ + .actions = (const struct xe_rtp_action[]) { \ + XE_RTP_PASTE_FOREACH(ACTION_, COMMA, (__VA_ARGS__)) \ + } + +/* + * Note: ARRAY_SIZE() cannot be used here because it expands through + * __must_be_array() -> __BUILD_BUG_ON_ZERO_MSG() -> _Static_assert inside + * sizeof(struct{}), which clang < 21 rejects when the compound literal + * contains non-compile-time-constant initializers. + */ +#define XE_RTP_TABLE_SR(...) { \ + .entries = (const struct xe_rtp_entry_sr[]){__VA_ARGS__}, \ + .n_entries = sizeof((const struct xe_rtp_entry_sr[]){__VA_ARGS__}) / \ + sizeof(struct xe_rtp_entry_sr), \ +} + +#define XE_RTP_TABLE(...) { \ + .entries = (const struct xe_rtp_entry[]){__VA_ARGS__}, \ + .n_entries = sizeof((const struct xe_rtp_entry[]){__VA_ARGS__}) / \ + sizeof(struct xe_rtp_entry), \ +} + +#define XE_RTP_PROCESS_CTX_INITIALIZER(arg__) _Generic((arg__), \ + struct xe_hw_engine * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_ENGINE }, \ + struct xe_gt * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_GT }, \ + struct xe_device * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_DEVICE }) + +void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx, + unsigned long *active_entries, + size_t n_entries); + +void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, + const struct xe_rtp_table_sr *table, + struct xe_reg_sr *sr, + bool process_in_vf); + +void xe_rtp_process(struct xe_rtp_process_ctx *ctx, + const struct xe_rtp_table *table); + +/* Match functions to be used with XE_RTP_MATCH_FUNC */ + +/** + * xe_rtp_match_always - Match RTP entry unconditionally + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true, regardless of inputs + */ +bool xe_rtp_match_always(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/** + * xe_rtp_match_even_instance - Match if engine instance is even + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true if engine instance is even, false otherwise + */ +bool xe_rtp_match_even_instance(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/* + * xe_rtp_match_first_render_or_compute - Match if it's first render or compute + * engine in the GT + * + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Registers on the render reset domain need to have their values re-applied + * when any of those engines are reset. Since the engines reset together, a + * programming can be set to just one of them. For simplicity the first engine + * of either render or compute class can be chosen. + * + * Returns: true if engine id is the first to match the render reset domain, + * false otherwise. + */ +bool xe_rtp_match_first_render_or_compute(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/* + * xe_rtp_match_not_sriov_vf - Match when not on SR-IOV VF device + * + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true if device is not VF, false otherwise. + */ +bool xe_rtp_match_not_sriov_vf(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +bool xe_rtp_match_psmi_enabled(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +bool xe_rtp_match_gt_has_discontiguous_dss_groups(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/** + * xe_rtp_match_has_flat_ccs - Match when platform has FlatCCS compression + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true if platform has FlatCCS compression, false otherwise + */ +bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/** + * xe_rtp_match_has_msix - Match when platform has MSI-X + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true if platform has MSI-X interrupt support + */ +bool xe_rtp_match_has_msix(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp_helpers.h b/drivers/gpu/drm/xe/xe_rtp_helpers.h new file mode 100644 index 000000000..ffa8b2f88 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp_helpers.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_RTP_HELPERS_H_ +#define _XE_RTP_HELPERS_H_ + +#ifndef _XE_RTP_INCLUDE_PRIVATE_HELPERS +#error "This header is supposed to be included by xe_rtp.h only" +#endif + +#include "xe_args.h" + +/* + * Helper macros - not to be used outside this header. + */ +#define _XE_ESC(...) __VA_ARGS__ + +#define _XE_TUPLE_TAIL(...) (DROP_FIRST_ARG(__VA_ARGS__)) + +#define _XE_RTP_CONCAT(a, b) CONCATENATE(XE_RTP_, CONCATENATE(a, b)) + +#define __XE_RTP_PASTE_SEP_COMMA , +#define __XE_RTP_PASTE_SEP_BITWISE_OR | + +/* + * XE_RTP_PASTE_FOREACH - Paste XE_RTP_<@prefix_> on each element of the tuple + * @args, with the end result separated by @sep_. @sep must be one of the + * previously declared macros __XE_RTP_PASTE_SEP_*, or declared with such + * prefix. + * + * Examples: + * + * 1) XE_RTP_PASTE_FOREACH(TEST_, COMMA, (FOO, BAR)) + * expands to: + * + * XE_RTP_TEST_FOO , XE_RTP_TEST_BAR + * + * 2) XE_RTP_PASTE_FOREACH(TEST2_, COMMA, (FOO)) + * expands to: + * + * XE_RTP_TEST2_FOO + * + * 3) XE_RTP_PASTE_FOREACH(TEST3, BITWISE_OR, (FOO, BAR)) + * expands to: + * + * XE_RTP_TEST3_FOO | XE_RTP_TEST3_BAR + * + * 4) #define __XE_RTP_PASTE_SEP_MY_SEP BANANA + * XE_RTP_PASTE_FOREACH(TEST_, MY_SEP, (FOO, BAR)) + * expands to: + * + * XE_RTP_TEST_FOO BANANA XE_RTP_TEST_BAR + */ +#define XE_RTP_PASTE_FOREACH(prefix_, sep_, args_) _XE_RTP_CONCAT(PASTE_, COUNT_ARGS args_)(prefix_, sep_, args_) +#define XE_RTP_PASTE_1(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) +#define XE_RTP_PASTE_2(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_1(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_3(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_2(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_4(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_3(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_5(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_4(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_6(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_5(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_7(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_6(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_8(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_7(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_9(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_8(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_10(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_9(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_11(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_10(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_12(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_11(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_13(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_12(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_14(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_13(prefix_, sep_, _XE_TUPLE_TAIL args_) + +/* + * XE_RTP_DROP_CAST - Drop cast to convert a compound statement to a initializer + * + * Example: + * + * #define foo(a_) ((struct foo){ .a = a_ }) + * XE_RTP_DROP_CAST(foo(10)) + * expands to: + * + * { .a = 10 } + */ +#define XE_RTP_DROP_CAST(...) _XE_ESC(DROP_FIRST_ARG _XE_ESC __VA_ARGS__) + +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h new file mode 100644 index 000000000..b78092fa0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp_types.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RTP_TYPES_H_ +#define _XE_RTP_TYPES_H_ + +#include + +#include "regs/xe_reg_defs.h" + +struct xe_device; +struct xe_hw_engine; +struct xe_gt; + +/** + * struct xe_rtp_action - action to take for any matching rule + * + * This struct records what action should be taken in a register that has a + * matching rule. Example of actions: set/clear bits. + */ +struct xe_rtp_action { + /** @reg: Register */ + struct xe_reg reg; + + /** + * @clr_bits: bits to clear when updating register. It's always a + * superset of bits being modified + */ + u32 clr_bits; + + union { + /** @set_bits: bits to set when updating register */ + u32 set_bits; + + /** @set_func: function to provide bits to set when updating register */ + u32 (*set_func)(struct xe_gt *gt, + struct xe_hw_engine *hwe); + }; + +#define XE_RTP_NOCHECK .read_mask = 0 + /** @read_mask: mask for bits to consider when reading value back */ + u32 read_mask; + +#define XE_RTP_ACTION_FLAG_ENGINE_BASE BIT(0) + /** @flags: flags to apply on rule evaluation or action */ + u8 flags; + + /** + * @use_func: + * Internal flag indicating @set_func should be called instead of + * using @set_bits. + */ + u8 use_func:1; +}; + +enum { + XE_RTP_MATCH_PLATFORM, + XE_RTP_MATCH_SUBPLATFORM, + XE_RTP_MATCH_PLATFORM_STEP, + XE_RTP_MATCH_GRAPHICS_VERSION, + XE_RTP_MATCH_GRAPHICS_VERSION_RANGE, + XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT, + XE_RTP_MATCH_GRAPHICS_STEP, + XE_RTP_MATCH_MEDIA_VERSION, + XE_RTP_MATCH_MEDIA_VERSION_RANGE, + XE_RTP_MATCH_MEDIA_VERSION_ANY_GT, + XE_RTP_MATCH_MEDIA_STEP, + XE_RTP_MATCH_INTEGRATED, + XE_RTP_MATCH_DISCRETE, + XE_RTP_MATCH_ENGINE_CLASS, + XE_RTP_MATCH_NOT_ENGINE_CLASS, + XE_RTP_MATCH_FUNC, + XE_RTP_MATCH_OR, +}; + +/** struct xe_rtp_rule - match rule for processing entry */ +struct xe_rtp_rule { + u8 match_type; + + /* match filters */ + union { + /* MATCH_PLATFORM / MATCH_SUBPLATFORM */ + struct { + u8 platform; + u8 subplatform; + }; + + /* + * MATCH_GRAPHICS_VERSION / XE_RTP_MATCH_GRAPHICS_VERSION_RANGE / + * MATCH_MEDIA_VERSION / XE_RTP_MATCH_MEDIA_VERSION_RANGE + */ + struct { + u32 ver_start; +#define XE_RTP_END_VERSION_UNDEFINED U32_MAX + u32 ver_end; + }; + + /* MATCH_STEP */ + struct { + u8 step_start; + u8 step_end; + }; + + /* MATCH_ENGINE_CLASS / MATCH_NOT_ENGINE_CLASS */ + struct { + u8 engine_class; + }; + + /* MATCH_FUNC */ + bool (*match_func)(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + }; +}; + +/** struct xe_rtp_entry_sr - Entry in an rtp table */ +struct xe_rtp_entry_sr { + const char *name; + const struct xe_rtp_action *actions; + const struct xe_rtp_rule *rules; + u8 n_rules; + u8 n_actions; +#define XE_RTP_ENTRY_FLAG_FOREACH_ENGINE BIT(0) + u8 flags; +}; + +/** struct xe_rtp_entry - Entry in an rtp table, with no action associated */ +struct xe_rtp_entry { + const char *name; + const struct xe_rtp_rule *rules; + u8 n_rules; +}; + +struct xe_rtp_table_sr { + const struct xe_rtp_entry_sr *entries; + size_t n_entries; +}; + +struct xe_rtp_table { + const struct xe_rtp_entry *entries; + size_t n_entries; +}; + +enum xe_rtp_process_type { + XE_RTP_PROCESS_TYPE_DEVICE, + XE_RTP_PROCESS_TYPE_GT, + XE_RTP_PROCESS_TYPE_ENGINE, +}; + +struct xe_rtp_process_ctx { + union { + struct xe_device *xe; + struct xe_gt *gt; + struct xe_hw_engine *hwe; + }; + enum xe_rtp_process_type type; + unsigned long *active_entries; + size_t n_entries; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c new file mode 100644 index 000000000..f32045f40 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sa.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_sa.h" + +#include + +#include + +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_map.h" + +static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) +{ + struct xe_sa_manager *sa_manager = arg; + struct xe_bo *bo = sa_manager->bo; + + if (!bo) { + drm_err(drm, "no bo for sa manager\n"); + return; + } + + drm_suballoc_manager_fini(&sa_manager->base); + + if (sa_manager->is_iomem) + kvfree(sa_manager->cpu_ptr); + + sa_manager->bo = NULL; + sa_manager->shadow = NULL; +} + +/** + * __xe_sa_bo_manager_init() - Create and initialize the suballocator + * @tile: the &xe_tile where allocate + * @size: number of bytes to allocate + * @guard: number of bytes to exclude from suballocations + * @align: alignment for each suballocated chunk + * @flags: flags for suballocator + * + * Prepares the suballocation manager for suballocations. + * + * Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure. + */ +struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, + u32 guard, u32 align, u32 flags) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_sa_manager *sa_manager; + u32 managed_size; + struct xe_bo *bo; + int ret; + + xe_tile_assert(tile, size > guard); + managed_size = size - guard; + + sa_manager = drmm_kzalloc(&xe->drm, sizeof(*sa_manager), GFP_KERNEL); + if (!sa_manager) + return ERR_PTR(-ENOMEM); + + bo = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) { + drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n", + size / SZ_1K, bo); + return ERR_CAST(bo); + } + sa_manager->bo = bo; + sa_manager->is_iomem = bo->vmap.is_iomem; + + if (bo->vmap.is_iomem) { + sa_manager->cpu_ptr = kvzalloc(managed_size, GFP_KERNEL); + if (!sa_manager->cpu_ptr) + return ERR_PTR(-ENOMEM); + } else { + sa_manager->cpu_ptr = bo->vmap.vaddr; + memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size); + } + + if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) { + struct xe_bo *shadow; + + ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard); + if (ret) + return ERR_PTR(ret); + + if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&sa_manager->swap_guard); + fs_reclaim_release(GFP_KERNEL); + } + + shadow = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(shadow)) { + drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n", + size / SZ_1K, shadow); + return ERR_CAST(shadow); + } + sa_manager->shadow = shadow; + } + + drm_suballoc_manager_init(&sa_manager->base, managed_size, align); + ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini, + sa_manager); + if (ret) + return ERR_PTR(ret); + + return sa_manager; +} + +/** + * xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO. + * @sa_manager: the XE sub allocator manager + * + * Swaps the sub-allocator primary buffer object with shadow buffer object. + * + * Return: None. + */ +void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager) +{ + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + xe_assert(xe, sa_manager->shadow); + lockdep_assert_held(&sa_manager->swap_guard); + + swap(sa_manager->bo, sa_manager->shadow); + if (!sa_manager->bo->vmap.is_iomem) + sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr; +} + +/** + * xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO. + * @sa_bo: the sub-allocator buffer object. + * + * Synchronize sub-allocator shadow buffer object with primary buffer object. + * + * Return: None. + */ +void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo) +{ + struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + xe_assert(xe, sa_manager->shadow); + lockdep_assert_held(&sa_manager->swap_guard); + + xe_map_memcpy_to(xe, &sa_manager->shadow->vmap, + drm_suballoc_soffset(sa_bo), + xe_sa_bo_cpu_addr(sa_bo), + drm_suballoc_size(sa_bo)); +} + +/** + * __xe_sa_bo_new() - Make a suballocation but use custom gfp flags. + * @sa_manager: the &xe_sa_manager + * @size: number of bytes we want to suballocate + * @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL. + * + * Try to make a suballocation of size @size. + * + * Return: a &drm_suballoc, or an ERR_PTR. + */ +struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp) +{ + /* + * BB to large, return -ENOBUFS indicating user should split + * array of binds into smaller chunks. + */ + if (size > sa_manager->base.size) + return ERR_PTR(-ENOBUFS); + + return drm_suballoc_new(&sa_manager->base, size, gfp, true, 0); +} + +/** + * xe_sa_bo_alloc() - Allocate uninitialized suballoc object. + * @gfp: gfp flags used for memory allocation. + * + * Allocate memory for an uninitialized suballoc object. Intended usage is + * allocate memory for suballoc object outside of a reclaim tainted context + * and then be initialized at a later time in a reclaim tainted context. + * + * Return: a new uninitialized suballoc object, or an ERR_PTR(-ENOMEM). + */ +struct drm_suballoc *xe_sa_bo_alloc(gfp_t gfp) +{ + return drm_suballoc_alloc(gfp); +} + +/** + * xe_sa_bo_init() - Initialize a suballocation. + * @sa_manager: pointer to the sa_manager + * @sa: The struct drm_suballoc. + * @size: number of bytes we want to suballocate. + * + * Try to make a suballocation on a pre-allocated suballoc object of size @size. + * + * Return: zero on success, errno on failure. + */ +int xe_sa_bo_init(struct xe_sa_manager *sa_manager, struct drm_suballoc *sa, size_t size) +{ + return drm_suballoc_insert(&sa_manager->base, sa, size, true, 0); +} + +/** + * xe_sa_bo_flush_write() - Copy the data from the sub-allocation to the GPU memory. + * @sa_bo: the &drm_suballoc to flush + */ +void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo) +{ + struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + if (!sa_manager->bo->vmap.is_iomem) + return; + + xe_map_memcpy_to(xe, &sa_manager->bo->vmap, drm_suballoc_soffset(sa_bo), + xe_sa_bo_cpu_addr(sa_bo), + drm_suballoc_size(sa_bo)); +} + +/** + * xe_sa_bo_sync_read() - Copy the data from GPU memory to the sub-allocation. + * @sa_bo: the &drm_suballoc to sync + */ +void xe_sa_bo_sync_read(struct drm_suballoc *sa_bo) +{ + struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + if (!sa_manager->bo->vmap.is_iomem) + return; + + xe_map_memcpy_from(xe, xe_sa_bo_cpu_addr(sa_bo), &sa_manager->bo->vmap, + drm_suballoc_soffset(sa_bo), + drm_suballoc_size(sa_bo)); +} + +void xe_sa_bo_free(struct drm_suballoc *sa_bo, + struct dma_fence *fence) +{ + drm_suballoc_free(sa_bo, fence); +} diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h new file mode 100644 index 000000000..50218b0d1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sa.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_SA_H_ +#define _XE_SA_H_ + +#include +#include + +#include "xe_bo.h" +#include "xe_sa_types.h" + +struct dma_fence; +struct xe_tile; + +#define XE_SA_BO_MANAGER_FLAG_SHADOW BIT(0) +struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, + u32 guard, u32 align, u32 flags); +struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp); + +static inline struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align) +{ + return __xe_sa_bo_manager_init(tile, size, SZ_4K, align, 0); +} + +/** + * xe_sa_bo_new() - Make a suballocation. + * @sa_manager: the &xe_sa_manager + * @size: number of bytes we want to suballocate + * + * Try to make a suballocation of size @size. + * + * Return: a &drm_suballoc, or an ERR_PTR. + */ +static inline struct drm_suballoc *xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size) +{ + return __xe_sa_bo_new(sa_manager, size, GFP_KERNEL); +} + +struct drm_suballoc *xe_sa_bo_alloc(gfp_t gfp); +int xe_sa_bo_init(struct xe_sa_manager *sa_manager, struct drm_suballoc *sa, size_t size); +void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo); +void xe_sa_bo_sync_read(struct drm_suballoc *sa_bo); +void xe_sa_bo_free(struct drm_suballoc *sa_bo, struct dma_fence *fence); + +static inline struct xe_sa_manager * +to_xe_sa_manager(struct drm_suballoc_manager *mng) +{ + return container_of(mng, struct xe_sa_manager, base); +} + +/** + * xe_sa_manager_gpu_addr - Retrieve GPU address of a back storage BO + * within suballocator. + * @sa_manager: the &xe_sa_manager struct instance + * Return: GGTT address of the back storage BO. + */ +static inline u64 xe_sa_manager_gpu_addr(struct xe_sa_manager *sa_manager) +{ + return xe_bo_ggtt_addr(sa_manager->bo); +} + +static inline u64 xe_sa_bo_gpu_addr(struct drm_suballoc *sa) +{ + return xe_sa_manager_gpu_addr(to_xe_sa_manager(sa->manager)) + + drm_suballoc_soffset(sa); +} + +static inline void *xe_sa_bo_cpu_addr(struct drm_suballoc *sa) +{ + return to_xe_sa_manager(sa->manager)->cpu_ptr + + drm_suballoc_soffset(sa); +} + +void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager); +void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo); + +/** + * xe_sa_bo_swap_guard() - Retrieve the SA BO swap guard within sub-allocator. + * @sa_manager: the &xe_sa_manager + * + * Return: Sub alloctor swap guard mutex. + */ +static inline struct mutex *xe_sa_bo_swap_guard(struct xe_sa_manager *sa_manager) +{ + return &sa_manager->swap_guard; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_sa_types.h b/drivers/gpu/drm/xe/xe_sa_types.h new file mode 100644 index 000000000..1085c9c37 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sa_types.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_SA_TYPES_H_ +#define _XE_SA_TYPES_H_ + +#include + +struct xe_bo; + +struct xe_sa_manager { + struct drm_suballoc_manager base; + struct xe_bo *bo; + struct xe_bo *shadow; + /** @swap_guard: Timeline guard updating @bo and @shadow */ + struct mutex swap_guard; + void *cpu_ptr; + bool is_iomem; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c new file mode 100644 index 000000000..a4fa00632 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job.c @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_sched_job.h" + +#include +#include +#include + +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_gt_types.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_pm.h" +#include "xe_sync_types.h" +#include "xe_trace.h" +#include "xe_vm.h" + +static struct kmem_cache *xe_sched_job_slab; +static struct kmem_cache *xe_sched_job_parallel_slab; + +int __init xe_sched_job_module_init(void) +{ + xe_sched_job_slab = + kmem_cache_create("xe_sched_job", + sizeof(struct xe_sched_job) + + sizeof(struct xe_job_ptrs), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_sched_job_slab) + return -ENOMEM; + + xe_sched_job_parallel_slab = + kmem_cache_create("xe_sched_job_parallel", + sizeof(struct xe_sched_job) + + sizeof(struct xe_job_ptrs) * + XE_HW_ENGINE_MAX_INSTANCE, 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_sched_job_parallel_slab) { + kmem_cache_destroy(xe_sched_job_slab); + return -ENOMEM; + } + + return 0; +} + +void xe_sched_job_module_exit(void) +{ + kmem_cache_destroy(xe_sched_job_slab); + kmem_cache_destroy(xe_sched_job_parallel_slab); +} + +static struct xe_sched_job *job_alloc(bool parallel) +{ + return kmem_cache_zalloc(parallel ? xe_sched_job_parallel_slab : + xe_sched_job_slab, GFP_KERNEL); +} + +bool xe_sched_job_is_migration(struct xe_exec_queue *q) +{ + return q->vm && (q->vm->flags & XE_VM_FLAG_MIGRATION); +} + +static void job_free(struct xe_sched_job *job) +{ + struct xe_exec_queue *q = job->q; + bool is_migration = xe_sched_job_is_migration(q); + + kmem_cache_free(xe_exec_queue_is_parallel(job->q) || is_migration ? + xe_sched_job_parallel_slab : xe_sched_job_slab, job); +} + +static struct xe_device *job_to_xe(struct xe_sched_job *job) +{ + return gt_to_xe(job->q->gt); +} + +/* Free unused pre-allocated fences */ +static void xe_sched_job_free_fences(struct xe_sched_job *job) +{ + int i; + + for (i = 0; i < job->q->width; ++i) { + struct xe_job_ptrs *ptrs = &job->ptrs[i]; + + if (ptrs->lrc_fence) + xe_lrc_free_seqno_fence(ptrs->lrc_fence); + dma_fence_chain_free(ptrs->chain_fence); + } +} + +struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, + u64 *batch_addr) +{ + bool is_migration = xe_sched_job_is_migration(q); + struct xe_sched_job *job; + int err; + int i; + u32 width; + + /* only a kernel context can submit a vm-less job */ + XE_WARN_ON(!q->vm && !(q->flags & EXEC_QUEUE_FLAG_KERNEL)); + + job = job_alloc(xe_exec_queue_is_parallel(q) || is_migration); + if (!job) + return ERR_PTR(-ENOMEM); + + job->q = q; + job->sample_timestamp = U64_MAX; + kref_init(&job->refcount); + xe_exec_queue_get(job->q); + + err = drm_sched_job_init(&job->drm, q->entity, 1, NULL, + q->xef ? q->xef->drm->client_id : 0); + if (err) + goto err_free; + + for (i = 0; i < q->width; ++i) { + struct dma_fence *fence = xe_lrc_alloc_seqno_fence(); + struct dma_fence_chain *chain; + + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto err_sched_job; + } + job->ptrs[i].lrc_fence = fence; + + if (i + 1 == q->width) + continue; + + chain = dma_fence_chain_alloc(); + if (!chain) { + err = -ENOMEM; + goto err_sched_job; + } + job->ptrs[i].chain_fence = chain; + } + + width = q->width; + if (is_migration) + width = 2; + + for (i = 0; i < width; ++i) + job->ptrs[i].batch_addr = batch_addr[i]; + + atomic_inc(&q->job_cnt); + xe_pm_runtime_get_noresume(job_to_xe(job)); + trace_xe_sched_job_create(job); + return job; + +err_sched_job: + xe_sched_job_free_fences(job); + drm_sched_job_cleanup(&job->drm); +err_free: + xe_exec_queue_put(q); + job_free(job); + return ERR_PTR(err); +} + +/** + * xe_sched_job_destroy - Destroy Xe schedule job + * @ref: reference to Xe schedule job + * + * Called when ref == 0, drop a reference to job's xe_engine + fence, cleanup + * base DRM schedule job, and free memory for Xe schedule job. + */ +void xe_sched_job_destroy(struct kref *ref) +{ + struct xe_sched_job *job = + container_of(ref, struct xe_sched_job, refcount); + struct xe_device *xe = job_to_xe(job); + struct xe_exec_queue *q = job->q; + + xe_sched_job_free_fences(job); + dma_fence_put(job->fence); + drm_sched_job_cleanup(&job->drm); + job_free(job); + atomic_dec(&q->job_cnt); + xe_exec_queue_put(q); + xe_pm_runtime_put(xe); +} + +/* Set the error status under the fence to avoid racing with signaling */ +static bool xe_fence_set_error(struct dma_fence *fence, int error) +{ + unsigned long irq_flags; + bool signaled; + + dma_fence_lock_irqsave(fence, irq_flags); + signaled = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); + if (!signaled) + dma_fence_set_error(fence, error); + dma_fence_unlock_irqrestore(fence, irq_flags); + + return signaled; +} + +void xe_sched_job_set_error(struct xe_sched_job *job, int error) +{ + if (xe_fence_set_error(job->fence, error)) + return; + + if (dma_fence_is_chain(job->fence)) { + struct dma_fence *iter; + + dma_fence_chain_for_each(iter, job->fence) + xe_fence_set_error(dma_fence_chain_contained(iter), + error); + } + + trace_xe_sched_job_set_error(job); + + dma_fence_enable_signaling(job->fence); + xe_hw_fence_irq_run(job->q->fence_irq); +} + +bool xe_sched_job_started(struct xe_sched_job *job) +{ + struct dma_fence *fence = dma_fence_chain_contained(job->fence); + struct xe_lrc *lrc = job->q->lrc[0]; + + return !__dma_fence_is_later(fence, + xe_sched_job_lrc_seqno(job), + xe_lrc_start_seqno(lrc)); +} + +bool xe_sched_job_completed(struct xe_sched_job *job) +{ + struct dma_fence *fence = dma_fence_chain_contained(job->fence); + struct xe_lrc *lrc = job->q->lrc[0]; + + /* + * Can safely check just LRC[0] seqno as that is last seqno written when + * parallel handshake is done. + */ + + return !__dma_fence_is_later(fence, + xe_sched_job_lrc_seqno(job), + xe_lrc_seqno(lrc)); +} + +void xe_sched_job_arm(struct xe_sched_job *job) +{ + struct xe_exec_queue *q = job->q; + struct dma_fence *fence, *prev; + struct xe_vm *vm = q->vm; + u64 seqno = 0; + int i; + + /* Migration and kernel engines have their own locking */ + if (IS_ENABLED(CONFIG_LOCKDEP) && + !(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { + lockdep_assert_held(&q->vm->lock); + if (!xe_vm_in_lr_mode(q->vm)) + xe_vm_assert_held(q->vm); + } + + if (vm && !xe_sched_job_is_migration(q) && !xe_vm_in_lr_mode(vm) && + (vm->batch_invalidate_tlb || vm->tlb_flush_seqno != q->tlb_flush_seqno)) { + xe_vm_assert_held(vm); + q->tlb_flush_seqno = vm->tlb_flush_seqno; + job->ring_ops_flush_tlb = true; + } + + /* Arm the pre-allocated fences */ + for (i = 0; i < q->width; prev = fence, ++i) { + struct dma_fence_chain *chain; + + fence = job->ptrs[i].lrc_fence; + xe_lrc_init_seqno_fence(q->lrc[i], fence); + job->ptrs[i].lrc_fence = NULL; + if (!i) { + job->lrc_seqno = fence->seqno; + continue; + } else { + xe_assert(gt_to_xe(q->gt), job->lrc_seqno == fence->seqno); + } + + chain = job->ptrs[i - 1].chain_fence; + dma_fence_chain_init(chain, prev, fence, seqno++); + job->ptrs[i - 1].chain_fence = NULL; + fence = &chain->base; + } + + job->fence = dma_fence_get(fence); /* Pairs with put in scheduler */ + drm_sched_job_arm(&job->drm); +} + +void xe_sched_job_push(struct xe_sched_job *job) +{ + xe_sched_job_get(job); + trace_xe_sched_job_exec(job); + drm_sched_entity_push_job(&job->drm); + xe_sched_job_put(job); +} + +/** + * xe_sched_job_init_user_fence - Initialize user_fence for the job + * @job: job whose user_fence needs an init + * @sync: sync to be use to init user_fence + */ +void xe_sched_job_init_user_fence(struct xe_sched_job *job, + struct xe_sync_entry *sync) +{ + if (sync->type != DRM_XE_SYNC_TYPE_USER_FENCE) + return; + + job->user_fence.used = true; + job->user_fence.addr = sync->addr; + job->user_fence.value = sync->timeline_value; +} + +struct xe_sched_job_snapshot * +xe_sched_job_snapshot_capture(struct xe_sched_job *job) +{ + struct xe_exec_queue *q = job->q; + struct xe_device *xe = q->gt->tile->xe; + struct xe_sched_job_snapshot *snapshot; + size_t len = sizeof(*snapshot) + (sizeof(u64) * q->width); + u16 i; + + snapshot = kzalloc(len, GFP_ATOMIC); + if (!snapshot) + return NULL; + + snapshot->batch_addr_len = q->width; + for (i = 0; i < q->width; i++) + snapshot->batch_addr[i] = + xe_device_uncanonicalize_addr(xe, job->ptrs[i].batch_addr); + + return snapshot; +} + +void xe_sched_job_snapshot_free(struct xe_sched_job_snapshot *snapshot) +{ + kfree(snapshot); +} + +void +xe_sched_job_snapshot_print(struct xe_sched_job_snapshot *snapshot, + struct drm_printer *p) +{ + u16 i; + + if (!snapshot) + return; + + for (i = 0; i < snapshot->batch_addr_len; i++) + drm_printf(p, "batch_addr[%u]: 0x%016llx\n", i, snapshot->batch_addr[i]); +} + +int xe_sched_job_add_deps(struct xe_sched_job *job, struct dma_resv *resv, + enum dma_resv_usage usage) +{ + return drm_sched_job_add_resv_dependencies(&job->drm, resv, usage); +} diff --git a/drivers/gpu/drm/xe/xe_sched_job.h b/drivers/gpu/drm/xe/xe_sched_job.h new file mode 100644 index 000000000..1c1cb4421 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_SCHED_JOB_H_ +#define _XE_SCHED_JOB_H_ + +#include "xe_sched_job_types.h" + +struct drm_printer; +struct xe_vm; +struct xe_sync_entry; + +#define XE_SCHED_HANG_LIMIT 1 +#define XE_SCHED_JOB_TIMEOUT LONG_MAX + +int xe_sched_job_module_init(void); +void xe_sched_job_module_exit(void); + +struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, + u64 *batch_addr); +void xe_sched_job_destroy(struct kref *ref); + +/** + * xe_sched_job_get - get reference to Xe schedule job + * @job: Xe schedule job object + * + * Increment Xe schedule job's reference count + */ +static inline struct xe_sched_job *xe_sched_job_get(struct xe_sched_job *job) +{ + kref_get(&job->refcount); + return job; +} + +/** + * xe_sched_job_put - put reference to Xe schedule job + * @job: Xe schedule job object + * + * Decrement Xe schedule job's reference count, call xe_sched_job_destroy when + * reference count == 0. + */ +static inline void xe_sched_job_put(struct xe_sched_job *job) +{ + kref_put(&job->refcount, xe_sched_job_destroy); +} + +void xe_sched_job_set_error(struct xe_sched_job *job, int error); +static inline bool xe_sched_job_is_error(struct xe_sched_job *job) +{ + return job->fence->error < 0; +} + +bool xe_sched_job_started(struct xe_sched_job *job); +bool xe_sched_job_completed(struct xe_sched_job *job); + +void xe_sched_job_arm(struct xe_sched_job *job); +void xe_sched_job_push(struct xe_sched_job *job); + +void xe_sched_job_init_user_fence(struct xe_sched_job *job, + struct xe_sync_entry *sync); + +static inline struct xe_sched_job * +to_xe_sched_job(struct drm_sched_job *drm) +{ + return container_of(drm, struct xe_sched_job, drm); +} + +static inline u32 xe_sched_job_seqno(struct xe_sched_job *job) +{ + return job->fence ? job->fence->seqno : 0; +} + +static inline u32 xe_sched_job_lrc_seqno(struct xe_sched_job *job) +{ + return job->lrc_seqno; +} + +static inline void +xe_sched_job_add_migrate_flush(struct xe_sched_job *job, u32 flags) +{ + job->migrate_flush_flags = flags; +} + +bool xe_sched_job_is_migration(struct xe_exec_queue *q); + +struct xe_sched_job_snapshot *xe_sched_job_snapshot_capture(struct xe_sched_job *job); +void xe_sched_job_snapshot_free(struct xe_sched_job_snapshot *snapshot); +void xe_sched_job_snapshot_print(struct xe_sched_job_snapshot *snapshot, struct drm_printer *p); + +int xe_sched_job_add_deps(struct xe_sched_job *job, struct dma_resv *resv, + enum dma_resv_usage usage); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h new file mode 100644 index 000000000..0490b1247 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job_types.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_SCHED_JOB_TYPES_H_ +#define _XE_SCHED_JOB_TYPES_H_ + +#include + +#include + +struct xe_exec_queue; +struct dma_fence; +struct dma_fence_chain; + +/** + * struct xe_job_ptrs - Per hw engine instance data + */ +struct xe_job_ptrs { + /** @lrc_fence: Pre-allocated uninitialized lrc fence.*/ + struct dma_fence *lrc_fence; + /** @chain_fence: Pre-allocated uninitialized fence chain node. */ + struct dma_fence_chain *chain_fence; + /** @batch_addr: Batch buffer address. */ + u64 batch_addr; + /** + * @head: The tail pointer of the LRC (so head pointer of job) when the + * job was submitted + */ + u32 head; +}; + +/** + * struct xe_sched_job - Xe schedule job (batch buffer tracking) + */ +struct xe_sched_job { + /** @drm: base DRM scheduler job */ + struct drm_sched_job drm; + /** @q: Exec queue */ + struct xe_exec_queue *q; + /** @refcount: ref count of this job */ + struct kref refcount; + /** + * @fence: dma fence to indicate completion. 1 way relationship - job + * can safely reference fence, fence cannot safely reference job. + */ + struct dma_fence *fence; + /** @user_fence: write back value when BB is complete */ + struct { + /** @user_fence.used: user fence is used */ + bool used; + /** @user_fence.addr: address to write to */ + u64 addr; + /** @user_fence.value: write back value */ + u64 value; + } user_fence; + /** @lrc_seqno: LRC seqno */ + u32 lrc_seqno; + /** @migrate_flush_flags: Additional flush flags for migration jobs */ + u32 migrate_flush_flags; + /** @sample_timestamp: Sampling of job timestamp in TDR */ + u64 sample_timestamp; + /** @ring_ops_flush_tlb: The ring ops need to flush TLB before payload. */ + bool ring_ops_flush_tlb; + /** @ring_ops_force_reset: The ring ops need to trigger a reset before payload. */ + bool ring_ops_force_reset; + /** @ggtt: mapped in ggtt. */ + bool ggtt; + /** @restore_replay: job being replayed for restore */ + bool restore_replay; + /** @last_replay: last job being replayed */ + bool last_replay; + /** @ptrs: per instance pointers. */ + struct xe_job_ptrs ptrs[]; +}; + +struct xe_sched_job_snapshot { + u16 batch_addr_len; + u64 batch_addr[] __counted_by(batch_addr_len); +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c new file mode 100644 index 000000000..83374cd57 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_shrinker.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_pm.h" +#include "xe_shrinker.h" + +/** + * struct xe_shrinker - per-device shrinker + * @xe: Back pointer to the device. + * @lock: Lock protecting accounting. + * @shrinkable_pages: Number of pages that are currently shrinkable. + * @purgeable_pages: Number of pages that are currently purgeable. + * @shrink: Pointer to the mm shrinker. + * @pm_worker: Worker to wake up the device if required. + */ +struct xe_shrinker { + struct xe_device *xe; + rwlock_t lock; + long shrinkable_pages; + long purgeable_pages; + struct shrinker *shrink; + struct work_struct pm_worker; +}; + +static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink) +{ + return shrink->private_data; +} + +/** + * xe_shrinker_mod_pages() - Modify shrinker page accounting + * @shrinker: Pointer to the struct xe_shrinker. + * @shrinkable: Shrinkable pages delta. May be negative. + * @purgeable: Purgeable page delta. May be negative. + * + * Modifies the shrinkable and purgeable pages accounting. + */ +void +xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable) +{ + write_lock(&shrinker->lock); + shrinker->shrinkable_pages += shrinkable; + shrinker->purgeable_pages += purgeable; + write_unlock(&shrinker->lock); +} + +static s64 __xe_shrinker_walk(struct xe_device *xe, + struct ttm_operation_ctx *ctx, + const struct xe_bo_shrink_flags flags, + unsigned long to_scan, unsigned long *scanned) +{ + unsigned int mem_type; + s64 freed = 0, lret; + + for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { + struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); + struct ttm_bo_lru_cursor curs; + struct ttm_buffer_object *ttm_bo; + struct ttm_lru_walk_arg arg = { + .ctx = ctx, + .trylock_only = true, + }; + + if (!man || !man->use_tt) + continue; + + ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) { + if (!ttm_bo_shrink_suitable(ttm_bo, ctx)) + continue; + + lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned); + if (lret < 0) + return lret; + + freed += lret; + if (*scanned >= to_scan) + break; + } + /* Trylocks should never error, just fail. */ + xe_assert(xe, !IS_ERR(ttm_bo)); + } + + return freed; +} + +/* + * Try shrinking idle objects without writeback first, then if not sufficient, + * try also non-idle objects and finally if that's not sufficient either, + * add writeback. This avoids stalls and explicit writebacks with light or + * moderate memory pressure. + */ +static s64 xe_shrinker_walk(struct xe_device *xe, + struct ttm_operation_ctx *ctx, + const struct xe_bo_shrink_flags flags, + unsigned long to_scan, unsigned long *scanned) +{ + bool no_wait_gpu = true; + struct xe_bo_shrink_flags save_flags = flags; + s64 lret, freed; + + swap(no_wait_gpu, ctx->no_wait_gpu); + save_flags.writeback = false; + lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); + swap(no_wait_gpu, ctx->no_wait_gpu); + if (lret < 0 || *scanned >= to_scan) + return lret; + + freed = lret; + if (!ctx->no_wait_gpu) { + lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); + if (lret < 0) + return lret; + freed += lret; + if (*scanned >= to_scan) + return freed; + } + + if (flags.writeback) { + lret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned); + if (lret < 0) + return lret; + freed += lret; + } + + return freed; +} + +static unsigned long +xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct xe_shrinker *shrinker = to_xe_shrinker(shrink); + unsigned long num_pages; + bool can_backup = !!(sc->gfp_mask & __GFP_FS); + + num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT; + read_lock(&shrinker->lock); + + if (can_backup) + num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages); + else + num_pages = 0; + + num_pages += shrinker->purgeable_pages; + read_unlock(&shrinker->lock); + + return num_pages ? num_pages : SHRINK_EMPTY; +} + +/* + * Check if we need runtime pm, and if so try to grab a reference if + * already active. If grabbing a reference fails, queue a worker that + * does it for us outside of reclaim, but don't wait for it to complete. + * If bo shrinking needs an rpm reference and we don't have it (yet), + * that bo will be skipped anyway. + */ +static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force, + unsigned long nr_to_scan, bool can_backup) +{ + struct xe_device *xe = shrinker->xe; + + if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) || + !ttm_backup_bytes_avail()) + return false; + + if (!force) { + read_lock(&shrinker->lock); + force = (nr_to_scan > shrinker->purgeable_pages && can_backup); + read_unlock(&shrinker->lock); + if (!force) + return false; + } + + if (!xe_pm_runtime_get_if_active(xe)) { + if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { + xe_pm_runtime_get(xe); + return true; + } + queue_work(xe->unordered_wq, &shrinker->pm_worker); + return false; + } + + return true; +} + +static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) +{ + if (runtime_pm) + xe_pm_runtime_put(shrinker->xe); +} + +static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + struct xe_shrinker *shrinker = to_xe_shrinker(shrink); + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = ttm_bo_shrink_avoid_wait(), + }; + unsigned long nr_to_scan, nr_scanned = 0, freed = 0; + struct xe_bo_shrink_flags shrink_flags = { + .purge = true, + /* Don't request writeback without __GFP_IO. */ + .writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO), + }; + bool runtime_pm; + bool purgeable; + bool can_backup = !!(sc->gfp_mask & __GFP_FS); + s64 lret; + + nr_to_scan = sc->nr_to_scan; + + read_lock(&shrinker->lock); + purgeable = !!shrinker->purgeable_pages; + read_unlock(&shrinker->lock); + + /* Might need runtime PM. Try to wake early if it looks like it. */ + runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); + + if (purgeable && nr_scanned < nr_to_scan) { + lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, + nr_to_scan, &nr_scanned); + if (lret >= 0) + freed += lret; + } + + sc->nr_scanned = nr_scanned; + if (nr_scanned >= nr_to_scan || !can_backup) + goto out; + + /* If we didn't wake before, try to do it now if needed. */ + if (!runtime_pm) + runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup); + + shrink_flags.purge = false; + + lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, + nr_to_scan, &nr_scanned); + if (lret >= 0) + freed += lret; + + sc->nr_scanned = nr_scanned; +out: + xe_shrinker_runtime_pm_put(shrinker, runtime_pm); + return nr_scanned ? freed : SHRINK_STOP; +} + +/* Wake up the device for shrinking. */ +static void xe_shrinker_pm(struct work_struct *work) +{ + struct xe_shrinker *shrinker = + container_of(work, typeof(*shrinker), pm_worker); + + xe_pm_runtime_get(shrinker->xe); + xe_pm_runtime_put(shrinker->xe); +} + +static void xe_shrinker_fini(struct drm_device *drm, void *arg) +{ + struct xe_shrinker *shrinker = arg; + + xe_assert(shrinker->xe, !shrinker->shrinkable_pages); + xe_assert(shrinker->xe, !shrinker->purgeable_pages); + shrinker_free(shrinker->shrink); + flush_work(&shrinker->pm_worker); + kfree(shrinker); +} + +/** + * xe_shrinker_create() - Create an xe per-device shrinker + * @xe: Pointer to the xe device. + * + * Return: %0 on success. Negative error code on failure. + */ +int xe_shrinker_create(struct xe_device *xe) +{ + struct xe_shrinker *shrinker = kzalloc_obj(*shrinker); + + if (!shrinker) + return -ENOMEM; + + shrinker->shrink = shrinker_alloc(0, "drm-xe_gem:%s", xe->drm.unique); + if (!shrinker->shrink) { + kfree(shrinker); + return -ENOMEM; + } + + INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm); + shrinker->xe = xe; + rwlock_init(&shrinker->lock); + shrinker->shrink->count_objects = xe_shrinker_count; + shrinker->shrink->scan_objects = xe_shrinker_scan; + shrinker->shrink->private_data = shrinker; + shrinker_register(shrinker->shrink); + xe->mem.shrinker = shrinker; + + return drmm_add_action_or_reset(&xe->drm, xe_shrinker_fini, shrinker); +} diff --git a/drivers/gpu/drm/xe/xe_shrinker.h b/drivers/gpu/drm/xe/xe_shrinker.h new file mode 100644 index 000000000..5132ae519 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_shrinker.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_SHRINKER_H_ +#define _XE_SHRINKER_H_ + +struct xe_shrinker; +struct xe_device; + +void xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable); + +int xe_shrinker_create(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sleep.h b/drivers/gpu/drm/xe/xe_sleep.h new file mode 100644 index 000000000..2bc3f4c0e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sleep.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SLEEP_H_ +#define _XE_SLEEP_H_ + +#include +#include + +/** + * xe_sleep_relaxed_ms() - Sleep for an approximate time. + * @delay_ms: time in msec to sleep + * + * For smaller timeouts, sleep with 0.5ms accuracy. + */ +static inline void xe_sleep_relaxed_ms(unsigned int delay_ms) +{ + unsigned long min_us, max_us; + + if (!delay_ms) + return; + + if (delay_ms > 20) { + msleep(delay_ms); + return; + } + + min_us = mul_u32_u32(delay_ms, 1000); + max_us = min_us + 500; + + usleep_range(min_us, max_us); +} + +/** + * xe_sleep_exponential_ms() - Sleep for a exponentially increased time. + * @sleep_period_ms: current time in msec to sleep + * @max_sleep_ms: maximum time in msec to sleep + * + * Sleep for the @sleep_period_ms and exponentially increase this time for the + * next loop, unless reaching the @max_sleep_ms limit. + * + * Return: approximate time in msec the task was delayed. + */ +static inline unsigned int xe_sleep_exponential_ms(unsigned int *sleep_period_ms, + unsigned int max_sleep_ms) +{ + unsigned int delay_ms = *sleep_period_ms; + unsigned int next_delay_ms = 2 * delay_ms; + + xe_sleep_relaxed_ms(delay_ms); + *sleep_period_ms = min(next_delay_ms, max_sleep_ms); + return delay_ms; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_soc_remapper.c b/drivers/gpu/drm/xe/xe_soc_remapper.c new file mode 100644 index 000000000..c031336a6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_soc_remapper.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "regs/xe_soc_remapper_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_soc_remapper.h" + +static void xe_soc_remapper_set_region(struct xe_device *xe, struct xe_reg reg, + u32 mask, u32 val) +{ + guard(spinlock_irqsave)(&xe->soc_remapper.lock); + xe_mmio_rmw32(xe_root_tile_mmio(xe), reg, mask, val); +} + +static void xe_soc_remapper_set_telem_region(struct xe_device *xe, u32 index) +{ + xe_soc_remapper_set_region(xe, SG_REMAP_INDEX1, SG_REMAP_TELEM_MASK, + REG_FIELD_PREP(SG_REMAP_TELEM_MASK, index)); +} + +static void xe_soc_remapper_set_sysctrl_region(struct xe_device *xe, u32 index) +{ + xe_soc_remapper_set_region(xe, SG_REMAP_INDEX1, SG_REMAP_SYSCTRL_MASK, + REG_FIELD_PREP(SG_REMAP_SYSCTRL_MASK, index)); +} + +/** + * xe_soc_remapper_init() - Initialize SoC remapper + * @xe: Pointer to xe device. + * + * Initialize SoC remapper. + * + * Return: 0 on success, error code on failure + */ +int xe_soc_remapper_init(struct xe_device *xe) +{ + bool has_soc_remapper = xe->info.has_soc_remapper_telem || + xe->info.has_soc_remapper_sysctrl; + + if (has_soc_remapper) + spin_lock_init(&xe->soc_remapper.lock); + + if (xe->info.has_soc_remapper_telem) + xe->soc_remapper.set_telem_region = xe_soc_remapper_set_telem_region; + + if (xe->info.has_soc_remapper_sysctrl) + xe->soc_remapper.set_sysctrl_region = xe_soc_remapper_set_sysctrl_region; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_soc_remapper.h b/drivers/gpu/drm/xe/xe_soc_remapper.h new file mode 100644 index 000000000..1060ad0f5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_soc_remapper.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SOC_REMAPPER_H_ +#define _XE_SOC_REMAPPER_H_ + +#include "xe_device_types.h" + +int xe_soc_remapper_init(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov.c b/drivers/gpu/drm/xe/xe_sriov.c new file mode 100644 index 000000000..f3835867f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include + +#include "regs/xe_regs.h" + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_vf.h" +#include "xe_sriov_vf_ccs.h" + +/** + * xe_sriov_mode_to_string - Convert enum value to string. + * @mode: the &xe_sriov_mode to convert + * + * Returns: SR-IOV mode as a user friendly string. + */ +const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode) +{ + switch (mode) { + case XE_SRIOV_MODE_NONE: + return "none"; + case XE_SRIOV_MODE_PF: + return "SR-IOV PF"; + case XE_SRIOV_MODE_VF: + return "SR-IOV VF"; + default: + return ""; + } +} + +static bool test_is_vf(struct xe_device *xe) +{ + u32 value = xe_mmio_read32(xe_root_tile_mmio(xe), VF_CAP_REG); + + return value & VF_CAP; +} + +/** + * xe_sriov_probe_early - Probe a SR-IOV mode. + * @xe: the &xe_device to probe mode on + * + * This function should be called only once and as soon as possible during + * driver probe to detect whether we are running a SR-IOV Physical Function + * (PF) or a Virtual Function (VF) device. + * + * SR-IOV PF mode detection is based on PCI @dev_is_pf() function. + * SR-IOV VF mode detection is based on dedicated MMIO register read. + */ +void xe_sriov_probe_early(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + enum xe_sriov_mode mode = XE_SRIOV_MODE_NONE; + bool has_sriov = xe->info.has_sriov; + + if (has_sriov) { + if (test_is_vf(xe)) + mode = XE_SRIOV_MODE_VF; + else if (xe_sriov_pf_readiness(xe)) + mode = XE_SRIOV_MODE_PF; + } else if (pci_sriov_get_totalvfs(pdev)) { + /* + * Even if we have not enabled SR-IOV support using the + * platform specific has_sriov flag, the hardware may still + * report SR-IOV capability and the PCI layer may wrongly + * advertise driver support to enable VFs. Explicitly reset + * the number of supported VFs to zero to avoid confusion. + */ + drm_info(&xe->drm, "Support for SR-IOV is not available\n"); + pci_sriov_set_totalvfs(pdev, 0); + } + + xe_assert(xe, !xe->sriov.__mode); + xe->sriov.__mode = mode; + xe_assert(xe, xe->sriov.__mode); + + if (IS_SRIOV(xe)) + drm_info(&xe->drm, "Running in %s mode\n", + xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); +} + +static void fini_sriov(struct drm_device *drm, void *arg) +{ + struct xe_device *xe = arg; + + destroy_workqueue(xe->sriov.wq); + xe->sriov.wq = NULL; +} + +/** + * xe_sriov_init - Initialize SR-IOV specific data. + * @xe: the &xe_device to initialize + * + * In this function we create dedicated workqueue that will be used + * by the SR-IOV specific workers. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_init(struct xe_device *xe) +{ + if (!IS_SRIOV(xe)) + return 0; + + if (IS_SRIOV_PF(xe)) { + int err = xe_sriov_pf_init_early(xe); + + if (err) + return err; + } + + if (IS_SRIOV_VF(xe)) + xe_sriov_vf_init_early(xe); + + xe_assert(xe, !xe->sriov.wq); + xe->sriov.wq = alloc_workqueue("xe-sriov-wq", WQ_PERCPU, 0); + if (!xe->sriov.wq) + return -ENOMEM; + + return drmm_add_action_or_reset(&xe->drm, fini_sriov, xe); +} +ALLOW_ERROR_INJECTION(xe_sriov_init, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_sriov_print_info - Print basic SR-IOV information. + * @xe: the &xe_device to print info from + * @p: the &drm_printer + * + * Print SR-IOV related information into provided DRM printer. + */ +void xe_sriov_print_info(struct xe_device *xe, struct drm_printer *p) +{ + drm_printf(p, "supported: %s\n", str_yes_no(xe_device_has_sriov(xe))); + drm_printf(p, "enabled: %s\n", str_yes_no(IS_SRIOV(xe))); + drm_printf(p, "mode: %s\n", xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); +} + +/** + * xe_sriov_function_name() - Get SR-IOV Function name. + * @n: the Function number (identifier) to get name of + * @buf: the buffer to format to + * @size: size of the buffer (shall be at least 5 bytes) + * + * Return: formatted function name ("PF" or "VF%u"). + */ +const char *xe_sriov_function_name(unsigned int n, char *buf, size_t size) +{ + if (n) + snprintf(buf, size, "VF%u", n); + else + strscpy(buf, "PF", size); + return buf; +} + +/** + * xe_sriov_init_late() - SR-IOV late initialization functions. + * @xe: the &xe_device to initialize + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_init_late(struct xe_device *xe) +{ + if (IS_SRIOV_PF(xe)) + return xe_sriov_pf_init_late(xe); + if (IS_SRIOV_VF(xe)) + return xe_sriov_vf_init_late(xe); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_sriov.h b/drivers/gpu/drm/xe/xe_sriov.h new file mode 100644 index 000000000..72e55543c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_SRIOV_H_ +#define _XE_SRIOV_H_ + +#include "xe_assert.h" +#include "xe_device_types.h" +#include "xe_sriov_types.h" + +struct drm_printer; + +const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode); +const char *xe_sriov_function_name(unsigned int n, char *buf, size_t len); + +void xe_sriov_probe_early(struct xe_device *xe); +void xe_sriov_print_info(struct xe_device *xe, struct drm_printer *p); +int xe_sriov_init(struct xe_device *xe); +int xe_sriov_init_late(struct xe_device *xe); + +static inline enum xe_sriov_mode xe_device_sriov_mode(const struct xe_device *xe) +{ + xe_assert(xe, xe->sriov.__mode); + return xe->sriov.__mode; +} + +static inline bool xe_device_is_sriov_pf(const struct xe_device *xe) +{ + return IS_ENABLED(CONFIG_PCI_IOV) && + xe_device_sriov_mode(xe) == XE_SRIOV_MODE_PF; +} + +static inline bool xe_device_is_sriov_vf(const struct xe_device *xe) +{ + return xe_device_sriov_mode(xe) == XE_SRIOV_MODE_VF; +} + +#define IS_SRIOV_PF(xe) xe_device_is_sriov_pf(xe) +#define IS_SRIOV_VF(xe) xe_device_is_sriov_vf(xe) + +#define IS_SRIOV(xe) (IS_SRIOV_PF(xe) || IS_SRIOV_VF(xe)) + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c new file mode 100644 index 000000000..e9ae9c974 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_packet.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "abi/xe_driver_klvs_abi.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_guc_klv_helpers.h" +#include "xe_sriov_packet.h" +#include "xe_sriov_packet_types.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" +#include "xe_sriov_printk.h" + +static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); + + return &xe->sriov.pf.vfs[vfid].migration.lock; +} + +static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); + lockdep_assert_held(pf_migration_mutex(xe, vfid)); + + return &xe->sriov.pf.vfs[vfid].migration.pending; +} + +static struct xe_sriov_packet ** +pf_pick_descriptor(struct xe_device *xe, unsigned int vfid) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); + lockdep_assert_held(pf_migration_mutex(xe, vfid)); + + return &xe->sriov.pf.vfs[vfid].migration.descriptor; +} + +static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); + lockdep_assert_held(pf_migration_mutex(xe, vfid)); + + return &xe->sriov.pf.vfs[vfid].migration.trailer; +} + +static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe, + unsigned int vfid) +{ + struct xe_sriov_packet **data; + + data = pf_pick_descriptor(xe, vfid); + if (*data) + return data; + + data = pf_pick_pending(xe, vfid); + if (!*data) + *data = xe_sriov_pf_migration_save_consume(xe, vfid); + if (*data) + return data; + + data = pf_pick_trailer(xe, vfid); + if (*data) + return data; + + return NULL; +} + +static bool pkt_needs_bo(struct xe_sriov_packet *data) +{ + return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM; +} + +/** + * xe_sriov_packet_alloc() - Allocate migration data packet + * @xe: the &xe_device + * + * Only allocates the "outer" structure, without initializing the migration + * data backing storage. + * + * Return: Pointer to &xe_sriov_packet on success, + * NULL in case of error. + */ +struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe) +{ + struct xe_sriov_packet *data; + + data = kzalloc_obj(*data); + if (!data) + return NULL; + + data->xe = xe; + data->hdr_remaining = sizeof(data->hdr); + + return data; +} + +/** + * xe_sriov_packet_free() - Free migration data packet. + * @data: the &xe_sriov_packet + */ +void xe_sriov_packet_free(struct xe_sriov_packet *data) +{ + if (IS_ERR_OR_NULL(data)) + return; + + if (pkt_needs_bo(data)) + xe_bo_unpin_map_no_vm(data->bo); + else + kvfree(data->buff); + + kfree(data); +} + +static int pkt_init(struct xe_sriov_packet *data) +{ + struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id); + + if (!gt) + return -EINVAL; + + if (data->hdr.size == 0) + return 0; + + if (pkt_needs_bo(data)) { + struct xe_bo *bo; + + bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size), + ttm_bo_type_kernel, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + data->bo = bo; + data->vaddr = bo->vmap.vaddr; + } else { + void *buff = kvzalloc(data->hdr.size, GFP_KERNEL); + + if (!buff) + return -ENOMEM; + + data->buff = buff; + data->vaddr = buff; + } + + return 0; +} + +#define XE_SRIOV_PACKET_SUPPORTED_VERSION 1 + +/** + * xe_sriov_packet_init() - Initialize migration packet header and backing storage. + * @data: the &xe_sriov_packet + * @tile_id: tile identifier + * @gt_id: GT identifier + * @type: &xe_sriov_packet_type + * @offset: offset of data packet payload (within wider resource) + * @size: size of data packet payload + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id, + enum xe_sriov_packet_type type, loff_t offset, size_t size) +{ + data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION; + data->hdr.type = type; + data->hdr.tile_id = tile_id; + data->hdr.gt_id = gt_id; + data->hdr.offset = offset; + data->hdr.size = size; + data->remaining = size; + + return pkt_init(data); +} + +/** + * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header. + * @data: the &xe_sriov_packet + * + * Header data is expected to be filled prior to calling this function. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data) +{ + xe_assert(data->xe, !data->hdr_remaining); + + if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION) + return -EINVAL; + + data->remaining = data->hdr.size; + + return pkt_init(data); +} + +static ssize_t pkt_hdr_read(struct xe_sriov_packet *data, + char __user *buf, size_t len) +{ + loff_t offset = sizeof(data->hdr) - data->hdr_remaining; + + if (!data->hdr_remaining) + return -EINVAL; + + if (len > data->hdr_remaining) + len = data->hdr_remaining; + + if (copy_to_user(buf, (void *)&data->hdr + offset, len)) + return -EFAULT; + + data->hdr_remaining -= len; + + return len; +} + +static ssize_t pkt_data_read(struct xe_sriov_packet *data, + char __user *buf, size_t len) +{ + if (len > data->remaining) + len = data->remaining; + + if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len)) + return -EFAULT; + + data->remaining -= len; + + return len; +} + +static ssize_t pkt_read_single(struct xe_sriov_packet **data, + unsigned int vfid, char __user *buf, size_t len) +{ + ssize_t copied = 0; + + if ((*data)->hdr_remaining) + copied = pkt_hdr_read(*data, buf, len); + else + copied = pkt_data_read(*data, buf, len); + + if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) { + xe_sriov_packet_free(*data); + *data = NULL; + } + + return copied; +} + +/** + * xe_sriov_packet_read_single() - Read migration data from a single packet. + * @xe: the &xe_device + * @vfid: the VF identifier + * @buf: start address of userspace buffer + * @len: requested read size from userspace + * + * Return: number of bytes that has been successfully read, + * 0 if no more migration data is available, + * -errno on failure. + */ +ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len) +{ + struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid); + + if (!data) + return -ENODATA; + if (IS_ERR(*data)) + return PTR_ERR(*data); + + return pkt_read_single(data, vfid, buf, len); +} + +static ssize_t pkt_hdr_write(struct xe_sriov_packet *data, + const char __user *buf, size_t len) +{ + loff_t offset = sizeof(data->hdr) - data->hdr_remaining; + int ret; + + if (len > data->hdr_remaining) + len = data->hdr_remaining; + + if (copy_from_user((void *)&data->hdr + offset, buf, len)) + return -EFAULT; + + data->hdr_remaining -= len; + + if (!data->hdr_remaining) { + ret = xe_sriov_packet_init_from_hdr(data); + if (ret) + return ret; + } + + return len; +} + +static ssize_t pkt_data_write(struct xe_sriov_packet *data, + const char __user *buf, size_t len) +{ + if (len > data->remaining) + len = data->remaining; + + if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len)) + return -EFAULT; + + data->remaining -= len; + + return len; +} + +/** + * xe_sriov_packet_write_single() - Write migration data to a single packet. + * @xe: the &xe_device + * @vfid: the VF identifier + * @buf: start address of userspace buffer + * @len: requested write size from userspace + * + * Return: number of bytes that has been successfully written, + * -errno on failure. + */ +ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len) +{ + struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); + int ret; + ssize_t copied; + + if (IS_ERR_OR_NULL(*data)) { + *data = xe_sriov_packet_alloc(xe); + if (!*data) + return -ENOMEM; + } + + if ((*data)->hdr_remaining) + copied = pkt_hdr_write(*data, buf, len); + else + copied = pkt_data_write(*data, buf, len); + + if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) { + ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); + if (ret) { + xe_sriov_packet_free(*data); + *data = NULL; + + return ret; + } + + *data = NULL; + } + + return copied; +} + +#define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \ + GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN) +static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid) +{ + struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid); + struct xe_sriov_packet *data; + u32 *klvs, *end; + int ret; + + data = xe_sriov_packet_alloc(xe); + if (!data) + return -ENOMEM; + + ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR, + 0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32)); + if (ret) { + xe_sriov_packet_free(data); + return ret; + } + + klvs = data->vaddr; + end = klvs + MIGRATION_DESCRIPTOR_DWORDS; + + klvs = xe_guc_klv_encode_u32(klvs, end - klvs, + MIGRATION_KLV_DEVICE_DEVID_KEY, + xe->info.devid); + klvs = xe_guc_klv_encode_u32(klvs, end - klvs, + MIGRATION_KLV_DEVICE_REVID_KEY, + xe->info.revid); + xe_assert(xe, !IS_ERR(klvs)); + xe_assert(xe, klvs == end); + + *desc = data; + + return 0; +} + +static int descriptor_decoder(void *arg, u16 key, u16 len, const u32 *value) +{ + struct xe_device *xe = arg; + + xe_sriov_dbg_verbose(xe, "found KLV %#x %s\n", key, xe_guc_klv_key_to_string(key)); + + switch (key) { + case MIGRATION_KLV_DEVICE_DEVID_KEY: + if (*value != xe->info.devid) { + xe_sriov_warn(xe, "Aborting migration, devid mismatch %#06x!=%#06x\n", + *value, xe->info.devid); + return -ENODEV; + } + break; + case MIGRATION_KLV_DEVICE_REVID_KEY: + if (*value != xe->info.revid) { + xe_sriov_warn(xe, "Aborting migration, revid mismatch %#06x!=%#06x\n", + *value, xe->info.revid); + return -ENODEV; + } + break; + default: + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + struct drm_printer p = xe_dbg_printer(xe); + + xe_sriov_dbg(xe, "unexpected KLV %#x in descriptor!\n", key); + xe_guc_klv_print_one(key, len, value, &p); + } + return 0; + } + return 1; +} + +/** + * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet. + * @xe: the &xe_device + * @vfid: the VF identifier + * @data: the &xe_sriov_packet containing the descriptor + * + * The descriptor uses the same KLV format as GuC, and contains metadata used for + * checking migration data compatibility. + * + * Return: 0 on success, -errno on failure. + */ +int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid, + struct xe_sriov_packet *data) +{ + u32 num_dwords = data->hdr.size / sizeof(u32); + u32 *klvs = data->vaddr; + int ret; + + xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR); + + if (data->hdr.size % sizeof(u32)) { + xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n", + data->hdr.size); + return -EINVAL; + } + + ret = xe_guc_klv_count(klvs, num_dwords); + if (ret < 0) { + xe_sriov_warn(xe, "Aborting migration, corrupted descriptor KLVs (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + ret = xe_guc_klv_parser(klvs, num_dwords, xe, descriptor_decoder); + if (ret < 0) { + xe_sriov_warn(xe, "Aborting migration, descriptor parsing failed (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + return 0; +} + +static void pf_pending_init(struct xe_device *xe, unsigned int vfid) +{ + struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); + + *data = NULL; +} + +#define MIGRATION_TRAILER_SIZE 0 +static int pf_trailer_init(struct xe_device *xe, unsigned int vfid) +{ + struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid); + struct xe_sriov_packet *data; + int ret; + + data = xe_sriov_packet_alloc(xe); + if (!data) + return -ENOMEM; + + ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER, + 0, MIGRATION_TRAILER_SIZE); + if (ret) { + xe_sriov_packet_free(data); + return ret; + } + + *trailer = data; + + return 0; +} + +/** + * xe_sriov_packet_save_init() - Initialize the pending save migration packets. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * Return: 0 on success, -errno on failure. + */ +int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid) +{ + int ret; + + scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) { + ret = pf_descriptor_init(xe, vfid); + if (ret) + return ret; + + ret = pf_trailer_init(xe, vfid); + if (ret) + return ret; + + pf_pending_init(xe, vfid); + } + + return 0; +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_sriov_packet_kunit.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.h b/drivers/gpu/drm/xe/xe_sriov_packet.h new file mode 100644 index 000000000..2731e52cf --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_packet.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PACKET_H_ +#define _XE_SRIOV_PACKET_H_ + +#include + +struct xe_device; +struct xe_sriov_packet; +enum xe_sriov_packet_type; + +struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe); +void xe_sriov_packet_free(struct xe_sriov_packet *data); + +int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id, + enum xe_sriov_packet_type, loff_t offset, size_t size); +int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data); + +ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len); +ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len); +int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid); +int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid, + struct xe_sriov_packet *data); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_packet_types.h b/drivers/gpu/drm/xe/xe_sriov_packet_types.h new file mode 100644 index 000000000..078a1c95e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_packet_types.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PACKET_TYPES_H_ +#define _XE_SRIOV_PACKET_TYPES_H_ + +#include + +/** + * enum xe_sriov_packet_type - Xe SR-IOV VF migration data packet type + * @XE_SRIOV_PACKET_TYPE_DESCRIPTOR: Descriptor with VF device metadata + * @XE_SRIOV_PACKET_TYPE_TRAILER: Trailer indicating end-of-stream + * @XE_SRIOV_PACKET_TYPE_GGTT: Global GTT migration data + * @XE_SRIOV_PACKET_TYPE_MMIO: MMIO registers migration data + * @XE_SRIOV_PACKET_TYPE_GUC: GuC firmware migration data + * @XE_SRIOV_PACKET_TYPE_VRAM: VRAM migration data + */ +enum xe_sriov_packet_type { + /* Skipping 0 to catch uninitialized data */ + XE_SRIOV_PACKET_TYPE_DESCRIPTOR = 1, + XE_SRIOV_PACKET_TYPE_TRAILER, + XE_SRIOV_PACKET_TYPE_GGTT, + XE_SRIOV_PACKET_TYPE_MMIO, + XE_SRIOV_PACKET_TYPE_GUC, + XE_SRIOV_PACKET_TYPE_VRAM, +}; + +/** + * struct xe_sriov_packet_hdr - Xe SR-IOV VF migration data packet header + */ +struct xe_sriov_packet_hdr { + /** @version: migration data protocol version */ + u8 version; + /** @type: migration data type */ + u8 type; + /** @tile_id: migration data tile id */ + u8 tile_id; + /** @gt_id: migration data gt id */ + u8 gt_id; + /** @flags: migration data flags */ + u32 flags; + /** + * @offset: offset into the resource; + * used when multiple packets of given type are used for migration + */ + u64 offset; + /** @size: migration data size */ + u64 size; +} __packed; + +/** + * struct xe_sriov_packet - Xe SR-IOV VF migration data packet + */ +struct xe_sriov_packet { + /** @xe: the PF &xe_device this data packet belongs to */ + struct xe_device *xe; + /** @vaddr: CPU pointer to payload data */ + void *vaddr; + /** @remaining: payload data remaining */ + size_t remaining; + /** @hdr_remaining: header data remaining */ + size_t hdr_remaining; + union { + /** @bo: Buffer object with migration data */ + struct xe_bo *bo; + /** @buff: Buffer with migration data */ + void *buff; + }; + /** @hdr: data packet header */ + struct xe_sriov_packet_hdr hdr; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c new file mode 100644 index 000000000..33bd754d1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf.c @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include +#include + +#include "xe_assert.h" +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_gt_sriov_pf.h" +#include "xe_module.h" +#include "xe_sriov.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" +#include "xe_sriov_pf_service.h" +#include "xe_sriov_pf_sysfs.h" +#include "xe_sriov_printk.h" + +static unsigned int wanted_max_vfs(struct xe_device *xe) +{ + return xe_configfs_get_max_vfs(to_pci_dev(xe->drm.dev)); +} + +static int pf_reduce_totalvfs(struct xe_device *xe, int limit) +{ + struct device *dev = xe->drm.dev; + struct pci_dev *pdev = to_pci_dev(dev); + int err; + + err = pci_sriov_set_totalvfs(pdev, limit); + if (err) + xe_sriov_notice(xe, "Failed to set number of VFs to %d (%pe)\n", + limit, ERR_PTR(err)); + return err; +} + +static bool pf_continue_as_native(struct xe_device *xe, const char *why) +{ + xe_sriov_dbg(xe, "%s, continuing as native\n", why); + pf_reduce_totalvfs(xe, 0); + return false; +} + +/** + * xe_sriov_pf_readiness - Check if PF functionality can be enabled. + * @xe: the &xe_device to check + * + * This function is called as part of the SR-IOV probe to validate if all + * PF prerequisites are satisfied and we can continue with enabling PF mode. + * + * Return: true if the PF mode can be turned on. + */ +bool xe_sriov_pf_readiness(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + struct pci_dev *pdev = to_pci_dev(dev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + int newlimit = min_t(u16, wanted_max_vfs(xe), totalvfs); + + xe_assert(xe, totalvfs <= U16_MAX); + + if (!dev_is_pf(dev)) + return false; + + if (!xe_device_uc_enabled(xe)) + return pf_continue_as_native(xe, "Guc submission disabled"); + + if (!newlimit) + return pf_continue_as_native(xe, "all VFs disabled"); + + pf_reduce_totalvfs(xe, newlimit); + + xe->sriov.pf.device_total_vfs = totalvfs; + xe->sriov.pf.driver_max_vfs = newlimit; + + return true; +} + +/** + * xe_sriov_pf_init_early - Initialize SR-IOV PF specific data. + * @xe: the &xe_device to initialize + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_init_early(struct xe_device *xe) +{ + int err; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + xe->sriov.pf.vfs = drmm_kcalloc(&xe->drm, 1 + xe_sriov_pf_get_totalvfs(xe), + sizeof(*xe->sriov.pf.vfs), GFP_KERNEL); + if (!xe->sriov.pf.vfs) + return -ENOMEM; + + err = drmm_mutex_init(&xe->drm, &xe->sriov.pf.master_lock); + if (err) + return err; + + err = xe_sriov_pf_migration_init(xe); + if (err) + return err; + + xe_guard_init(&xe->sriov.pf.guard_vfs_enabling, "vfs_enabling"); + + xe_sriov_pf_service_init(xe); + + xe_mert_init_early(xe); + + return 0; +} + +/** + * xe_sriov_pf_init_late() - Late initialization of the SR-IOV PF. + * @xe: the &xe_device to initialize + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_init_late(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int err; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_init(gt); + if (err) + return err; + } + + err = xe_sriov_pf_sysfs_init(xe); + if (err) + return err; + + return 0; +} + +/** + * xe_sriov_pf_wait_ready() - Wait until PF is ready to operate. + * @xe: the &xe_device to test + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_wait_ready(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int err; + + if (xe_device_wedged(xe)) + return -ECANCELED; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_wait_ready(gt); + if (err) + return err; + } + + return 0; +} + +/** + * xe_sriov_pf_arm_guard() - Arm the guard for exclusive/lockdown mode. + * @xe: the PF &xe_device + * @guard: the &xe_guard to arm + * @lockdown: arm for lockdown(true) or exclusive(false) mode + * @who: the address of the new owner, or NULL if it's a caller + * + * This function can only be called on PF. + * + * It is a simple wrapper for xe_guard_arm() with additional debug + * messages. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_arm_guard(struct xe_device *xe, struct xe_guard *guard, + bool lockdown, void *who) +{ + void *new_owner = who ?: __builtin_return_address(0); + int err; + + err = xe_guard_arm(guard, lockdown, new_owner); + if (err) { + xe_sriov_dbg(xe, "%s/%s mode denied (%pe) last owner %ps\n", + guard->name, xe_guard_mode_str(lockdown), + ERR_PTR(err), guard->owner); + return err; + } + + xe_sriov_dbg_verbose(xe, "%s/%s by %ps\n", + guard->name, xe_guard_mode_str(lockdown), + new_owner); + return 0; +} + +/** + * xe_sriov_pf_disarm_guard() - Disarm the guard. + * @xe: the PF &xe_device + * @guard: the &xe_guard to disarm + * @lockdown: disarm from lockdown(true) or exclusive(false) mode + * @who: the address of the indirect owner, or NULL if it's a caller + * + * This function can only be called on PF. + * + * It is a simple wrapper for xe_guard_disarm() with additional debug + * messages and xe_assert() to easily catch any illegal calls. + */ +void xe_sriov_pf_disarm_guard(struct xe_device *xe, struct xe_guard *guard, + bool lockdown, void *who) +{ + bool disarmed; + + xe_sriov_dbg_verbose(xe, "%s/%s by %ps\n", + guard->name, xe_guard_mode_str(lockdown), + who ?: __builtin_return_address(0)); + + disarmed = xe_guard_disarm(guard, lockdown); + xe_assert_msg(xe, disarmed, "%s/%s not armed? last owner %ps", + guard->name, xe_guard_mode_str(lockdown), guard->owner); +} + +/** + * xe_sriov_pf_lockdown() - Lockdown the PF to prevent VFs enabling. + * @xe: the PF &xe_device + * + * This function can only be called on PF. + * + * Once the PF is locked down, it will not enable VFs. + * If VFs are already enabled, the -EBUSY will be returned. + * To allow the PF enable VFs again call xe_sriov_pf_end_lockdown(). + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_lockdown(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + return xe_sriov_pf_arm_guard(xe, &xe->sriov.pf.guard_vfs_enabling, true, + __builtin_return_address(0)); +} + +/** + * xe_sriov_pf_end_lockdown() - Allow the PF to enable VFs again. + * @xe: the PF &xe_device + * + * This function can only be called on PF. + * See xe_sriov_pf_lockdown() for details. + */ +void xe_sriov_pf_end_lockdown(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + xe_sriov_pf_disarm_guard(xe, &xe->sriov.pf.guard_vfs_enabling, true, + __builtin_return_address(0)); +} + +/** + * xe_sriov_pf_print_vfs_summary - Print SR-IOV PF information. + * @xe: the &xe_device to print info from + * @p: the &drm_printer + * + * Print SR-IOV PF related information into provided DRM printer. + */ +void xe_sriov_pf_print_vfs_summary(struct xe_device *xe, struct drm_printer *p) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + xe_assert(xe, IS_SRIOV_PF(xe)); + + drm_printf(p, "total: %u\n", xe->sriov.pf.device_total_vfs); + drm_printf(p, "supported: %u\n", xe->sriov.pf.driver_max_vfs); + drm_printf(p, "enabled: %u\n", pci_num_vf(pdev)); +} diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.h b/drivers/gpu/drm/xe/xe_sriov_pf.h new file mode 100644 index 000000000..b4d050ad5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_H_ +#define _XE_SRIOV_PF_H_ + +#include + +struct dentry; +struct drm_printer; +struct xe_device; + +#ifdef CONFIG_PCI_IOV +bool xe_sriov_pf_readiness(struct xe_device *xe); +int xe_sriov_pf_init_early(struct xe_device *xe); +int xe_sriov_pf_init_late(struct xe_device *xe); +int xe_sriov_pf_wait_ready(struct xe_device *xe); +int xe_sriov_pf_lockdown(struct xe_device *xe); +void xe_sriov_pf_end_lockdown(struct xe_device *xe); +void xe_sriov_pf_print_vfs_summary(struct xe_device *xe, struct drm_printer *p); +#else +static inline bool xe_sriov_pf_readiness(struct xe_device *xe) { return false; } +static inline int xe_sriov_pf_init_early(struct xe_device *xe) { return 0; } +static inline int xe_sriov_pf_init_late(struct xe_device *xe) { return 0; } +static inline int xe_sriov_pf_lockdown(struct xe_device *xe) { return 0; } +static inline void xe_sriov_pf_end_lockdown(struct xe_device *xe) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_control.c b/drivers/gpu/drm/xe/xe_sriov_pf_control.c new file mode 100644 index 000000000..15b4341d7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_control.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_sriov_packet.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_printk.h" + +/** + * xe_sriov_pf_control_pause_vf() - Pause a VF on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier (can't be 0 == PFID) + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_pause_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_pause_vf(gt, vfid); + result = result ? -EUCLEAN : err; + } + + if (result) + return result; + + xe_sriov_info(xe, "VF%u paused!\n", vfid); + return 0; +} + +/** + * xe_sriov_pf_control_resume_vf() - Resume a VF on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_resume_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_resume_vf(gt, vfid); + result = result ? -EUCLEAN : err; + } + + if (result) + return result; + + xe_sriov_info(xe, "VF%u resumed!\n", vfid); + return 0; +} + +/** + * xe_sriov_pf_control_stop_vf - Stop a VF on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_stop_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_stop_vf(gt, vfid); + result = result ? -EUCLEAN : err; + } + + if (result) + return result; + + xe_sriov_info(xe, "VF%u stopped!\n", vfid); + return 0; +} + +/** + * xe_sriov_pf_control_reset_vf() - Perform a VF reset (FLR). + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_reset_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_trigger_flr(gt, vfid); + result = result ? -EUCLEAN : err; + } + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_wait_flr(gt, vfid); + result = result ? -EUCLEAN : err; + } + + return result; +} + +/** + * xe_sriov_pf_control_prepare_flr() - Notify PF that VF FLR prepare has started. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_prepare_flr(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_prepare_flr(gt, vfid); + result = result ? -EUCLEAN : err; + } + + return result; +} + +/** + * xe_sriov_pf_control_wait_flr() - Wait for a VF reset (FLR) to complete. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_wait_flr(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_control_wait_flr(gt, vfid); + result = result ? -EUCLEAN : err; + } + + return result; +} + +/** + * xe_sriov_pf_control_sync_flr() - Synchronize a VF FLR between all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_sync_flr(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int ret; + + for_each_gt(gt, xe, id) { + ret = xe_gt_sriov_pf_control_sync_flr(gt, vfid, false); + if (ret < 0) + return ret; + } + for_each_gt(gt, xe, id) { + ret = xe_gt_sriov_pf_control_sync_flr(gt, vfid, true); + if (ret < 0) + return ret; + } + + return 0; +} + +/** + * xe_sriov_pf_control_trigger_save_vf() - Start VF migration data SAVE sequence on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_trigger_save_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int ret; + + ret = xe_sriov_packet_save_init(xe, vfid); + if (ret) + return ret; + + for_each_gt(gt, xe, id) { + xe_gt_sriov_pf_migration_save_init(gt, vfid); + + ret = xe_gt_sriov_pf_control_trigger_save_vf(gt, vfid); + if (ret) + return ret; + } + + return 0; +} + +/** + * xe_sriov_pf_control_finish_save_vf() - Complete VF migration data SAVE sequence on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_finish_save_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int ret; + + for_each_gt(gt, xe, id) { + ret = xe_gt_sriov_pf_control_finish_save_vf(gt, vfid); + if (ret) + break; + } + + return ret; +} + +/** + * xe_sriov_pf_control_trigger_restore_vf() - Start VF migration data RESTORE sequence on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_trigger_restore_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int ret; + + for_each_gt(gt, xe, id) { + ret = xe_gt_sriov_pf_control_trigger_restore_vf(gt, vfid); + if (ret) + return ret; + } + + return ret; +} + +/** + * xe_sriov_pf_control_finish_restore_vf() - Complete VF migration data RESTORE sequence on all GTs. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_control_finish_restore_vf(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + unsigned int id; + int ret; + + for_each_gt(gt, xe, id) { + ret = xe_gt_sriov_pf_control_finish_restore_vf(gt, vfid); + if (ret) + break; + } + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_control.h b/drivers/gpu/drm/xe/xe_sriov_pf_control.h new file mode 100644 index 000000000..74981a67d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_control.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_CONTROL_H_ +#define _XE_SRIOV_PF_CONTROL_H_ + +struct xe_device; + +int xe_sriov_pf_control_pause_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_resume_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_stop_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_reset_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_prepare_flr(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_wait_flr(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_sync_flr(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_trigger_save_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_finish_save_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_trigger_restore_vf(struct xe_device *xe, unsigned int vfid); +int xe_sriov_pf_control_finish_restore_vf(struct xe_device *xe, unsigned int vfid); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c new file mode 100644 index 000000000..81b377830 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_pm.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_debugfs.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" +#include "xe_sriov_pf_provision.h" +#include "xe_sriov_pf_service.h" +#include "xe_tile_sriov_pf_debugfs.h" + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov # d_inode->i_private = (xe_device*) + * │ ├── pf # d_inode->i_private = (xe_device*) + * │ ├── vf1 # d_inode->i_private = VFID(1) + * : : + * │ ├── vfN # d_inode->i_private = VFID(N) + */ + +static void *extract_priv(struct dentry *d) +{ + return d->d_inode->i_private; +} + +static struct xe_device *extract_xe(struct dentry *d) +{ + return extract_priv(d->d_parent); +} + +static unsigned int extract_vfid(struct dentry *d) +{ + void *p = extract_priv(d); + + return p == extract_xe(d) ? PFID : (uintptr_t)p; +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * │ ├── restore_auto_provisioning + * │ : + * │ ├── pf/ + * │ ├── vf1 + * │ │ ├── ... + */ + +static ssize_t from_file_write_to_xe_call(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos, + int (*call)(struct xe_device *)) +{ + struct dentry *dent = file_dentry(file); + struct xe_device *xe = extract_xe(dent); + bool yes; + int ret; + + if (*ppos) + return -EINVAL; + ret = kstrtobool_from_user(userbuf, count, &yes); + if (ret < 0) + return ret; + if (yes) { + guard(xe_pm_runtime)(xe); + ret = call(xe); + } + if (ret < 0) + return ret; + return count; +} + +#define DEFINE_SRIOV_ATTRIBUTE(OP) \ +static int OP##_show(struct seq_file *s, void *unused) \ +{ \ + return 0; \ +} \ +static ssize_t OP##_write(struct file *file, const char __user *userbuf, \ + size_t count, loff_t *ppos) \ +{ \ + return from_file_write_to_xe_call(file, userbuf, count, ppos, \ + xe_sriov_pf_##OP); \ +} \ +DEFINE_SHOW_STORE_ATTRIBUTE(OP) + +static inline int xe_sriov_pf_restore_auto_provisioning(struct xe_device *xe) +{ + return xe_sriov_pf_provision_set_mode(xe, XE_SRIOV_PROVISIONING_MODE_AUTO); +} + +DEFINE_SRIOV_ATTRIBUTE(restore_auto_provisioning); + +static int lockdown_vfs_enabling_open(struct inode *inode, struct file *file) +{ + struct dentry *dent = file_dentry(file); + struct xe_device *xe = extract_xe(dent); + ssize_t ret; + + ret = xe_sriov_pf_lockdown(xe); + if (ret < 0) + return ret; + + file->private_data = xe; + return nonseekable_open(inode, file); +} + +static int lockdown_vfs_enabling_release(struct inode *inode, struct file *file) +{ + struct xe_device *xe = file->private_data; + + xe_sriov_pf_end_lockdown(xe); + return 0; +} + +static const struct file_operations lockdown_vfs_enabling_fops = { + .owner = THIS_MODULE, + .open = lockdown_vfs_enabling_open, + .release = lockdown_vfs_enabling_release, +}; + +static void pf_populate_root(struct xe_device *xe, struct dentry *dent) +{ + debugfs_create_file("restore_auto_provisioning", 0200, dent, xe, + &restore_auto_provisioning_fops); + debugfs_create_file("lockdown_vfs_enabling", 0400, dent, xe, + &lockdown_vfs_enabling_fops); +} + +static int simple_show(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + struct drm_info_node *node = m->private; + struct dentry *parent = node->dent->d_parent; + struct xe_device *xe = parent->d_inode->i_private; + void (*print)(struct xe_device *, struct drm_printer *) = node->info_ent->data; + + print(xe, &p); + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + { .name = "vfs", .show = simple_show, .data = xe_sriov_pf_print_vfs_summary }, + { .name = "versions", .show = simple_show, .data = xe_sriov_pf_service_print_versions }, +}; + +static void pf_populate_pf(struct xe_device *xe, struct dentry *pfdent) +{ + struct drm_minor *minor = xe->drm.primary; + + drm_debugfs_create_files(debugfs_list, ARRAY_SIZE(debugfs_list), pfdent, minor); +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * │ ├── vf1 + * │ │ ├── migration_data + * │ │ ├── pause + * │ │ ├── reset + * │ │ ├── resume + * │ │ ├── stop + * │ │ ├── save + * │ │ ├── restore + * │ │ : + * │ ├── vf2 + * │ │ ├── ... + */ + +static int from_file_read_to_vf_call(struct seq_file *s, + int (*call)(struct xe_device *, unsigned int)) +{ + struct dentry *dent = file_dentry(s->file)->d_parent; + struct xe_device *xe = extract_xe(dent); + unsigned int vfid = extract_vfid(dent); + int ret; + + xe_pm_runtime_get(xe); + ret = call(xe, vfid); + xe_pm_runtime_put(xe); + + if (ret < 0) + return ret; + + return 0; +} + +static ssize_t from_file_write_to_vf_call(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos, + int (*call)(struct xe_device *, unsigned int)) +{ + struct dentry *dent = file_dentry(file)->d_parent; + struct xe_device *xe = extract_xe(dent); + unsigned int vfid = extract_vfid(dent); + bool yes; + int ret; + + if (*ppos) + return -EINVAL; + ret = kstrtobool_from_user(userbuf, count, &yes); + if (ret < 0) + return ret; + if (yes) { + guard(xe_pm_runtime)(xe); + ret = call(xe, vfid); + } + if (ret < 0) + return ret; + return count; +} + +#define DEFINE_VF_CONTROL_ATTRIBUTE(OP) \ +static int OP##_show(struct seq_file *s, void *unused) \ +{ \ + return 0; \ +} \ +static ssize_t OP##_write(struct file *file, const char __user *userbuf, \ + size_t count, loff_t *ppos) \ +{ \ + return from_file_write_to_vf_call(file, userbuf, count, ppos, \ + xe_sriov_pf_control_##OP); \ +} \ +DEFINE_SHOW_STORE_ATTRIBUTE(OP) + +#define DEFINE_VF_CONTROL_ATTRIBUTE_RW(OP) \ +static int OP##_show(struct seq_file *s, void *unused) \ +{ \ + return from_file_read_to_vf_call(s, \ + xe_sriov_pf_control_finish_##OP); \ +} \ +static ssize_t OP##_write(struct file *file, const char __user *userbuf, \ + size_t count, loff_t *ppos) \ +{ \ + return from_file_write_to_vf_call(file, userbuf, count, ppos, \ + xe_sriov_pf_control_trigger_##OP); \ +} \ +DEFINE_SHOW_STORE_ATTRIBUTE(OP) + +DEFINE_VF_CONTROL_ATTRIBUTE(pause_vf); +DEFINE_VF_CONTROL_ATTRIBUTE(resume_vf); +DEFINE_VF_CONTROL_ATTRIBUTE(stop_vf); +DEFINE_VF_CONTROL_ATTRIBUTE(reset_vf); +DEFINE_VF_CONTROL_ATTRIBUTE_RW(save_vf); +DEFINE_VF_CONTROL_ATTRIBUTE_RW(restore_vf); + +static ssize_t data_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) +{ + struct dentry *dent = file_dentry(file)->d_parent; + struct xe_device *xe = extract_xe(dent); + unsigned int vfid = extract_vfid(dent); + + if (*pos) + return -ESPIPE; + + return xe_sriov_pf_migration_write(xe, vfid, buf, count); +} + +static ssize_t data_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) +{ + struct dentry *dent = file_dentry(file)->d_parent; + struct xe_device *xe = extract_xe(dent); + unsigned int vfid = extract_vfid(dent); + + if (*ppos) + return -ESPIPE; + + return xe_sriov_pf_migration_read(xe, vfid, buf, count); +} + +static const struct file_operations data_vf_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = data_write, + .read = data_read, + .llseek = default_llseek, +}; + +static ssize_t size_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) +{ + struct dentry *dent = file_dentry(file)->d_parent; + struct xe_device *xe = extract_xe(dent); + unsigned int vfid = extract_vfid(dent); + char buf[21]; + ssize_t ret; + int len; + + xe_pm_runtime_get(xe); + ret = xe_sriov_pf_migration_size(xe, vfid); + xe_pm_runtime_put(xe); + if (ret < 0) + return ret; + + len = scnprintf(buf, sizeof(buf), "%zd\n", ret); + + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations size_vf_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = size_read, + .llseek = default_llseek, +}; + +static void pf_populate_vf(struct xe_device *xe, struct dentry *vfdent) +{ + debugfs_create_file("pause", 0200, vfdent, xe, &pause_vf_fops); + debugfs_create_file("resume", 0200, vfdent, xe, &resume_vf_fops); + debugfs_create_file("stop", 0200, vfdent, xe, &stop_vf_fops); + debugfs_create_file("reset", 0200, vfdent, xe, &reset_vf_fops); + debugfs_create_file("save", 0600, vfdent, xe, &save_vf_fops); + debugfs_create_file("restore", 0600, vfdent, xe, &restore_vf_fops); + debugfs_create_file("migration_data", 0600, vfdent, xe, &data_vf_fops); + debugfs_create_file("migration_size", 0400, vfdent, xe, &size_vf_fops); +} + +static void pf_populate_with_tiles(struct xe_device *xe, struct dentry *dent, unsigned int vfid) +{ + struct xe_tile *tile; + unsigned int id; + + for_each_tile(tile, xe, id) + xe_tile_sriov_pf_debugfs_populate(tile, dent, vfid); +} + +/** + * xe_sriov_pf_debugfs_register - Register PF debugfs attributes. + * @xe: the &xe_device + * @root: the root &dentry + * + * Create separate directory that will contain all SR-IOV related files, + * organized per each SR-IOV function (PF, VF1, VF2, ..., VFn). + */ +void xe_sriov_pf_debugfs_register(struct xe_device *xe, struct dentry *root) +{ + int totalvfs = xe_sriov_pf_get_totalvfs(xe); + struct dentry *pfdent; + struct dentry *vfdent; + struct dentry *dent; + char vfname[16]; /* should be more than enough for "vf%u\0" and VFID(UINT_MAX) */ + unsigned int n; + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov # d_inode->i_private = (xe_device*) + * │ ├── ... + */ + dent = debugfs_create_dir("sriov", root); + if (IS_ERR(dent)) + return; + dent->d_inode->i_private = xe; + + pf_populate_root(xe, dent); + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov # d_inode->i_private = (xe_device*) + * │ ├── pf # d_inode->i_private = (xe_device*) + * │ │ ├── ... + */ + pfdent = debugfs_create_dir("pf", dent); + if (IS_ERR(pfdent)) + return; + pfdent->d_inode->i_private = xe; + + pf_populate_pf(xe, pfdent); + pf_populate_with_tiles(xe, pfdent, PFID); + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov # d_inode->i_private = (xe_device*) + * │ ├── vf1 # d_inode->i_private = VFID(1) + * │ ├── vf2 # d_inode->i_private = VFID(2) + * │ ├── ... + */ + for (n = 1; n <= totalvfs; n++) { + snprintf(vfname, sizeof(vfname), "vf%u", VFID(n)); + vfdent = debugfs_create_dir(vfname, dent); + if (IS_ERR(vfdent)) + return; + vfdent->d_inode->i_private = (void *)(uintptr_t)VFID(n); + + pf_populate_vf(xe, vfdent); + pf_populate_with_tiles(xe, vfdent, VFID(n)); + } +} diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.h b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.h new file mode 100644 index 000000000..93db13585 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_DEBUGFS_H_ +#define _XE_SRIOV_PF_DEBUGFS_H_ + +struct dentry; +struct xe_device; + +#ifdef CONFIG_PCI_IOV +void xe_sriov_pf_debugfs_register(struct xe_device *xe, struct dentry *root); +#else +static inline void xe_sriov_pf_debugfs_register(struct xe_device *xe, struct dentry *root) { } +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h new file mode 100644 index 000000000..19f6f8331 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_HELPERS_H_ +#define _XE_SRIOV_PF_HELPERS_H_ + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_sriov.h" +#include "xe_sriov_types.h" + +/** + * xe_sriov_pf_assert_vfid() - warn if &id is not a supported VF number when debugging. + * @xe: the PF &xe_device to assert on + * @vfid: the VF number to assert + * + * Assert that &xe represents the Physical Function (PF) device and provided &vfid + * is within a range of supported VF numbers (up to maximum number of VFs that + * driver can support, including VF0 that represents the PF itself). + * + * Note: Effective only on debug builds. See `Xe Asserts`_ for more information. + */ +#define xe_sriov_pf_assert_vfid(xe, vfid) \ + xe_assert((xe), (vfid) <= xe_sriov_pf_get_totalvfs(xe)) + +/** + * xe_sriov_pf_get_totalvfs() - Get maximum number of VFs that driver can support. + * @xe: the &xe_device to query (shall be PF) + * + * Return: Maximum number of VFs that this PF driver supports. + */ +static inline int xe_sriov_pf_get_totalvfs(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + return xe->sriov.pf.driver_max_vfs; +} + +/** + * xe_sriov_pf_num_vfs() - Number of enabled VFs on the PF. + * @xe: the PF &xe_device + * + * Return: Number of enabled VFs on the PF. + */ +static inline unsigned int xe_sriov_pf_num_vfs(const struct xe_device *xe) +{ + return pci_num_vf(to_pci_dev(xe->drm.dev)); +} + +/** + * xe_sriov_pf_admin_only() - Check if PF is mainly used for VFs administration. + * @xe: the PF &xe_device + * + * Return: True if PF is mainly used for VFs administration. + */ +static inline bool xe_sriov_pf_admin_only(const struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + return xe_device_is_admin_only(xe); +} + +static inline struct mutex *xe_sriov_pf_master_mutex(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + return &xe->sriov.pf.master_lock; +} + +int xe_sriov_pf_arm_guard(struct xe_device *xe, struct xe_guard *guard, + bool write, void *who); +void xe_sriov_pf_disarm_guard(struct xe_device *xe, struct xe_guard *guard, + bool write, void *who); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c new file mode 100644 index 000000000..150a24111 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "xe_device.h" +#include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_migration.h" +#include "xe_pm.h" +#include "xe_sriov.h" +#include "xe_sriov_packet.h" +#include "xe_sriov_packet_types.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" +#include "xe_sriov_printk.h" + +static struct xe_sriov_migration_state *pf_pick_migration(struct xe_device *xe, unsigned int vfid) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); + + return &xe->sriov.pf.vfs[vfid].migration; +} + +/** + * xe_sriov_pf_migration_waitqueue() - Get waitqueue for migration. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * Return: pointer to the migration waitqueue. + */ +wait_queue_head_t *xe_sriov_pf_migration_waitqueue(struct xe_device *xe, unsigned int vfid) +{ + return &pf_pick_migration(xe, vfid)->wq; +} + +/** + * xe_sriov_pf_migration_supported() - Check if SR-IOV VF migration is supported by the device + * @xe: the &xe_device + * + * Return: true if migration is supported, false otherwise + */ +bool xe_sriov_pf_migration_supported(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + return IS_ENABLED(CONFIG_DRM_XE_DEBUG) || !xe->sriov.pf.migration.disabled; +} + +/** + * xe_sriov_pf_migration_disable() - Turn off SR-IOV VF migration support on PF. + * @xe: the &xe_device instance. + * @fmt: format string for the log message, to be combined with following VAs. + */ +void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...) +{ + struct va_format vaf; + va_list va_args; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + va_start(va_args, fmt); + vaf.fmt = fmt; + vaf.va = &va_args; + xe_sriov_notice(xe, "migration %s: %pV\n", + IS_ENABLED(CONFIG_DRM_XE_DEBUG) ? + "missing prerequisite" : "disabled", + &vaf); + va_end(va_args); + + xe->sriov.pf.migration.disabled = true; +} + +static void pf_migration_check_support(struct xe_device *xe) +{ + if (!xe_device_has_memirq(xe)) + xe_sriov_pf_migration_disable(xe, "requires memory-based IRQ support"); +} + +static void pf_migration_cleanup(void *arg) +{ + struct xe_sriov_migration_state *migration = arg; + + xe_sriov_packet_free(migration->pending); + xe_sriov_packet_free(migration->trailer); + xe_sriov_packet_free(migration->descriptor); +} + +/** + * xe_sriov_pf_migration_init() - Initialize support for SR-IOV VF migration. + * @xe: the &xe_device + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_migration_init(struct xe_device *xe) +{ + unsigned int n, totalvfs; + int err; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + pf_migration_check_support(xe); + + if (!xe_sriov_pf_migration_supported(xe)) + return 0; + + totalvfs = xe_sriov_pf_get_totalvfs(xe); + for (n = 1; n <= totalvfs; n++) { + struct xe_sriov_migration_state *migration = pf_pick_migration(xe, n); + + err = drmm_mutex_init(&xe->drm, &migration->lock); + if (err) + return err; + + init_waitqueue_head(&migration->wq); + + err = devm_add_action_or_reset(xe->drm.dev, pf_migration_cleanup, migration); + if (err) + return err; + } + + return 0; +} + +static bool pf_migration_data_ready(struct xe_device *xe, unsigned int vfid) +{ + struct xe_gt *gt; + u8 gt_id; + + for_each_gt(gt, xe, gt_id) { + if (xe_gt_sriov_pf_control_check_save_failed(gt, vfid) || + xe_gt_sriov_pf_control_check_save_data_done(gt, vfid) || + !xe_gt_sriov_pf_migration_ring_empty(gt, vfid)) + return true; + } + + return false; +} + +static struct xe_sriov_packet * +pf_migration_consume(struct xe_device *xe, unsigned int vfid) +{ + struct xe_sriov_packet *data; + bool more_data = false; + struct xe_gt *gt; + u8 gt_id; + + for_each_gt(gt, xe, gt_id) { + data = xe_gt_sriov_pf_migration_save_consume(gt, vfid); + if (!data) + continue; + if (!IS_ERR(data) || PTR_ERR(data) != -EAGAIN) + return data; + more_data = true; + } + + if (!more_data) + return NULL; + + return ERR_PTR(-EAGAIN); +} + +/** + * xe_sriov_pf_migration_save_consume() - Consume a VF migration data packet from the device. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * Called by the save migration data consumer (userspace) when + * processing migration data. + * If there is no migration data to process, wait until more data is available. + * + * Return: Pointer to &xe_sriov_packet on success, + * NULL if ring is empty and no more migration data is expected, + * ERR_PTR value in case of error. + */ +struct xe_sriov_packet * +xe_sriov_pf_migration_save_consume(struct xe_device *xe, unsigned int vfid) +{ + struct xe_sriov_migration_state *migration = pf_pick_migration(xe, vfid); + struct xe_sriov_packet *data; + int ret; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + for (;;) { + data = pf_migration_consume(xe, vfid); + if (PTR_ERR(data) != -EAGAIN) + break; + + ret = wait_event_interruptible(migration->wq, + pf_migration_data_ready(xe, vfid)); + if (ret) + return ERR_PTR(ret); + } + + return data; +} + +static int pf_handle_descriptor(struct xe_device *xe, unsigned int vfid, + struct xe_sriov_packet *data) +{ + int ret; + + if (data->hdr.tile_id != 0 || data->hdr.gt_id != 0) + return -EINVAL; + + ret = xe_sriov_packet_process_descriptor(xe, vfid, data); + if (ret) + return ret; + + xe_sriov_packet_free(data); + + return 0; +} + +static int pf_handle_trailer(struct xe_device *xe, unsigned int vfid, + struct xe_sriov_packet *data) +{ + struct xe_gt *gt; + u8 gt_id; + + if (data->hdr.tile_id != 0 || data->hdr.gt_id != 0) + return -EINVAL; + if (data->hdr.offset != 0 || data->hdr.size != 0 || data->buff || data->bo) + return -EINVAL; + + xe_sriov_packet_free(data); + + for_each_gt(gt, xe, gt_id) + xe_gt_sriov_pf_control_restore_data_done(gt, vfid); + + return 0; +} + +/** + * xe_sriov_pf_migration_restore_produce() - Produce a VF migration data packet to the device. + * @xe: the &xe_device + * @vfid: the VF identifier + * @data: Pointer to &xe_sriov_packet + * + * Called by the restore migration data producer (userspace) when processing + * migration data. + * If the underlying data structure is full, wait until there is space. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_migration_restore_produce(struct xe_device *xe, unsigned int vfid, + struct xe_sriov_packet *data) +{ + struct xe_gt *gt; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR) + return pf_handle_descriptor(xe, vfid, data); + if (data->hdr.type == XE_SRIOV_PACKET_TYPE_TRAILER) + return pf_handle_trailer(xe, vfid, data); + + gt = xe_device_get_gt(xe, data->hdr.gt_id); + if (!gt || data->hdr.tile_id != gt->tile->id || data->hdr.type == 0) { + xe_sriov_err_ratelimited(xe, "Received invalid restore packet for VF%u (type:%u, tile:%u, GT:%u)\n", + vfid, data->hdr.type, data->hdr.tile_id, data->hdr.gt_id); + return -EINVAL; + } + + return xe_gt_sriov_pf_migration_restore_produce(gt, vfid, data); +} + +/** + * xe_sriov_pf_migration_read() - Read migration data from the device. + * @xe: the &xe_device + * @vfid: the VF identifier + * @buf: start address of userspace buffer + * @len: requested read size from userspace + * + * Return: number of bytes that has been successfully read, + * 0 if no more migration data is available, + * -errno on failure. + */ +ssize_t xe_sriov_pf_migration_read(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len) +{ + struct xe_sriov_migration_state *migration = pf_pick_migration(xe, vfid); + ssize_t ret, consumed = 0; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + scoped_cond_guard(mutex_intr, return -EINTR, &migration->lock) { + while (consumed < len) { + ret = xe_sriov_packet_read_single(xe, vfid, buf, len - consumed); + if (ret == -ENODATA) + break; + if (ret < 0) + return ret; + + consumed += ret; + buf += ret; + } + } + + return consumed; +} + +/** + * xe_sriov_pf_migration_write() - Write migration data to the device. + * @xe: the &xe_device + * @vfid: the VF identifier + * @buf: start address of userspace buffer + * @len: requested write size from userspace + * + * Return: number of bytes that has been successfully written, + * -errno on failure. + */ +ssize_t xe_sriov_pf_migration_write(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len) +{ + struct xe_sriov_migration_state *migration = pf_pick_migration(xe, vfid); + ssize_t ret, produced = 0; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + scoped_cond_guard(mutex_intr, return -EINTR, &migration->lock) { + while (produced < len) { + ret = xe_sriov_packet_write_single(xe, vfid, buf, len - produced); + if (ret < 0) + return ret; + + produced += ret; + buf += ret; + } + } + + return produced; +} + +/** + * xe_sriov_pf_migration_size() - Total size of migration data from all components within a device + * @xe: the &xe_device + * @vfid: the VF identifier (can't be 0) + * + * This function is for PF only. + * + * Return: total migration data size in bytes or a negative error code on failure. + */ +ssize_t xe_sriov_pf_migration_size(struct xe_device *xe, unsigned int vfid) +{ + size_t size = 0; + struct xe_gt *gt; + ssize_t ret; + u8 gt_id; + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, vfid); + + for_each_gt(gt, xe, gt_id) { + ret = xe_gt_sriov_pf_migration_size(gt, vfid); + if (ret < 0) + return ret; + + size += ret; + } + + return size; +} diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.h b/drivers/gpu/drm/xe/xe_sriov_pf_migration.h new file mode 100644 index 000000000..f8f408df8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_MIGRATION_H_ +#define _XE_SRIOV_PF_MIGRATION_H_ + +#include +#include + +struct xe_device; +struct xe_sriov_packet; + +int xe_sriov_pf_migration_init(struct xe_device *xe); +bool xe_sriov_pf_migration_supported(struct xe_device *xe); +void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...); +int xe_sriov_pf_migration_restore_produce(struct xe_device *xe, unsigned int vfid, + struct xe_sriov_packet *data); +struct xe_sriov_packet * +xe_sriov_pf_migration_save_consume(struct xe_device *xe, unsigned int vfid); +ssize_t xe_sriov_pf_migration_size(struct xe_device *xe, unsigned int vfid); +wait_queue_head_t *xe_sriov_pf_migration_waitqueue(struct xe_device *xe, unsigned int vfid); + +ssize_t xe_sriov_pf_migration_read(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len); +ssize_t xe_sriov_pf_migration_write(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h new file mode 100644 index 000000000..7d9a8a278 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_MIGRATION_TYPES_H_ +#define _XE_SRIOV_PF_MIGRATION_TYPES_H_ + +#include +#include +#include + +/** + * struct xe_sriov_pf_migration - Xe device level VF migration data + */ +struct xe_sriov_pf_migration { + /** @disabled: indicates whether VF migration feature is disabled */ + bool disabled; +}; + +/** + * struct xe_sriov_migration_state - Per VF device-level migration related data + */ +struct xe_sriov_migration_state { + /** @wq: waitqueue used to avoid busy-waiting for snapshot production/consumption */ + wait_queue_head_t wq; + /** @lock: Mutex protecting the migration data */ + struct mutex lock; + /** @pending: currently processed data packet of VF resource */ + struct xe_sriov_packet *pending; + /** @trailer: data packet used to indicate the end of stream */ + struct xe_sriov_packet *trailer; + /** @descriptor: data packet containing the metadata describing the device */ + struct xe_sriov_packet *descriptor; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c new file mode 100644 index 000000000..0ec7ea83f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_policy.h" +#include "xe_lmtt.h" +#include "xe_sriov.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_provision.h" +#include "xe_sriov_pf_provision_types.h" +#include "xe_sriov_printk.h" + +static const char *mode_to_string(enum xe_sriov_provisioning_mode mode) +{ + switch (mode) { + case XE_SRIOV_PROVISIONING_MODE_AUTO: + return "auto"; + case XE_SRIOV_PROVISIONING_MODE_CUSTOM: + return "custom"; + default: + return ""; + } +} + +static bool pf_auto_provisioning_mode(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + return xe->sriov.pf.provision.mode == XE_SRIOV_PROVISIONING_MODE_AUTO; +} + +static int pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_set_fair_sched(gt, num_vfs); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_fair(gt, VFID(1), num_vfs); + result = result ?: err; + } + + return result; +} + +static void pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + struct xe_gt *gt; + unsigned int id; + unsigned int n; + + for_each_gt(gt, xe, id) + for (n = 1; n <= num_vfs; n++) + xe_gt_sriov_pf_config_release(gt, n, true); +} + +static void pf_unprovision_all_vfs(struct xe_device *xe) +{ + pf_unprovision_vfs(xe, xe_sriov_pf_get_totalvfs(xe)); +} + +/** + * xe_sriov_pf_provision_vfs() - Provision VFs in auto-mode. + * @xe: the PF &xe_device + * @num_vfs: the number of VFs to auto-provision + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (!pf_auto_provisioning_mode(xe)) + return 0; + + return pf_provision_vfs(xe, num_vfs); +} + +/** + * xe_sriov_pf_unprovision_vfs() - Unprovision VFs in auto-mode. + * @xe: the PF &xe_device + * @num_vfs: the number of VFs to unprovision + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (!pf_auto_provisioning_mode(xe)) + return 0; + + pf_unprovision_vfs(xe, num_vfs); + return 0; +} + +static int pf_reprovision_default(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_exec_quantum_locked(gt, PFID, 0); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_preempt_timeout_locked(gt, PFID, 0); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_reprovision_default() - Reprovision default PF in auto-mode. + * @xe: the PF &xe_device + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_reprovision_default(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (!pf_auto_provisioning_mode(xe)) + return 0; + + return pf_reprovision_default(xe); +} + +/** + * xe_sriov_pf_provision_set_mode() - Change VFs provision mode. + * @xe: the PF &xe_device + * @mode: the new VFs provisioning mode + * + * When changing from AUTO to CUSTOM mode, any already allocated VFs resources + * will remain allocated and will not be released upon VFs disabling. + * + * When changing back to AUTO mode, if VFs are not enabled, already allocated + * VFs resources will be immediately released. If VFs are still enabled, such + * mode change is rejected. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_set_mode(struct xe_device *xe, enum xe_sriov_provisioning_mode mode) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (mode == xe->sriov.pf.provision.mode) + return 0; + + if (mode == XE_SRIOV_PROVISIONING_MODE_AUTO) { + if (xe_sriov_pf_num_vfs(xe)) { + xe_sriov_dbg(xe, "can't restore %s: VFs must be disabled!\n", + mode_to_string(mode)); + return -EBUSY; + } + pf_unprovision_all_vfs(xe); + } + + xe_sriov_dbg(xe, "mode %s changed to %s by %ps\n", + mode_to_string(xe->sriov.pf.provision.mode), + mode_to_string(mode), __builtin_return_address(0)); + xe->sriov.pf.provision.mode = mode; + return 0; +} + +/** + * xe_sriov_pf_provision_bulk_apply_eq() - Change execution quantum for all VFs and PF. + * @xe: the PF &xe_device + * @eq: execution quantum in [ms] to set + * + * Change execution quantum (EQ) provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_bulk_apply_eq(struct xe_device *xe, u32 eq) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(gt, eq); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_apply_vf_eq() - Change VF's execution quantum. + * @xe: the PF &xe_device + * @vfid: the VF identifier + * @eq: execution quantum in [ms] to set + * + * Change VF's execution quantum (EQ) provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_apply_vf_eq(struct xe_device *xe, unsigned int vfid, u32 eq) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_set_exec_quantum_locked(gt, vfid, eq); + result = result ?: err; + } + + return result; +} + +static int pf_report_unclean(struct xe_gt *gt, unsigned int vfid, + const char *what, u32 found, u32 expected) +{ + char name[8]; + + xe_sriov_dbg(gt_to_xe(gt), "%s on GT%u has %s=%u (expected %u)\n", + xe_sriov_function_name(vfid, name, sizeof(name)), + gt->info.id, what, found, expected); + return -EUCLEAN; +} + +/** + * xe_sriov_pf_provision_query_vf_eq() - Query VF's execution quantum. + * @xe: the PF &xe_device + * @vfid: the VF identifier + * @eq: placeholder for the returned execution quantum in [ms] + * + * Query VF's execution quantum (EQ) provisioning from all tiles/GTs. + * If values across tiles/GTs are inconsistent then -EUCLEAN error will be returned. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_query_vf_eq(struct xe_device *xe, unsigned int vfid, u32 *eq) +{ + struct xe_gt *gt; + unsigned int id; + int count = 0; + u32 value; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + value = xe_gt_sriov_pf_config_get_exec_quantum_locked(gt, vfid); + if (!count++) + *eq = value; + else if (value != *eq) + return pf_report_unclean(gt, vfid, "EQ", value, *eq); + } + + return !count ? -ENODATA : 0; +} + +/** + * xe_sriov_pf_provision_bulk_apply_pt() - Change preemption timeout for all VFs and PF. + * @xe: the PF &xe_device + * @pt: preemption timeout in [us] to set + * + * Change preemption timeout (PT) provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_bulk_apply_pt(struct xe_device *xe, u32 pt) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(gt, pt); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_apply_vf_pt() - Change VF's preemption timeout. + * @xe: the PF &xe_device + * @vfid: the VF identifier + * @pt: preemption timeout in [us] to set + * + * Change VF's preemption timeout (PT) provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_apply_vf_pt(struct xe_device *xe, unsigned int vfid, u32 pt) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_set_preempt_timeout_locked(gt, vfid, pt); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_query_vf_pt() - Query VF's preemption timeout. + * @xe: the PF &xe_device + * @vfid: the VF identifier + * @pt: placeholder for the returned preemption timeout in [us] + * + * Query VF's preemption timeout (PT) provisioning from all tiles/GTs. + * If values across tiles/GTs are inconsistent then -EUCLEAN error will be returned. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_query_vf_pt(struct xe_device *xe, unsigned int vfid, u32 *pt) +{ + struct xe_gt *gt; + unsigned int id; + int count = 0; + u32 value; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + value = xe_gt_sriov_pf_config_get_preempt_timeout_locked(gt, vfid); + if (!count++) + *pt = value; + else if (value != *pt) + return pf_report_unclean(gt, vfid, "PT", value, *pt); + } + + return !count ? -ENODATA : 0; +} + +/** + * xe_sriov_pf_provision_bulk_apply_priority() - Change scheduling priority of all VFs and PF. + * @xe: the PF &xe_device + * @prio: scheduling priority to set + * + * Change the scheduling priority provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_bulk_apply_priority(struct xe_device *xe, u32 prio) +{ + bool sched_if_idle; + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + /* + * Currently, priority changes that involves VFs are only allowed using + * the 'sched_if_idle' policy KLV, so only LOW and NORMAL are supported. + */ + xe_assert(xe, prio < GUC_SCHED_PRIORITY_HIGH); + sched_if_idle = prio == GUC_SCHED_PRIORITY_NORMAL; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_policy_set_sched_if_idle(gt, sched_if_idle); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_apply_vf_priority() - Change VF's scheduling priority. + * @xe: the PF &xe_device + * @vfid: the VF identifier + * @prio: scheduling priority to set + * + * Change VF's scheduling priority provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_apply_vf_priority(struct xe_device *xe, unsigned int vfid, u32 prio) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_set_sched_priority(gt, vfid, prio); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_query_vf_priority() - Query VF's scheduling priority. + * @xe: the PF &xe_device + * @vfid: the VF identifier + * @prio: placeholder for the returned scheduling priority + * + * Query VF's scheduling priority provisioning from all tiles/GTs. + * If values across tiles/GTs are inconsistent then -EUCLEAN error will be returned. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_query_vf_priority(struct xe_device *xe, unsigned int vfid, u32 *prio) +{ + struct xe_gt *gt; + unsigned int id; + int count = 0; + u32 value; + + for_each_gt(gt, xe, id) { + value = xe_gt_sriov_pf_config_get_sched_priority(gt, vfid); + if (!count++) + *prio = value; + else if (value != *prio) + return pf_report_unclean(gt, vfid, "priority", value, *prio); + } + + return !count ? -ENODATA : 0; +} + +static u64 vram_per_tile(struct xe_tile *tile, u64 total) +{ + struct xe_device *xe = tile->xe; + unsigned int tcount = xe->info.tile_count; + u64 alignment = xe_lmtt_page_size(&tile->sriov.pf.lmtt); + + total = round_up(total, tcount * alignment); + return div_u64(total, tcount); +} + +/** + * xe_sriov_pf_provision_bulk_apply_vram() - Change VRAM provisioning for all VFs. + * @xe: the PF &xe_device + * @size: the VRAM size in [bytes] to set + * + * Change all VFs VRAM (LMEM) provisioning on all tiles. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_bulk_apply_vram(struct xe_device *xe, u64 size) +{ + unsigned int num_vfs = xe_sriov_pf_get_totalvfs(xe); + struct xe_tile *tile; + unsigned int id; + int result = 0; + int err; + + xe_assert(xe, xe_device_has_lmtt(xe)); + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_tile(tile, xe, id) { + err = xe_gt_sriov_pf_config_bulk_set_lmem_locked(tile->primary_gt, + VFID(1), num_vfs, + vram_per_tile(tile, size)); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_apply_vf_vram() - Change single VF VRAM allocation. + * @xe: the PF &xe_device + * @vfid: the VF identifier (can't be 0 == PFID) + * @size: VRAM size to set + * + * Change VF's VRAM provisioning on all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_apply_vf_vram(struct xe_device *xe, unsigned int vfid, u64 size) +{ + struct xe_tile *tile; + unsigned int id; + int result = 0; + int err; + + xe_assert(xe, vfid); + xe_assert(xe, xe_device_has_lmtt(xe)); + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_tile(tile, xe, id) { + err = xe_gt_sriov_pf_config_set_lmem_locked(tile->primary_gt, vfid, + vram_per_tile(tile, size)); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_provision_query_vf_vram() - Query VF's VRAM allocation. + * @xe: the PF &xe_device + * @vfid: the VF identifier (can't be 0 == PFID) + * @size: placeholder for the returned VRAM size + * + * Query VF's VRAM provisioning from all tiles/GTs. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid, u64 *size) +{ + struct xe_tile *tile; + unsigned int id; + u64 total = 0; + + xe_assert(xe, vfid); + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_tile(tile, xe, id) + total += xe_gt_sriov_pf_config_get_lmem_locked(tile->primary_gt, vfid); + + *size = total; + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h new file mode 100644 index 000000000..b15e4d7ad --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_PROVISION_H_ +#define _XE_SRIOV_PF_PROVISION_H_ + +#include + +#include "xe_sriov_pf_provision_types.h" + +struct xe_device; + +int xe_sriov_pf_provision_bulk_apply_eq(struct xe_device *xe, u32 eq); +int xe_sriov_pf_provision_apply_vf_eq(struct xe_device *xe, unsigned int vfid, u32 eq); +int xe_sriov_pf_provision_query_vf_eq(struct xe_device *xe, unsigned int vfid, u32 *eq); + +int xe_sriov_pf_provision_bulk_apply_pt(struct xe_device *xe, u32 pt); +int xe_sriov_pf_provision_apply_vf_pt(struct xe_device *xe, unsigned int vfid, u32 pt); +int xe_sriov_pf_provision_query_vf_pt(struct xe_device *xe, unsigned int vfid, u32 *pt); + +int xe_sriov_pf_provision_bulk_apply_priority(struct xe_device *xe, u32 prio); +int xe_sriov_pf_provision_apply_vf_priority(struct xe_device *xe, unsigned int vfid, u32 prio); +int xe_sriov_pf_provision_query_vf_priority(struct xe_device *xe, unsigned int vfid, u32 *prio); + +int xe_sriov_pf_provision_bulk_apply_vram(struct xe_device *xe, u64 size); +int xe_sriov_pf_provision_apply_vf_vram(struct xe_device *xe, unsigned int vfid, u64 size); +int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid, u64 *size); + +int xe_sriov_pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs); +int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs); +int xe_sriov_pf_reprovision_default(struct xe_device *xe); + +int xe_sriov_pf_provision_set_mode(struct xe_device *xe, enum xe_sriov_provisioning_mode mode); + +/** + * xe_sriov_pf_provision_set_custom_mode() - Change VFs provision mode to custom. + * @xe: the PF &xe_device + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +static inline int xe_sriov_pf_provision_set_custom_mode(struct xe_device *xe) +{ + return xe_sriov_pf_provision_set_mode(xe, XE_SRIOV_PROVISIONING_MODE_CUSTOM); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_provision_types.h new file mode 100644 index 000000000..a847b8a4c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision_types.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_PROVISION_TYPES_H_ +#define _XE_SRIOV_PF_PROVISION_TYPES_H_ + +#include + +/** + * enum xe_sriov_provisioning_mode - SR-IOV provisioning mode. + * + * @XE_SRIOV_PROVISIONING_MODE_AUTO: VFs are provisioned during VFs enabling. + * Any allocated resources to the VFs will be + * automatically released when disabling VFs. + * This is a default mode. + * @XE_SRIOV_PROVISIONING_MODE_CUSTOM: Explicit VFs provisioning using uABI interfaces. + * VFs resources remains allocated regardless if + * VFs are enabled or not. + */ +enum xe_sriov_provisioning_mode { + XE_SRIOV_PROVISIONING_MODE_AUTO, + XE_SRIOV_PROVISIONING_MODE_CUSTOM, +}; +static_assert(XE_SRIOV_PROVISIONING_MODE_AUTO == 0); + +/** + * struct xe_sriov_pf_provision - Data used by the PF provisioning. + */ +struct xe_sriov_pf_provision { + /** @mode: selected provisioning mode. */ + enum xe_sriov_provisioning_mode mode; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_service.c b/drivers/gpu/drm/xe/xe_sriov_pf_service.c new file mode 100644 index 000000000..eee3b2a1b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_service.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2025 Intel Corporation + */ + +#include "abi/guc_relay_actions_abi.h" + +#include "xe_device_types.h" +#include "xe_sriov.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_printk.h" + +#include "xe_sriov_pf_service.h" +#include "xe_sriov_pf_service_types.h" + +/** + * xe_sriov_pf_service_init - Early initialization of the SR-IOV PF service. + * @xe: the &xe_device to initialize + * + * Performs early initialization of the SR-IOV PF service. + * + * This function can only be called on PF. + */ +void xe_sriov_pf_service_init(struct xe_device *xe) +{ + BUILD_BUG_ON(!GUC_RELAY_VERSION_BASE_MAJOR && !GUC_RELAY_VERSION_BASE_MINOR); + BUILD_BUG_ON(GUC_RELAY_VERSION_BASE_MAJOR > GUC_RELAY_VERSION_LATEST_MAJOR); + + xe_assert(xe, IS_SRIOV_PF(xe)); + + /* base versions may differ between platforms */ + xe->sriov.pf.service.version.base.major = GUC_RELAY_VERSION_BASE_MAJOR; + xe->sriov.pf.service.version.base.minor = GUC_RELAY_VERSION_BASE_MINOR; + + /* latest version is same for all platforms */ + xe->sriov.pf.service.version.latest.major = GUC_RELAY_VERSION_LATEST_MAJOR; + xe->sriov.pf.service.version.latest.minor = GUC_RELAY_VERSION_LATEST_MINOR; +} + +/* Return: 0 on success or a negative error code on failure. */ +static int pf_negotiate_version(struct xe_device *xe, + u32 wanted_major, u32 wanted_minor, + u32 *major, u32 *minor) +{ + struct xe_sriov_pf_service_version base = xe->sriov.pf.service.version.base; + struct xe_sriov_pf_service_version latest = xe->sriov.pf.service.version.latest; + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, base.major); + xe_assert(xe, base.major <= latest.major); + xe_assert(xe, (base.major < latest.major) || (base.minor <= latest.minor)); + + /* VF doesn't care - return our latest */ + if (wanted_major == VF2PF_HANDSHAKE_MAJOR_ANY && + wanted_minor == VF2PF_HANDSHAKE_MINOR_ANY) { + *major = latest.major; + *minor = latest.minor; + return 0; + } + + /* VF wants newer than our - return our latest */ + if (wanted_major > latest.major) { + *major = latest.major; + *minor = latest.minor; + return 0; + } + + /* VF wants older than min required - reject */ + if (wanted_major < base.major || + (wanted_major == base.major && wanted_minor < base.minor)) { + return -EPERM; + } + + /* previous major - return wanted, as we should still support it */ + if (wanted_major < latest.major) { + /* XXX: we are not prepared for multi-versions yet */ + xe_assert(xe, base.major == latest.major); + return -ENOPKG; + } + + /* same major - return common minor */ + *major = wanted_major; + *minor = min_t(u32, latest.minor, wanted_minor); + return 0; +} + +static void pf_connect(struct xe_device *xe, u32 vfid, u32 major, u32 minor) +{ + xe_sriov_pf_assert_vfid(xe, vfid); + xe_assert(xe, major || minor); + + xe->sriov.pf.vfs[vfid].version.major = major; + xe->sriov.pf.vfs[vfid].version.minor = minor; +} + +static void pf_disconnect(struct xe_device *xe, u32 vfid) +{ + xe_sriov_pf_assert_vfid(xe, vfid); + + xe->sriov.pf.vfs[vfid].version.major = 0; + xe->sriov.pf.vfs[vfid].version.minor = 0; +} + +/** + * xe_sriov_pf_service_is_negotiated - Check if VF has negotiated given ABI version. + * @xe: the &xe_device + * @vfid: the VF identifier + * @major: the major version to check + * @minor: the minor version to check + * + * Performs early initialization of the SR-IOV PF service. + * + * This function can only be called on PF. + * + * Returns: true if VF can use given ABI version functionality. + */ +bool xe_sriov_pf_service_is_negotiated(struct xe_device *xe, u32 vfid, u32 major, u32 minor) +{ + xe_sriov_pf_assert_vfid(xe, vfid); + + return major == xe->sriov.pf.vfs[vfid].version.major && + minor <= xe->sriov.pf.vfs[vfid].version.minor; +} + +/** + * xe_sriov_pf_service_handshake_vf - Confirm a connection with the VF. + * @xe: the &xe_device + * @vfid: the VF identifier + * @wanted_major: the major service version expected by the VF + * @wanted_minor: the minor service version expected by the VF + * @major: the major service version to be used by the VF + * @minor: the minor service version to be used by the VF + * + * Negotiate a VF/PF ABI version to allow VF use the PF services. + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_service_handshake_vf(struct xe_device *xe, u32 vfid, + u32 wanted_major, u32 wanted_minor, + u32 *major, u32 *minor) +{ + int err; + + xe_sriov_dbg_verbose(xe, "VF%u wants ABI version %u.%u\n", + vfid, wanted_major, wanted_minor); + + err = pf_negotiate_version(xe, wanted_major, wanted_minor, major, minor); + + if (err < 0) { + xe_sriov_notice(xe, "VF%u failed to negotiate ABI %u.%u (%pe)\n", + vfid, wanted_major, wanted_minor, ERR_PTR(err)); + pf_disconnect(xe, vfid); + } else { + xe_sriov_dbg(xe, "VF%u negotiated ABI version %u.%u\n", + vfid, *major, *minor); + pf_connect(xe, vfid, *major, *minor); + } + + return err; +} + +/** + * xe_sriov_pf_service_reset_vf - Reset a connection with the VF. + * @xe: the &xe_device + * @vfid: the VF identifier + * + * Reset a VF driver negotiated VF/PF ABI version. + * + * After that point, the VF driver will have to perform new version handshake + * to continue use of the PF services again. + * + * This function can only be called on PF. + */ +void xe_sriov_pf_service_reset_vf(struct xe_device *xe, unsigned int vfid) +{ + pf_disconnect(xe, vfid); +} + +static void print_pf_version(struct drm_printer *p, const char *name, + const struct xe_sriov_pf_service_version *version) +{ + drm_printf(p, "%s:\t%u.%u\n", name, version->major, version->minor); +} + +/** + * xe_sriov_pf_service_print_versions - Print ABI versions negotiated with VFs. + * @xe: the &xe_device + * @p: the &drm_printer + * + * This function is for PF use only. + */ +void xe_sriov_pf_service_print_versions(struct xe_device *xe, struct drm_printer *p) +{ + unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(xe); + struct xe_sriov_pf_service_version *version; + char name[8]; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + print_pf_version(p, "base", &xe->sriov.pf.service.version.base); + print_pf_version(p, "latest", &xe->sriov.pf.service.version.latest); + + for (n = 1; n <= total_vfs; n++) { + version = &xe->sriov.pf.vfs[n].version; + if (!version->major && !version->minor) + continue; + + print_pf_version(p, xe_sriov_function_name(n, name, sizeof(name)), version); + } +} + +#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_sriov_pf_service_kunit.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_service.h b/drivers/gpu/drm/xe/xe_sriov_pf_service.h new file mode 100644 index 000000000..d38c18f5e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_service.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_SERVICE_H_ +#define _XE_SRIOV_PF_SERVICE_H_ + +#include + +struct drm_printer; +struct xe_device; + +void xe_sriov_pf_service_init(struct xe_device *xe); +void xe_sriov_pf_service_print_versions(struct xe_device *xe, struct drm_printer *p); + +int xe_sriov_pf_service_handshake_vf(struct xe_device *xe, u32 vfid, + u32 wanted_major, u32 wanted_minor, + u32 *major, u32 *minor); +bool xe_sriov_pf_service_is_negotiated(struct xe_device *xe, u32 vfid, u32 major, u32 minor); +void xe_sriov_pf_service_reset_vf(struct xe_device *xe, unsigned int vfid); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_service_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_service_types.h new file mode 100644 index 000000000..0835dde35 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_service_types.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_SERVICE_TYPES_H_ +#define _XE_SRIOV_PF_SERVICE_TYPES_H_ + +#include + +/** + * struct xe_sriov_pf_service_version - VF/PF ABI Version. + * @major: the major version of the VF/PF ABI + * @minor: the minor version of the VF/PF ABI + * + * See `GuC Relay Communication`_. + */ +struct xe_sriov_pf_service_version { + u16 major; + u16 minor; +}; + +/** + * struct xe_sriov_pf_service - Data used by the PF service. + * @version: information about VF/PF ABI versions for current platform. + * @version.base: lowest VF/PF ABI version that could be negotiated with VF. + * @version.latest: latest VF/PF ABI version supported by the PF driver. + */ +struct xe_sriov_pf_service { + struct { + struct xe_sriov_pf_service_version base; + struct xe_sriov_pf_service_version latest; + } version; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c new file mode 100644 index 000000000..ffc3447fc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_pci_sriov.h" +#include "xe_pm.h" +#include "xe_sriov.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_provision.h" +#include "xe_sriov_pf_sysfs.h" +#include "xe_sriov_printk.h" + +static int emit_choice(char *buf, int choice, const char * const *array, size_t size) +{ + int pos = 0; + int n; + + for (n = 0; n < size; n++) { + pos += sysfs_emit_at(buf, pos, "%s%s%s%s", + n ? " " : "", + n == choice ? "[" : "", + array[n], + n == choice ? "]" : ""); + } + pos += sysfs_emit_at(buf, pos, "\n"); + + return pos; +} + +/* + * /sys/bus/pci/drivers/xe/BDF/ + * : + * ├── sriov_admin/ + * ├── ... + * ├── .bulk_profile + * │ ├── exec_quantum_ms + * │ ├── preempt_timeout_us + * │ ├── sched_priority + * │ └── vram_quota + * ├── pf/ + * │ ├── ... + * │ ├── device -> ../../../BDF + * │ └── profile + * │ ├── exec_quantum_ms + * │ ├── preempt_timeout_us + * │ └── sched_priority + * ├── vf1/ + * │ ├── ... + * │ ├── device -> ../../../BDF.1 + * │ ├── stop + * │ └── profile + * │ ├── exec_quantum_ms + * │ ├── preempt_timeout_us + * │ ├── sched_priority + * │ └── vram_quota + * ├── vf2/ + * : + * └── vfN/ + */ + +struct xe_sriov_kobj { + struct kobject base; + struct xe_device *xe; + unsigned int vfid; +}; +#define to_xe_sriov_kobj(p) container_of_const((p), struct xe_sriov_kobj, base) + +struct xe_sriov_dev_attr { + struct attribute attr; + ssize_t (*show)(struct xe_device *xe, char *buf); + ssize_t (*store)(struct xe_device *xe, const char *buf, size_t count); +}; +#define to_xe_sriov_dev_attr(p) container_of_const((p), struct xe_sriov_dev_attr, attr) + +#define XE_SRIOV_DEV_ATTR(NAME) \ +struct xe_sriov_dev_attr xe_sriov_dev_attr_##NAME = \ + __ATTR(NAME, 0644, xe_sriov_dev_attr_##NAME##_show, xe_sriov_dev_attr_##NAME##_store) + +#define XE_SRIOV_DEV_ATTR_RO(NAME) \ +struct xe_sriov_dev_attr xe_sriov_dev_attr_##NAME = \ + __ATTR(NAME, 0444, xe_sriov_dev_attr_##NAME##_show, NULL) + +#define XE_SRIOV_DEV_ATTR_WO(NAME) \ +struct xe_sriov_dev_attr xe_sriov_dev_attr_##NAME = \ + __ATTR(NAME, 0200, NULL, xe_sriov_dev_attr_##NAME##_store) + +struct xe_sriov_vf_attr { + struct attribute attr; + ssize_t (*show)(struct xe_device *xe, unsigned int vfid, char *buf); + ssize_t (*store)(struct xe_device *xe, unsigned int vfid, const char *buf, size_t count); +}; +#define to_xe_sriov_vf_attr(p) container_of_const((p), struct xe_sriov_vf_attr, attr) + +#define XE_SRIOV_VF_ATTR(NAME) \ +struct xe_sriov_vf_attr xe_sriov_vf_attr_##NAME = \ + __ATTR(NAME, 0644, xe_sriov_vf_attr_##NAME##_show, xe_sriov_vf_attr_##NAME##_store) + +#define XE_SRIOV_VF_ATTR_RO(NAME) \ +struct xe_sriov_vf_attr xe_sriov_vf_attr_##NAME = \ + __ATTR(NAME, 0444, xe_sriov_vf_attr_##NAME##_show, NULL) + +#define XE_SRIOV_VF_ATTR_WO(NAME) \ +struct xe_sriov_vf_attr xe_sriov_vf_attr_##NAME = \ + __ATTR(NAME, 0200, NULL, xe_sriov_vf_attr_##NAME##_store) + +/* device level attributes go here */ + +#define DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(NAME, ITEM, TYPE) \ + \ +static ssize_t xe_sriov_dev_attr_##NAME##_store(struct xe_device *xe, \ + const char *buf, size_t count) \ +{ \ + TYPE value; \ + int err; \ + \ + err = kstrto##TYPE(buf, 0, &value); \ + if (err) \ + return err; \ + \ + err = xe_sriov_pf_provision_bulk_apply_##ITEM(xe, value); \ + return err ?: count; \ +} \ + \ +static XE_SRIOV_DEV_ATTR_WO(NAME) + +DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(exec_quantum_ms, eq, u32); +DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(preempt_timeout_us, pt, u32); +DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(vram_quota, vram, u64); + +static const char * const sched_priority_names[] = { + [GUC_SCHED_PRIORITY_LOW] = "low", + [GUC_SCHED_PRIORITY_NORMAL] = "normal", + [GUC_SCHED_PRIORITY_HIGH] = "high", +}; + +static bool sched_priority_change_allowed(unsigned int vfid) +{ + /* As of today GuC FW allows to selectively change only the PF priority. */ + return vfid == PFID; +} + +static bool sched_priority_high_allowed(unsigned int vfid) +{ + /* As of today GuC FW allows to select 'high' priority only for the PF. */ + return vfid == PFID; +} + +static bool sched_priority_bulk_high_allowed(struct xe_device *xe) +{ + /* all VFs are equal - it's sufficient to check VF1 only */ + return sched_priority_high_allowed(VFID(1)); +} + +static ssize_t xe_sriov_dev_attr_sched_priority_store(struct xe_device *xe, + const char *buf, size_t count) +{ + size_t num_priorities = ARRAY_SIZE(sched_priority_names); + int match; + int err; + + if (!sched_priority_bulk_high_allowed(xe)) + num_priorities--; + + match = __sysfs_match_string(sched_priority_names, num_priorities, buf); + if (match < 0) + return -EINVAL; + + err = xe_sriov_pf_provision_bulk_apply_priority(xe, match); + return err ?: count; +} + +static XE_SRIOV_DEV_ATTR_WO(sched_priority); + +static struct attribute *bulk_profile_dev_attrs[] = { + &xe_sriov_dev_attr_exec_quantum_ms.attr, + &xe_sriov_dev_attr_preempt_timeout_us.attr, + &xe_sriov_dev_attr_sched_priority.attr, + &xe_sriov_dev_attr_vram_quota.attr, + NULL +}; + +static umode_t profile_dev_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int index) +{ + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + + if (attr == &xe_sriov_dev_attr_vram_quota.attr && + !xe_device_has_lmtt(vkobj->xe)) + return 0; + + return attr->mode; +} + +static const struct attribute_group bulk_profile_dev_attr_group = { + .name = ".bulk_profile", + .attrs = bulk_profile_dev_attrs, + .is_visible = profile_dev_attr_is_visible, +}; + +static const struct attribute_group *xe_sriov_dev_attr_groups[] = { + &bulk_profile_dev_attr_group, + NULL +}; + +/* and VF-level attributes go here */ + +#define DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(NAME, ITEM, TYPE, FORMAT) \ +static ssize_t xe_sriov_vf_attr_##NAME##_show(struct xe_device *xe, unsigned int vfid, \ + char *buf) \ +{ \ + TYPE value = 0; \ + int err; \ + \ + err = xe_sriov_pf_provision_query_vf_##ITEM(xe, vfid, &value); \ + if (err) \ + return err; \ + \ + return sysfs_emit(buf, FORMAT, value); \ +} \ + \ +static ssize_t xe_sriov_vf_attr_##NAME##_store(struct xe_device *xe, unsigned int vfid, \ + const char *buf, size_t count) \ +{ \ + TYPE value; \ + int err; \ + \ + err = kstrto##TYPE(buf, 0, &value); \ + if (err) \ + return err; \ + \ + err = xe_sriov_pf_provision_apply_vf_##ITEM(xe, vfid, value); \ + return err ?: count; \ +} \ + \ +static XE_SRIOV_VF_ATTR(NAME) + +DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(exec_quantum_ms, eq, u32, "%u\n"); +DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(preempt_timeout_us, pt, u32, "%u\n"); +DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(vram_quota, vram, u64, "%llu\n"); + +static ssize_t xe_sriov_vf_attr_sched_priority_show(struct xe_device *xe, unsigned int vfid, + char *buf) +{ + size_t num_priorities = ARRAY_SIZE(sched_priority_names); + u32 priority; + int err; + + err = xe_sriov_pf_provision_query_vf_priority(xe, vfid, &priority); + if (err) + return err; + + if (!sched_priority_high_allowed(vfid)) + num_priorities--; + + xe_assert(xe, priority < num_priorities); + return emit_choice(buf, priority, sched_priority_names, num_priorities); +} + +static ssize_t xe_sriov_vf_attr_sched_priority_store(struct xe_device *xe, unsigned int vfid, + const char *buf, size_t count) +{ + size_t num_priorities = ARRAY_SIZE(sched_priority_names); + int match; + int err; + + if (!sched_priority_change_allowed(vfid)) + return -EOPNOTSUPP; + + if (!sched_priority_high_allowed(vfid)) + num_priorities--; + + match = __sysfs_match_string(sched_priority_names, num_priorities, buf); + if (match < 0) + return -EINVAL; + + err = xe_sriov_pf_provision_apply_vf_priority(xe, vfid, match); + return err ?: count; +} + +static XE_SRIOV_VF_ATTR(sched_priority); + +static struct attribute *profile_vf_attrs[] = { + &xe_sriov_vf_attr_exec_quantum_ms.attr, + &xe_sriov_vf_attr_preempt_timeout_us.attr, + &xe_sriov_vf_attr_sched_priority.attr, + &xe_sriov_vf_attr_vram_quota.attr, + NULL +}; + +static umode_t profile_vf_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int index) +{ + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + + if (attr == &xe_sriov_vf_attr_sched_priority.attr && + !sched_priority_change_allowed(vkobj->vfid)) + return attr->mode & 0444; + + if (attr == &xe_sriov_vf_attr_vram_quota.attr) { + if (!IS_DGFX(vkobj->xe) || vkobj->vfid == PFID) + return 0; + if (!xe_device_has_lmtt(vkobj->xe)) + return attr->mode & 0444; + } + + return attr->mode; +} + +static const struct attribute_group profile_vf_attr_group = { + .name = "profile", + .attrs = profile_vf_attrs, + .is_visible = profile_vf_attr_is_visible, +}; + +#define DEFINE_SIMPLE_CONTROL_SRIOV_VF_ATTR(NAME) \ + \ +static ssize_t xe_sriov_vf_attr_##NAME##_store(struct xe_device *xe, unsigned int vfid, \ + const char *buf, size_t count) \ +{ \ + bool yes; \ + int err; \ + \ + if (!vfid) \ + return -EPERM; \ + \ + err = kstrtobool(buf, &yes); \ + if (err) \ + return err; \ + if (!yes) \ + return count; \ + \ + err = xe_sriov_pf_control_##NAME##_vf(xe, vfid); \ + return err ?: count; \ +} \ + \ +static XE_SRIOV_VF_ATTR_WO(NAME) + +DEFINE_SIMPLE_CONTROL_SRIOV_VF_ATTR(stop); + +static struct attribute *control_vf_attrs[] = { + &xe_sriov_vf_attr_stop.attr, + NULL +}; + +static umode_t control_vf_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int index) +{ + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + + if (vkobj->vfid == PFID) + return 0; + + return attr->mode; +} + +static const struct attribute_group control_vf_attr_group = { + .attrs = control_vf_attrs, + .is_visible = control_vf_attr_is_visible, +}; + +static const struct attribute_group *xe_sriov_vf_attr_groups[] = { + &profile_vf_attr_group, + &control_vf_attr_group, + NULL +}; + +/* no user serviceable parts below */ + +static void action_put_kobject(void *arg) +{ + struct kobject *kobj = arg; + + kobject_put(kobj); +} + +static struct kobject *create_xe_sriov_kobj(struct xe_device *xe, unsigned int vfid, + const struct kobj_type *ktype) +{ + struct xe_sriov_kobj *vkobj; + int err; + + xe_sriov_pf_assert_vfid(xe, vfid); + + vkobj = kzalloc_obj(*vkobj); + if (!vkobj) + return ERR_PTR(-ENOMEM); + + vkobj->xe = xe; + vkobj->vfid = vfid; + kobject_init(&vkobj->base, ktype); + + err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, &vkobj->base); + if (err) + return ERR_PTR(err); + + return &vkobj->base; +} + +static void release_xe_sriov_kobj(struct kobject *kobj) +{ + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + + kfree(vkobj); +} + +static ssize_t xe_sriov_dev_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct xe_sriov_dev_attr *vattr = to_xe_sriov_dev_attr(attr); + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + struct xe_device *xe = vkobj->xe; + + if (!vattr->show) + return -EPERM; + + return vattr->show(xe, buf); +} + +static ssize_t xe_sriov_dev_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct xe_sriov_dev_attr *vattr = to_xe_sriov_dev_attr(attr); + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + struct xe_device *xe = vkobj->xe; + + if (!vattr->store) + return -EPERM; + + guard(xe_pm_runtime)(xe); + return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count); +} + +static ssize_t xe_sriov_vf_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct xe_sriov_vf_attr *vattr = to_xe_sriov_vf_attr(attr); + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + struct xe_device *xe = vkobj->xe; + unsigned int vfid = vkobj->vfid; + + xe_sriov_pf_assert_vfid(xe, vfid); + + if (!vattr->show) + return -EPERM; + + return vattr->show(xe, vfid, buf); +} + +static ssize_t xe_sriov_vf_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct xe_sriov_vf_attr *vattr = to_xe_sriov_vf_attr(attr); + struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); + struct xe_device *xe = vkobj->xe; + unsigned int vfid = vkobj->vfid; + + xe_sriov_pf_assert_vfid(xe, vfid); + + if (!vattr->store) + return -EPERM; + + guard(xe_pm_runtime)(xe); + return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count); +} + +static const struct sysfs_ops xe_sriov_dev_sysfs_ops = { + .show = xe_sriov_dev_attr_show, + .store = xe_sriov_dev_attr_store, +}; + +static const struct sysfs_ops xe_sriov_vf_sysfs_ops = { + .show = xe_sriov_vf_attr_show, + .store = xe_sriov_vf_attr_store, +}; + +static const struct kobj_type xe_sriov_dev_ktype = { + .release = release_xe_sriov_kobj, + .sysfs_ops = &xe_sriov_dev_sysfs_ops, + .default_groups = xe_sriov_dev_attr_groups, +}; + +static const struct kobj_type xe_sriov_vf_ktype = { + .release = release_xe_sriov_kobj, + .sysfs_ops = &xe_sriov_vf_sysfs_ops, + .default_groups = xe_sriov_vf_attr_groups, +}; + +static int pf_sysfs_error(struct xe_device *xe, int err, const char *what) +{ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) + xe_sriov_dbg(xe, "Failed to setup sysfs %s (%pe)\n", what, ERR_PTR(err)); + return err; +} + +static void pf_sysfs_note(struct xe_device *xe, int err, const char *what) +{ + xe_sriov_dbg(xe, "Failed to setup sysfs %s (%pe)\n", what, ERR_PTR(err)); +} + +static int pf_setup_root(struct xe_device *xe) +{ + struct kobject *parent = &xe->drm.dev->kobj; + struct kobject *root; + int err; + + root = create_xe_sriov_kobj(xe, PFID, &xe_sriov_dev_ktype); + if (IS_ERR(root)) + return pf_sysfs_error(xe, PTR_ERR(root), "root obj"); + + err = kobject_add(root, parent, "sriov_admin"); + if (err) + return pf_sysfs_error(xe, err, "root init"); + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, !xe->sriov.pf.sysfs.root); + xe->sriov.pf.sysfs.root = root; + return 0; +} + +static int pf_setup_tree(struct xe_device *xe) +{ + unsigned int totalvfs = xe_sriov_pf_get_totalvfs(xe); + struct kobject *root, *kobj; + unsigned int n; + int err; + + xe_assert(xe, IS_SRIOV_PF(xe)); + root = xe->sriov.pf.sysfs.root; + + for (n = 0; n <= totalvfs; n++) { + kobj = create_xe_sriov_kobj(xe, VFID(n), &xe_sriov_vf_ktype); + if (IS_ERR(kobj)) + return pf_sysfs_error(xe, PTR_ERR(kobj), "tree obj"); + + if (n) + err = kobject_add(kobj, root, "vf%u", n); + else + err = kobject_add(kobj, root, "pf"); + if (err) + return pf_sysfs_error(xe, err, "tree init"); + + xe_assert(xe, !xe->sriov.pf.vfs[n].kobj); + xe->sriov.pf.vfs[n].kobj = kobj; + } + + return 0; +} + +static void action_rm_device_link(void *arg) +{ + struct kobject *kobj = arg; + + sysfs_remove_link(kobj, "device"); +} + +static int pf_link_pf_device(struct xe_device *xe) +{ + struct kobject *kobj = xe->sriov.pf.vfs[PFID].kobj; + int err; + + err = sysfs_create_link(kobj, &xe->drm.dev->kobj, "device"); + if (err) + return pf_sysfs_error(xe, err, "PF device link"); + + err = devm_add_action_or_reset(xe->drm.dev, action_rm_device_link, kobj); + if (err) + return pf_sysfs_error(xe, err, "PF unlink action"); + + return 0; +} + +/** + * xe_sriov_pf_sysfs_init() - Setup PF's SR-IOV sysfs tree. + * @xe: the PF &xe_device to setup sysfs + * + * This function will create additional nodes that will represent PF and VFs + * devices, each populated with SR-IOV Xe specific attributes. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_sysfs_init(struct xe_device *xe) +{ + int err; + + err = pf_setup_root(xe); + if (err) + return err; + + err = pf_setup_tree(xe); + if (err) + return err; + + err = pf_link_pf_device(xe); + if (err) + return err; + + return 0; +} + +/** + * xe_sriov_pf_sysfs_link_vfs() - Add VF's links in SR-IOV sysfs tree. + * @xe: the &xe_device where to update sysfs + * @num_vfs: number of enabled VFs to link + * + * This function is specific for the PF driver. + * + * This function will add symbolic links between VFs represented in the SR-IOV + * sysfs tree maintained by the PF and enabled VF PCI devices. + * + * The @xe_sriov_pf_sysfs_unlink_vfs() shall be used to remove those links. + */ +void xe_sriov_pf_sysfs_link_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + unsigned int totalvfs = xe_sriov_pf_get_totalvfs(xe); + struct pci_dev *pf_pdev = to_pci_dev(xe->drm.dev); + struct pci_dev *vf_pdev = NULL; + unsigned int n; + int err; + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, num_vfs <= totalvfs); + + for (n = 1; n <= num_vfs; n++) { + vf_pdev = xe_pci_sriov_get_vf_pdev(pf_pdev, VFID(n)); + if (!vf_pdev) + return pf_sysfs_note(xe, -ENOENT, "VF link"); + + err = sysfs_create_link(xe->sriov.pf.vfs[VFID(n)].kobj, + &vf_pdev->dev.kobj, "device"); + + /* must balance xe_pci_sriov_get_vf_pdev() */ + pci_dev_put(vf_pdev); + + if (err) + return pf_sysfs_note(xe, err, "VF link"); + } +} + +/** + * xe_sriov_pf_sysfs_unlink_vfs() - Remove VF's links from SR-IOV sysfs tree. + * @xe: the &xe_device where to update sysfs + * @num_vfs: number of VFs to unlink + * + * This function shall be called only on the PF. + * This function will remove "device" links added by @xe_sriov_sysfs_link_vfs(). + */ +void xe_sriov_pf_sysfs_unlink_vfs(struct xe_device *xe, unsigned int num_vfs) +{ + unsigned int n; + + xe_assert(xe, IS_SRIOV_PF(xe)); + xe_assert(xe, num_vfs <= xe_sriov_pf_get_totalvfs(xe)); + + for (n = 1; n <= num_vfs; n++) + sysfs_remove_link(xe->sriov.pf.vfs[VFID(n)].kobj, "device"); +} diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.h b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.h new file mode 100644 index 000000000..ae92ed176 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_SYSFS_H_ +#define _XE_SRIOV_PF_SYSFS_H_ + +struct xe_device; + +int xe_sriov_pf_sysfs_init(struct xe_device *xe); + +void xe_sriov_pf_sysfs_link_vfs(struct xe_device *xe, unsigned int num_vfs); +void xe_sriov_pf_sysfs_unlink_vfs(struct xe_device *xe, unsigned int num_vfs); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_types.h new file mode 100644 index 000000000..b0253e1ae --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_pf_types.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PF_TYPES_H_ +#define _XE_SRIOV_PF_TYPES_H_ + +#include +#include + +#include "xe_guard.h" +#include "xe_sriov_pf_migration_types.h" +#include "xe_sriov_pf_provision_types.h" +#include "xe_sriov_pf_service_types.h" + +struct kobject; + +/** + * struct xe_sriov_metadata - per-VF device level metadata + */ +struct xe_sriov_metadata { + /** @kobj: kobject representing VF in PF's SR-IOV sysfs tree. */ + struct kobject *kobj; + + /** @version: negotiated VF/PF ABI version */ + struct xe_sriov_pf_service_version version; + /** @migration: migration state */ + struct xe_sriov_migration_state migration; +}; + +/** + * struct xe_device_pf - Xe PF related data + * + * The data in this structure is valid only if driver is running in the + * @XE_SRIOV_MODE_PF mode. + */ +struct xe_device_pf { + /** @device_total_vfs: Maximum number of VFs supported by the device. */ + u16 device_total_vfs; + + /** @driver_max_vfs: Maximum number of VFs supported by the driver. */ + u16 driver_max_vfs; + + /** @guard_vfs_enabling: guards VFs enabling */ + struct xe_guard guard_vfs_enabling; + + /** @master_lock: protects all VFs configurations across GTs */ + struct mutex master_lock; + + /** @provision: device level provisioning data. */ + struct xe_sriov_pf_provision provision; + + /** @migration: device level migration data. */ + struct xe_sriov_pf_migration migration; + + /** @service: device level service data. */ + struct xe_sriov_pf_service service; + + /** @sysfs: device level sysfs data. */ + struct { + /** @sysfs.root: the root kobject for all SR-IOV entries in sysfs. */ + struct kobject *root; + } sysfs; + + /** @vfs: metadata for all VFs. */ + struct xe_sriov_metadata *vfs; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_printk.h b/drivers/gpu/drm/xe/xe_sriov_printk.h new file mode 100644 index 000000000..4c6b5c3d2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_printk.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_PRINTK_H_ +#define _XE_SRIOV_PRINTK_H_ + +#include "xe_printk.h" + +#define xe_sriov_printk_prefix(xe) \ + ((xe)->sriov.__mode == XE_SRIOV_MODE_PF ? "PF: " : \ + (xe)->sriov.__mode == XE_SRIOV_MODE_VF ? "VF: " : "") + +#define __XE_SRIOV_PRINTK_FMT(_xe, _fmt, _args...) \ + "%s" _fmt, xe_sriov_printk_prefix(_xe), ##_args + +#define xe_sriov_printk(xe, _level, fmt, ...) \ + xe_##_level((xe), __XE_SRIOV_PRINTK_FMT((xe), fmt, ##__VA_ARGS__)) + +#define xe_sriov_err(xe, fmt, ...) \ + xe_sriov_printk((xe), err, fmt, ##__VA_ARGS__) + +#define xe_sriov_err_ratelimited(xe, fmt, ...) \ + xe_sriov_printk((xe), err_ratelimited, fmt, ##__VA_ARGS__) + +#define xe_sriov_warn(xe, fmt, ...) \ + xe_sriov_printk((xe), warn, fmt, ##__VA_ARGS__) + +#define xe_sriov_notice(xe, fmt, ...) \ + xe_sriov_printk((xe), notice, fmt, ##__VA_ARGS__) + +#define xe_sriov_info(xe, fmt, ...) \ + xe_sriov_printk((xe), info, fmt, ##__VA_ARGS__) + +#define xe_sriov_dbg(xe, fmt, ...) \ + xe_sriov_printk((xe), dbg, fmt, ##__VA_ARGS__) + +/* for low level noisy debug messages */ +#ifdef CONFIG_DRM_XE_DEBUG_SRIOV +#define xe_sriov_dbg_verbose(xe, fmt, ...) xe_sriov_dbg(xe, fmt, ##__VA_ARGS__) +#else +#define xe_sriov_dbg_verbose(xe, fmt, ...) typecheck(struct xe_device *, (xe)) +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_types.h b/drivers/gpu/drm/xe/xe_sriov_types.h new file mode 100644 index 000000000..1a138108d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_types.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_SRIOV_TYPES_H_ +#define _XE_SRIOV_TYPES_H_ + +#include + +/** + * VFID - Virtual Function Identifier + * @n: VF number + * + * Helper macro to represent Virtual Function (VF) Identifier. + * VFID(0) is used as alias to the PFID that represents Physical Function. + * + * Note: According to PCI spec, SR-IOV VF's numbers are 1-based (VF1, VF2, ...). + */ +#define VFID(n) (n) +#define PFID VFID(0) + +/** + * enum xe_sriov_mode - SR-IOV mode + * @XE_SRIOV_MODE_NONE: bare-metal mode (non-virtualized) + * @XE_SRIOV_MODE_PF: SR-IOV Physical Function (PF) mode + * @XE_SRIOV_MODE_VF: SR-IOV Virtual Function (VF) mode + */ +enum xe_sriov_mode { + /* + * Note: We don't use default enum value 0 to allow catch any too early + * attempt of checking the SR-IOV mode prior to the actual mode probe. + */ + XE_SRIOV_MODE_NONE = 1, + XE_SRIOV_MODE_PF, + XE_SRIOV_MODE_VF, +}; +static_assert(XE_SRIOV_MODE_NONE); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c new file mode 100644 index 000000000..29894bd08 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#include +#include + +#include "xe_gt_sriov_vf.h" +#include "xe_guc.h" +#include "xe_sriov_printk.h" +#include "xe_sriov_vf.h" +#include "xe_sriov_vf_ccs.h" + +/** + * DOC: VF restore procedure in PF KMD and VF KMD + * + * Restoring previously saved state of a VF is one of core features of + * SR-IOV. All major VM Management applications allow saving and restoring + * the VM state, and doing that to a VM which uses SRIOV VF as one of + * the accessible devices requires support from KMD on both PF and VF side. + * VMM initiates all required operations through VFIO module, which then + * translates them into PF KMD calls. This description will focus on these + * calls, leaving out the module which initiates these steps (VFIO). + * + * In order to start the restore procedure, GuC needs to keep the VF in + * proper state. The PF driver can ensure GuC set it to VF_READY state + * by provisioning the VF, which in turn can be done after Function Level + * Reset of said VF (or after it was freshly created - in that case FLR + * is not needed). The FLR procedure ends with GuC sending message + * `GUC_PF_NOTIFY_VF_FLR_DONE`, and then provisioning data is sent to GuC. + * After the provisioning is completed, the VF needs to be paused, and + * at that point the actual restore can begin. + * + * During VF Restore, state of several resources is restored. These may + * include local memory content (system memory is restored by VMM itself), + * values of MMIO registers, stateless compression metadata and others. + * The final resource which also needs restoring is state of the VF + * submission maintained within GuC. For that, `GUC_PF_OPCODE_VF_RESTORE` + * message is used, with reference to the state blob to be consumed by + * GuC. + * + * Next, when VFIO is asked to set the VM into running state, the PF driver + * sends `GUC_PF_TRIGGER_VF_RESUME` to GuC. When sent after restore, this + * changes VF state within GuC to `VF_RESFIX_BLOCKED` rather than the + * usual `VF_RUNNING`. At this point GuC triggers an interrupt to inform + * the VF KMD within the VM that it was migrated. + * + * As soon as Virtual GPU of the VM starts, the VF driver within receives + * the MIGRATED interrupt and schedules post-migration recovery worker. + * That worker sends `VF2GUC_RESFIX_START` action along with non-zero + * marker, queries GuC for new provisioning (using MMIO communication), + * and applies fixups to any non-virtualized resources used by the VF. + * + * When the VF driver is ready to continue operation on the newly connected + * hardware, it sends `VF2GUC_RESFIX_DONE` action along with the same + * marker which was sent with `VF2GUC_RESFIX_START` which causes it to + * enter the long awaited `VF_RUNNING` state, and therefore start handling + * CTB messages and scheduling workloads from the VF:: + * + * PF GuC VF + * [ ] | | + * [ ] PF2GUC_VF_CONTROL(pause) | | + * [ ]---------------------------> [ ] | + * [ ] [ ] GuC sets new VF state to | + * [ ] [ ]------- VF_READY_PAUSED | + * [ ] [ ] | | + * [ ] [ ] <----- | + * [ ] success [ ] | + * [ ] <---------------------------[ ] | + * [ ] | | + * [ ] PF loads resources from the | | + * [ ]------- saved image supplied | | + * [ ] | | | + * [ ] <----- | | + * [ ] | | + * [ ] GUC_PF_OPCODE_VF_RESTORE | | + * [ ]---------------------------> [ ] | + * [ ] [ ] GuC loads contexts and CTB | + * [ ] [ ]------- state from image | + * [ ] [ ] | | + * [ ] [ ] <----- | + * [ ] [ ] | + * [ ] [ ] GuC sets new VF state to | + * [ ] [ ]------- VF_RESFIX_PAUSED | + * [ ] [ ] | | + * [ ] success [ ] <----- | + * [ ] <---------------------------[ ] | + * [ ] | | + * [ ] GUC_PF_TRIGGER_VF_RESUME | | + * [ ]---------------------------> [ ] | + * [ ] [ ] GuC sets new VF state to | + * [ ] [ ]------- VF_RESFIX_BLOCKED | + * [ ] [ ] | | + * [ ] [ ] <----- | + * [ ] [ ] | + * [ ] [ ] GUC_INTR_SW_INT_0 | + * [ ] success [ ]---------------------------> [ ] + * [ ] <---------------------------[ ] [ ] + * | | VF2GUC_QUERY_SINGLE_KLV [ ] + * | [ ] <---------------------------[ ] + * | [ ] [ ] + * | [ ] new VF provisioning [ ] + * | [ ]---------------------------> [ ] + * | | [ ] + * | | VF2GUC_RESFIX_START [ ] + * | [ ] <---------------------------[ ] + * | [ ] [ ] + * | [ ] success [ ] + * | [ ]---------------------------> [ ] + * | | VF driver applies post [ ] + * | | migration fixups -------[ ] + * | | | [ ] + * | | -----> [ ] + * | | [ ] + * | | VF2GUC_RESFIX_DONE [ ] + * | [ ] <---------------------------[ ] + * | [ ] [ ] + * | [ ] GuC sets new VF state to [ ] + * | [ ]------- VF_RUNNING [ ] + * | [ ] | [ ] + * | [ ] <----- [ ] + * | [ ] success [ ] + * | [ ]---------------------------> [ ] + * | | | + * | | | + * + * Handling of VF double migration flow is shown below:: + * + * GuC1 VF + * | | + * | [ ]<--- start fixups + * | VF2GUC_RESFIX_START(marker) [ ] + * [ ] <-------------------------------------------[ ] + * [ ] [ ] + * [ ]---\ [ ] + * [ ] store marker [ ] + * [ ]<--/ [ ] + * [ ] [ ] + * [ ] success [ ] + * [ ] ------------------------------------------> [ ] + * | [ ] + * | [ ]---\ + * | [ ] do fixups + * | [ ]<--/ + * | [ ] + * -------------- VF paused / saved ---------------- + * : + * + * GuC2 + * | + * ----------------- VF restored ------------------ + * | + * [ ] + * [ ]---\ + * [ ] reset marker + * [ ]<--/ + * [ ] + * ----------------- VF resumed ------------------ + * | [ ] + * | [ ] + * | VF2GUC_RESFIX_DONE(marker) [ ] + * [ ] <-------------------------------------------[ ] + * [ ] [ ] + * [ ]---\ [ ] + * [ ] check marker [ ] + * [ ] (mismatch) [ ] + * [ ]<--/ [ ] + * [ ] [ ] + * [ ] RESPONSE_VF_MIGRATED [ ] + * [ ] ------------------------------------------> [ ] + * | [ ]---\ + * | [ ] reschedule fixups + * | [ ]<--/ + * | | + */ + +/** + * xe_sriov_vf_migration_supported - Report whether SR-IOV VF migration is + * supported or not. + * @xe: the &xe_device to check + * + * Returns: true if VF migration is supported, false otherwise. + */ +bool xe_sriov_vf_migration_supported(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_VF(xe)); + return !xe->sriov.vf.migration.disabled; +} + +/** + * xe_sriov_vf_migration_disable - Turn off VF migration with given log message. + * @xe: the &xe_device instance. + * @fmt: format string for the log message, to be combined with following VAs. + */ +void xe_sriov_vf_migration_disable(struct xe_device *xe, const char *fmt, ...) +{ + struct va_format vaf; + va_list va_args; + + xe_assert(xe, IS_SRIOV_VF(xe)); + + va_start(va_args, fmt); + vaf.fmt = fmt; + vaf.va = &va_args; + xe_sriov_notice(xe, "migration disabled: %pV\n", &vaf); + va_end(va_args); + + xe->sriov.vf.migration.disabled = true; +} + +static void vf_migration_init_early(struct xe_device *xe) +{ + if (!xe_device_has_memirq(xe)) + return xe_sriov_vf_migration_disable(xe, "requires memory-based IRQ support"); + +} + +/** + * xe_sriov_vf_init_early - Initialize SR-IOV VF specific data. + * @xe: the &xe_device to initialize + */ +void xe_sriov_vf_init_early(struct xe_device *xe) +{ + vf_migration_init_early(xe); +} + +static int vf_migration_init_late(struct xe_device *xe) +{ + struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_uc_fw_version guc_version; + + if (!xe_sriov_vf_migration_supported(xe)) + return 0; + + xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version); + if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 27, 0)) { + xe_sriov_vf_migration_disable(xe, + "requires GuC ABI >= 1.27.0, but only %u.%u.%u found", + guc_version.major, guc_version.minor, + guc_version.patch); + return 0; + } + + return xe_sriov_vf_ccs_init(xe); +} + +/** + * xe_sriov_vf_init_late() - SR-IOV VF late initialization functions. + * @xe: the &xe_device to initialize + * + * This function initializes code for CCS migration. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vf_init_late(struct xe_device *xe) +{ + return vf_migration_init_late(xe); +} + +static int sa_info_vf_ccs(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct xe_device *xe = to_xe_device(node->minor->dev); + struct drm_printer p = drm_seq_file_printer(m); + + xe_sriov_vf_ccs_print(xe, &p); + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + { .name = "sa_info_vf_ccs", .show = sa_info_vf_ccs }, +}; + +/** + * xe_sriov_vf_debugfs_register - Register VF debugfs attributes. + * @xe: the &xe_device + * @root: the root &dentry + * + * Prepare debugfs attributes exposed by the VF. + */ +void xe_sriov_vf_debugfs_register(struct xe_device *xe, struct dentry *root) +{ + drm_debugfs_create_files(debugfs_list, ARRAY_SIZE(debugfs_list), + root, xe->drm.primary); +} diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.h b/drivers/gpu/drm/xe/xe_sriov_vf.h new file mode 100644 index 000000000..e967d4166 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2024 Intel Corporation + */ + +#ifndef _XE_SRIOV_VF_H_ +#define _XE_SRIOV_VF_H_ + +#include + +struct dentry; +struct xe_device; + +void xe_sriov_vf_init_early(struct xe_device *xe); +int xe_sriov_vf_init_late(struct xe_device *xe); +bool xe_sriov_vf_migration_supported(struct xe_device *xe); +void xe_sriov_vf_migration_disable(struct xe_device *xe, const char *fmt, ...); +void xe_sriov_vf_debugfs_register(struct xe_device *xe, struct dentry *root); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c new file mode 100644 index 000000000..a8c831fbe --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "instructions/xe_mi_commands.h" +#include "instructions/xe_gpu_commands.h" +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_exec_queue_types.h" +#include "xe_gt_sriov_vf.h" +#include "xe_guc.h" +#include "xe_guc_submit.h" +#include "xe_lrc.h" +#include "xe_mem_pool.h" +#include "xe_migrate.h" +#include "xe_pm.h" +#include "xe_sriov_printk.h" +#include "xe_sriov_vf.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_sriov_vf_ccs_types.h" + +/** + * DOC: VF save/restore of compression Meta Data + * + * VF KMD registers two special contexts/LRCAs. + * + * Save Context/LRCA: contain necessary cmds+page table to trigger Meta data / + * compression control surface (Aka CCS) save in regular System memory in VM. + * + * Restore Context/LRCA: contain necessary cmds+page table to trigger Meta data / + * compression control surface (Aka CCS) Restore from regular System memory in + * VM to corresponding CCS pool. + * + * Below diagram explain steps needed for VF save/Restore of compression Meta Data:: + * + * CCS Save CCS Restore VF KMD Guc BCS + * LRCA LRCA + * | | | | | + * | | | | | + * | Create Save LRCA | | | + * [ ]<----------------------------- [ ] | | + * | | | | | + * | | | | | + * | | | Register save LRCA | | + * | | | with Guc | | + * | | [ ]--------------------------->[ ] | + * | | | | | + * | | Create restore LRCA | | | + * | [ ]<------------------[ ] | | + * | | | | | + * | | | Register restore LRCA | | + * | | | with Guc | | + * | | [ ]--------------------------->[ ] | + * | | | | | + * | | | | | + * | | [ ]------------------------- | | + * | | [ ] Allocate main memory. | | | + * | | [ ] Allocate CCS memory. | | | + * | | [ ] Update Main memory & | | | + * [ ]<------------------------------[ ] CCS pages PPGTT + BB | | | + * | [ ]<------------------[ ] cmds to save & restore.| | | + * | | [ ]<------------------------ | | + * | | | | | + * | | | | | + * | | | | | + * : : : : : + * ---------------------------- VF Paused ------------------------------------- + * | | | | | + * | | | | | + * | | | |Schedule | + * | | | |CCS Save | + * | | | | LRCA | + * | | | [ ]------>[ ] + * | | | | | + * | | | | | + * | | | |CCS save | + * | | | |completed| + * | | | [ ]<------[ ] + * | | | | | + * : : : : : + * ---------------------------- VM Migrated ----------------------------------- + * | | | | | + * | | | | | + * : : : : : + * ---------------------------- VF Resumed ------------------------------------ + * | | | | | + * | | | | | + * | | [ ]-------------- | | + * | | [ ] Fix up GGTT | | | + * | | [ ]<------------- | | + * | | | | | + * | | | | | + * | | | Notify VF_RESFIX_DONE | | + * | | [ ]--------------------------->[ ] | + * | | | | | + * | | | |Schedule | + * | | | |CCS | + * | | | |Restore | + * | | | |LRCA | + * | | | [ ]------>[ ] + * | | | | | + * | | | | | + * | | | |CCS | + * | | | |restore | + * | | | |completed| + * | | | [ ]<------[ ] + * | | | | | + * | | | | | + * | | | VF_RESFIX_DONE complete | | + * | | | notification | | + * | | [ ]<---------------------------[ ] | + * | | | | | + * | | | | | + * : : : : : + * ------------------------- Continue VM restore ------------------------------ + */ + +static u64 get_ccs_bb_pool_size(struct xe_device *xe) +{ + u64 sys_mem_size, ccs_mem_size, ptes, bb_pool_size; + struct sysinfo si; + + si_meminfo(&si); + sys_mem_size = si.totalram * si.mem_unit; + ccs_mem_size = div64_u64(sys_mem_size, NUM_BYTES_PER_CCS_BYTE(xe)); + ptes = DIV_ROUND_UP_ULL(sys_mem_size + ccs_mem_size, XE_PAGE_SIZE); + + /** + * We need below BB size to hold PTE mappings and some DWs for copy + * command. In reality, we need space for many copy commands. So, let + * us allocate double the calculated size which is enough to holds GPU + * instructions for the whole region. + */ + bb_pool_size = ptes * sizeof(u32); + + return round_up(bb_pool_size * 2, SZ_1M); +} + +static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) +{ + struct xe_mem_pool *pool; + struct xe_device *xe = tile_to_xe(tile); + u32 *pool_cpu_addr, *last_dw_addr; + u64 bb_pool_size; + int err; + + bb_pool_size = get_ccs_bb_pool_size(xe); + xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n", + ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M); + + pool = xe_mem_pool_init(tile, bb_pool_size, sizeof(u32), + XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY); + if (IS_ERR(pool)) { + xe_sriov_err(xe, "xe_mem_pool_init failed with error: %pe\n", + pool); + err = PTR_ERR(pool); + return err; + } + + pool_cpu_addr = xe_mem_pool_cpu_addr(pool); + memset(pool_cpu_addr, 0, bb_pool_size); + + last_dw_addr = pool_cpu_addr + (bb_pool_size / sizeof(u32)) - 1; + *last_dw_addr = MI_BATCH_BUFFER_END; + + /** + * Sync the main copy and shadow copy so that the shadow copy is + * replica of main copy. We sync only BBs after init part. So, we + * need to make sure the main pool and shadow copy are in sync after + * this point. This is needed as GuC may read the BB commands from + * shadow copy. + */ + xe_mem_pool_sync(pool); + + ctx->mem.ccs_bb_pool = pool; + return 0; +} + +static void ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx *ctx) +{ + u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool); + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); + u32 dw[10], i = 0; + + /* + * XXX: Save/restore fixes — for some reason, the GuC only accepts the + * save/restore context if the LRC head pointer is zero. This is evident + * from repeated VF migrations failing when the LRC head pointer is + * non-zero. + */ + lrc->ring.tail = 0; + xe_lrc_set_ring_head(lrc, 0); + + dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; + dw[i++] = MI_BATCH_BUFFER_START | XE_INSTR_NUM_DW(3); + dw[i++] = lower_32_bits(addr); + dw[i++] = upper_32_bits(addr); + dw[i++] = MI_NOOP; + dw[i++] = MI_NOOP; + + xe_lrc_write_ring(lrc, dw, i * sizeof(u32)); + xe_lrc_set_ring_tail(lrc, lrc->ring.tail); +} + +/** + * xe_sriov_vf_ccs_rebase - Rebase GGTT addresses for CCS save / restore + * @xe: the &xe_device. + */ +void xe_sriov_vf_ccs_rebase(struct xe_device *xe) +{ + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + + if (!IS_VF_CCS_READY(xe)) + return; + + for_each_ccs_rw_ctx(ctx_id) { + struct xe_sriov_vf_ccs_ctx *ctx = + &xe->sriov.vf.ccs.contexts[ctx_id]; + + ccs_rw_update_ring(ctx); + } +} + +static int register_save_restore_context(struct xe_sriov_vf_ccs_ctx *ctx) +{ + int ctx_type; + + switch (ctx->ctx_id) { + case XE_SRIOV_VF_CCS_READ_CTX: + ctx_type = GUC_CONTEXT_COMPRESSION_SAVE; + break; + case XE_SRIOV_VF_CCS_WRITE_CTX: + ctx_type = GUC_CONTEXT_COMPRESSION_RESTORE; + break; + default: + return -EINVAL; + } + + xe_guc_register_vf_exec_queue(ctx->mig_q, ctx_type); + return 0; +} + +/** + * xe_sriov_vf_ccs_register_context - Register read/write contexts with guc. + * @xe: the &xe_device to register contexts on. + * + * This function registers read and write contexts with Guc. Re-registration + * is needed whenever resuming from pm runtime suspend. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_sriov_vf_ccs_register_context(struct xe_device *xe) +{ + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_sriov_vf_ccs_ctx *ctx; + int err; + + xe_assert(xe, IS_VF_CCS_READY(xe)); + + for_each_ccs_rw_ctx(ctx_id) { + ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; + err = register_save_restore_context(ctx); + if (err) + return err; + } + + return err; +} + +/* + * Whether GuC requires CCS copy BBs for VF migration. + * @xe: the &xe_device instance. + * + * Only selected platforms require VF KMD to maintain CCS copy BBs and linked LRCAs. + * + * Return: true if VF driver must participate in the CCS migration, false otherwise. + */ +static bool vf_migration_ccs_bb_needed(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_VF(xe)); + + return !IS_DGFX(xe) && xe_device_has_flat_ccs(xe); +} + +/* + * Check for disable migration due to no CCS BBs support in GuC FW. + * @xe: the &xe_device instance. + * + * Performs late disable of VF migration feature in case GuC FW cannot support it. + * + * Returns: True if VF migration with CCS BBs is supported, false otherwise. + */ +static bool vf_migration_ccs_bb_support_check(struct xe_device *xe) +{ + struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_uc_fw_version guc_version; + + xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version); + if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 23, 0)) { + xe_sriov_vf_migration_disable(xe, + "CCS migration requires GuC ABI >= 1.23 but only %u.%u found", + guc_version.major, guc_version.minor); + return false; + } + + return true; +} + +static void xe_sriov_vf_ccs_fini(void *arg) +{ + struct xe_sriov_vf_ccs_ctx *ctx = arg; + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); + + /* + * Make TAIL = HEAD in the ring so that no issues are seen if Guc + * submits this context to HW on VF pause after unbinding device. + */ + xe_lrc_set_ring_tail(lrc, xe_lrc_ring_head(lrc)); + xe_exec_queue_put(ctx->mig_q); +} + +/** + * xe_sriov_vf_ccs_init - Setup LRCA for save & restore. + * @xe: the &xe_device to start recovery on + * + * This function shall be called only by VF. It initializes + * LRCA and suballocator needed for CCS save & restore. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_sriov_vf_ccs_init(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_exec_queue *q; + u32 flags; + int err; + + xe_assert(xe, IS_SRIOV_VF(xe)); + + if (!xe_sriov_vf_migration_supported(xe) || + !vf_migration_ccs_bb_needed(xe) || + !vf_migration_ccs_bb_support_check(xe)) + return 0; + + for_each_ccs_rw_ctx(ctx_id) { + ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; + ctx->ctx_id = ctx_id; + + flags = EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_PERMANENT | + EXEC_QUEUE_FLAG_MIGRATE; + q = xe_exec_queue_create_bind(xe, tile, NULL, flags, 0); + if (IS_ERR(q)) { + err = PTR_ERR(q); + goto err_ret; + } + ctx->mig_q = q; + + err = alloc_bb_pool(tile, ctx); + if (err) + goto err_free_queue; + + ccs_rw_update_ring(ctx); + + err = register_save_restore_context(ctx); + if (err) + goto err_free_queue; + + err = devm_add_action_or_reset(xe->drm.dev, + xe_sriov_vf_ccs_fini, + ctx); + if (err) + goto err_ret; + } + + xe->sriov.vf.ccs.initialized = 1; + + return 0; + +err_free_queue: + xe_exec_queue_put(q); + +err_ret: + return err; +} + +#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32)) +void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx) +{ + u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool); + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); + struct xe_device *xe = gt_to_xe(ctx->mig_q->gt); + + xe_device_wmb(xe); + xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr); + xe_device_wmb(xe); +} + +/** + * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO. + * @bo: the &buffer object to which batch buffer commands will be added. + * @new_mem: the (not yet committed) destination resource @bo is being moved + * into; bo->ttm.resource is still the old resource at this point. + * + * This function shall be called only by VF. It inserts the PTEs and copy + * command instructions in the BO by calling xe_migrate_ccs_rw_copy() + * function. + * + * Returns: 0 if successful, negative error code on failure. + */ +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem) +{ + struct xe_device *xe = xe_bo_device(bo); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_mem_pool_node *bb; + struct xe_tile *tile; + int err = 0; + + xe_assert(xe, IS_VF_CCS_READY(xe)); + + tile = xe_device_get_root_tile(xe); + + for_each_ccs_rw_ctx(ctx_id) { + bb = bo->bb_ccs[ctx_id]; + /* bb should be NULL here. Assert if not NULL */ + xe_assert(xe, !bb); + + ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; + err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, new_mem, ctx_id); + if (err) + goto err_unwind; + } + return 0; + +err_unwind: + /* + * Clean up any contexts already attached. Can't reuse + * xe_sriov_vf_ccs_detach_bo() here as it requires both contexts + * attached before cleaning up either one. + */ + for_each_ccs_rw_ctx(ctx_id) { + if (bo->bb_ccs[ctx_id]) + xe_migrate_ccs_rw_copy_clear(bo, ctx_id, true); + } + return err; +} + +/** + * xe_sriov_vf_ccs_detach_bo - Remove CCS read write commands from the BO. + * @bo: the &buffer object from which batch buffer commands will be removed. + * + * This function shall be called only by VF. It removes the PTEs and copy + * command instructions from the BO. Make sure to update the BB with MI_NOOP + * before freeing. + * + * Returns: 0 if successful. + */ +int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_mem_pool_node *bb; + bool bound; + int idx; + + xe_assert(xe, IS_VF_CCS_READY(xe)); + + if (!xe_bo_has_valid_ccs_bb(bo)) + return 0; + + bound = drm_dev_enter(&xe->drm, &idx); + + for_each_ccs_rw_ctx(ctx_id) { + bb = bo->bb_ccs[ctx_id]; + if (!bb) + continue; + + xe_migrate_ccs_rw_copy_clear(bo, ctx_id, bound); + } + + if (bound) + drm_dev_exit(idx); + + return 0; +} + +/** + * xe_sriov_vf_ccs_print - Print VF CCS details. + * @xe: the &xe_device + * @p: the &drm_printer + * + * This function is for VF use only. + */ +void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p) +{ + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_mem_pool *bb_pool; + + if (!IS_VF_CCS_READY(xe)) + return; + + guard(xe_pm_runtime)(xe); + for_each_ccs_rw_ctx(ctx_id) { + bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool; + if (!bb_pool) + break; + + drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read"); + drm_printf(p, "-------------------------\n"); + xe_mem_pool_dump(bb_pool, p); + drm_puts(p, "\n"); + } +} diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h new file mode 100644 index 000000000..e1034d852 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_VF_CCS_H_ +#define _XE_SRIOV_VF_CCS_H_ + +#include "xe_device_types.h" +#include "xe_sriov.h" +#include "xe_sriov_vf_ccs_types.h" + +struct drm_printer; +struct ttm_resource; +struct xe_device; +struct xe_bo; + +int xe_sriov_vf_ccs_init(struct xe_device *xe); +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem); +int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); +int xe_sriov_vf_ccs_register_context(struct xe_device *xe); +void xe_sriov_vf_ccs_rebase(struct xe_device *xe); +void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p); +void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx); + +static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_VF(xe)); + return xe->sriov.vf.ccs.initialized; +} + +#define IS_VF_CCS_READY(xe) ({ \ + struct xe_device *xe__ = (xe); \ + IS_SRIOV_VF(xe__) && xe_sriov_vf_ccs_ready(xe__); \ + }) + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h new file mode 100644 index 000000000..6fc8f97ef --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_VF_CCS_TYPES_H_ +#define _XE_SRIOV_VF_CCS_TYPES_H_ + +#include + +#define for_each_ccs_rw_ctx(id__) \ + for ((id__) = 0; (id__) < XE_SRIOV_VF_CCS_CTX_COUNT; (id__)++) + +enum xe_sriov_vf_ccs_rw_ctxs { + XE_SRIOV_VF_CCS_READ_CTX, + XE_SRIOV_VF_CCS_WRITE_CTX, + XE_SRIOV_VF_CCS_CTX_COUNT +}; + +/** + * struct xe_sriov_vf_ccs_ctx - VF CCS migration context data. + */ +struct xe_sriov_vf_ccs_ctx { + /** @ctx_id: Id to which context it belongs to */ + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + + /** @mig_q: exec queues used for migration */ + struct xe_exec_queue *mig_q; + + /** @mem: memory data */ + struct { + /** @mem.ccs_bb_pool: Pool from which batch buffers are allocated. */ + struct xe_mem_pool *ccs_bb_pool; + } mem; +}; + +/** + * struct xe_sriov_vf_ccs - The VF CCS migration support data. + */ +struct xe_sriov_vf_ccs { + /** @contexts: CCS read and write contexts for VF. */ + struct xe_sriov_vf_ccs_ctx contexts[XE_SRIOV_VF_CCS_CTX_COUNT]; + + /** @initialized: Initialization of VF CCS is completed or not. */ + bool initialized; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_types.h new file mode 100644 index 000000000..d5f72d667 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_types.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023-2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_VF_TYPES_H_ +#define _XE_SRIOV_VF_TYPES_H_ + +#include +#include + +#include "xe_sriov_vf_ccs_types.h" + +/** + * struct xe_sriov_vf_relay_version - PF ABI version details. + */ +struct xe_sriov_vf_relay_version { + /** @major: major version. */ + u16 major; + /** @minor: minor version. */ + u16 minor; +}; + +/** + * struct xe_device_vf - Xe Virtual Function related data + * + * The data in this structure is valid only if driver is running in the + * @XE_SRIOV_MODE_VF mode. + */ +struct xe_device_vf { + /** @pf_version: negotiated VF/PF ABI version. */ + struct xe_sriov_vf_relay_version pf_version; + + /** @migration: VF Migration state data */ + struct { + /** + * @migration.disabled: flag indicating if migration support + * was turned off due to missing prerequisites + */ + bool disabled; + } migration; + + /** @ccs: VF CCS state data */ + struct xe_sriov_vf_ccs ccs; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vfio.c b/drivers/gpu/drm/xe/xe_sriov_vfio.c new file mode 100644 index 000000000..00f96b097 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vfio.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "xe_pci.h" +#include "xe_pm.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" + +struct xe_device *xe_sriov_vfio_get_pf(struct pci_dev *pdev) +{ + return xe_pci_to_pf_device(pdev); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_get_pf, "xe-vfio-pci"); + +bool xe_sriov_vfio_migration_supported(struct xe_device *xe) +{ + if (!IS_SRIOV_PF(xe)) + return false; + + return xe_sriov_pf_migration_supported(xe); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_migration_supported, "xe-vfio-pci"); + +#define DEFINE_XE_SRIOV_VFIO_FUNCTION(_type, _func, _impl) \ +_type xe_sriov_vfio_##_func(struct xe_device *xe, unsigned int vfid) \ +{ \ + if (!IS_SRIOV_PF(xe)) \ + return -EPERM; \ + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) \ + return -EINVAL; \ + \ + guard(xe_pm_runtime_noresume)(xe); \ + \ + return xe_sriov_pf_##_impl(xe, vfid); \ +} \ +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_##_func, "xe-vfio-pci") + +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, wait_flr_done, control_wait_flr); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, flr_prepare, control_prepare_flr); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, suspend_device, control_pause_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_device, control_resume_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_enter, control_trigger_save_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_exit, control_finish_save_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_enter, control_trigger_restore_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_exit, control_finish_restore_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, error, control_stop_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(ssize_t, stop_copy_size, migration_size); + +ssize_t xe_sriov_vfio_data_read(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) + return -EINVAL; + + guard(xe_pm_runtime_noresume)(xe); + + return xe_sriov_pf_migration_read(xe, vfid, buf, len); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_read, "xe-vfio-pci"); + +ssize_t xe_sriov_vfio_data_write(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) + return -EINVAL; + + guard(xe_pm_runtime_noresume)(xe); + + return xe_sriov_pf_migration_write(xe, vfid, buf, len); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_write, "xe-vfio-pci"); diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c new file mode 100644 index 000000000..55c1996f6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_step.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_step.h" + +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_platform_types.h" + +/* + * Provide mapping between PCI's revision ID to the individual GMD + * (Graphics/Media/Display) stepping values that can be compared numerically. + * + * Some platforms may have unusual ways of mapping PCI revision ID to GMD + * steppings. E.g., in some cases a higher PCI revision may translate to a + * lower stepping of the GT and/or display IP. + * + * Also note that some revisions/steppings may have been set aside as + * placeholders but never materialized in real hardware; in those cases there + * may be jumps in the revision IDs or stepping values in the tables below. + */ + +/* + * Some platforms always have the same stepping value for GT and display; + * use a macro to define these to make it easier to identify the platforms + * where the two steppings can deviate. + */ +#define COMMON_STEP(x_) \ + .graphics = STEP_##x_, \ + .media = STEP_##x_ + +__diag_push(); +__diag_ignore_all("-Woverride-init", "Allow field overrides in table"); + +/* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */ +static const struct xe_step_info tgl_revids[] = { + [0] = { COMMON_STEP(A0) }, + [1] = { COMMON_STEP(B0) }, +}; + +static const struct xe_step_info dg1_revids[] = { + [0] = { COMMON_STEP(A0) }, + [1] = { COMMON_STEP(B0) }, +}; + +static const struct xe_step_info adls_revids[] = { + [0x0] = { COMMON_STEP(A0) }, + [0x1] = { COMMON_STEP(A0) }, + [0x4] = { COMMON_STEP(B0) }, + [0x8] = { COMMON_STEP(C0) }, + [0xC] = { COMMON_STEP(D0) }, +}; + +static const struct xe_step_info adls_rpls_revids[] = { + [0x4] = { COMMON_STEP(D0) }, + [0xC] = { COMMON_STEP(D0) }, +}; + +static const struct xe_step_info adlp_revids[] = { + [0x0] = { COMMON_STEP(A0) }, + [0x4] = { COMMON_STEP(B0) }, + [0x8] = { COMMON_STEP(C0) }, + [0xC] = { COMMON_STEP(C0) }, +}; + +static const struct xe_step_info adlp_rpl_revids[] = { + [0x4] = { COMMON_STEP(C0) }, +}; + +static const struct xe_step_info adln_revids[] = { + [0x0] = { COMMON_STEP(A0) }, +}; + +static const struct xe_step_info dg2_g10_revid_step_tbl[] = { + [0x0] = { COMMON_STEP(A0) }, + [0x1] = { COMMON_STEP(A1) }, + [0x4] = { COMMON_STEP(B0) }, + [0x8] = { COMMON_STEP(C0) }, +}; + +static const struct xe_step_info dg2_g11_revid_step_tbl[] = { + [0x0] = { COMMON_STEP(A0) }, + [0x4] = { COMMON_STEP(B0) }, + [0x5] = { COMMON_STEP(B1) }, +}; + +static const struct xe_step_info dg2_g12_revid_step_tbl[] = { + [0x0] = { COMMON_STEP(A0) }, + [0x1] = { COMMON_STEP(A1) }, +}; + +static const struct xe_step_info pvc_revid_step_tbl[] = { + [0x5] = { .graphics = STEP_B0 }, + [0x6] = { .graphics = STEP_B1 }, + [0x7] = { .graphics = STEP_C0 }, +}; + +static const int pvc_basedie_subids[] = { + [0x3] = STEP_B0, + [0x4] = STEP_B1, + [0x5] = STEP_B3, +}; + +__diag_pop(); + +/** + * xe_step_platform_get - Determine platform-level stepping from PCI revid + * @platform: The Xe platform + * @revid: The PCI revid + * @step: Pointer to the step struct to update + * + * Convert the PCI revid into a platform-level stepping value and store that + * in @step->platform. + */ +void xe_step_platform_get(enum xe_platform platform, u8 revid, struct xe_step_info *step) +{ + /* + * Not all platforms map PCI revid directly into our symbolic stepping + * enumeration. Some platforms will have a single PCI revid used for a + * range platform level steppings and some might even require specific + * mappings. So prefer to err on the side of caution and include only + * the platforms from which we need the stepping info for workaround + * checks. + */ + + if (platform == XE_NOVALAKE_P) + step->platform = STEP_A0 + revid; +} + +/** + * xe_step_pre_gmdid_get - Determine IP steppings from PCI revid + * @xe: Xe device + * @step: Pointer to the step struct to update + * + * Convert the PCI revid into proper IP steppings and update @step->basedie, + * @step->graphics and @step->media accordingly. + * + * This should only be used on platforms that do not have GMD_ID support. + */ +void xe_step_pre_gmdid_get(struct xe_device *xe, struct xe_step_info *step) +{ + const struct xe_step_info *revids = NULL; + u16 revid = xe->info.revid; + int size = 0; + const int *basedie_info = NULL; + int basedie_size = 0; + int baseid = 0; + u8 graphics = STEP_NONE; + u8 media = STEP_NONE; + u8 basedie = STEP_NONE; + + if (xe->info.platform == XE_PVC) { + baseid = FIELD_GET(GENMASK(5, 3), xe->info.revid); + revid = FIELD_GET(GENMASK(2, 0), xe->info.revid); + revids = pvc_revid_step_tbl; + size = ARRAY_SIZE(pvc_revid_step_tbl); + basedie_info = pvc_basedie_subids; + basedie_size = ARRAY_SIZE(pvc_basedie_subids); + } else if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G10) { + revids = dg2_g10_revid_step_tbl; + size = ARRAY_SIZE(dg2_g10_revid_step_tbl); + } else if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G11) { + revids = dg2_g11_revid_step_tbl; + size = ARRAY_SIZE(dg2_g11_revid_step_tbl); + } else if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G12) { + revids = dg2_g12_revid_step_tbl; + size = ARRAY_SIZE(dg2_g12_revid_step_tbl); + } else if (xe->info.platform == XE_ALDERLAKE_N) { + revids = adln_revids; + size = ARRAY_SIZE(adln_revids); + } else if (xe->info.subplatform == XE_SUBPLATFORM_ALDERLAKE_S_RPLS) { + revids = adls_rpls_revids; + size = ARRAY_SIZE(adls_rpls_revids); + } else if (xe->info.subplatform == XE_SUBPLATFORM_ALDERLAKE_P_RPLU) { + revids = adlp_rpl_revids; + size = ARRAY_SIZE(adlp_rpl_revids); + } else if (xe->info.platform == XE_ALDERLAKE_P) { + revids = adlp_revids; + size = ARRAY_SIZE(adlp_revids); + } else if (xe->info.platform == XE_ALDERLAKE_S) { + revids = adls_revids; + size = ARRAY_SIZE(adls_revids); + } else if (xe->info.platform == XE_DG1) { + revids = dg1_revids; + size = ARRAY_SIZE(dg1_revids); + } else if (xe->info.platform == XE_TIGERLAKE) { + revids = tgl_revids; + size = ARRAY_SIZE(tgl_revids); + } + + /* Not using the stepping scheme for the platform yet. */ + if (!revids) + goto done; + + if (revid < size && revids[revid].graphics != STEP_NONE) { + graphics = revids[revid].graphics; + media = revids[revid].media; + basedie = revids[revid].basedie; + } else { + drm_warn(&xe->drm, "Unknown revid 0x%02x\n", revid); + + /* + * If we hit a gap in the revid array, use the information for + * the next revid. + * + * This may be wrong in all sorts of ways, especially if the + * steppings in the array are not monotonically increasing, but + * it's better than defaulting to 0. + */ + while (revid < size && revids[revid].graphics == STEP_NONE) + revid++; + + if (revid < size) { + drm_dbg(&xe->drm, "Using steppings for revid 0x%02x\n", + revid); + graphics = revids[revid].graphics; + media = revids[revid].media; + basedie = revids[revid].basedie; + } else { + drm_dbg(&xe->drm, "Using future steppings\n"); + graphics = STEP_FUTURE; + } + } + + drm_WARN_ON(&xe->drm, graphics == STEP_NONE); + + if (basedie_info && basedie_size) { + if (baseid < basedie_size && basedie_info[baseid] != STEP_NONE) { + basedie = basedie_info[baseid]; + } else { + drm_warn(&xe->drm, "Unknown baseid 0x%02x\n", baseid); + basedie = STEP_FUTURE; + } + } + +done: + step->graphics = graphics; + step->media = media; + step->basedie = basedie; +} + +/** + * xe_step_gmdid_get - Determine IP steppings from GMD_ID revid fields + * @xe: Xe device + * @graphics_gmdid_revid: value of graphics GMD_ID register's revid field + * @media_gmdid_revid: value of media GMD_ID register's revid field + * @step: Poninter to the step struct to update. + * + * Convert the revid fields of the GMD_ID registers into proper IP steppings + * and update @step->graphics and @step->media accordingly. + * + * GMD_ID revid values are currently expected to have consistent meanings on + * all platforms: major steppings (A0, B0, etc.) are 4 apart, with minor + * steppings (A1, A2, etc.) taking the values in between. + */ +void xe_step_gmdid_get(struct xe_device *xe, + u32 graphics_gmdid_revid, + u32 media_gmdid_revid, + struct xe_step_info *step) +{ + u8 graphics = STEP_A0 + graphics_gmdid_revid; + u8 media = STEP_A0 + media_gmdid_revid; + + if (graphics >= STEP_FUTURE) { + graphics = STEP_FUTURE; + drm_dbg(&xe->drm, "Graphics GMD_ID revid value %d treated as future stepping\n", + graphics_gmdid_revid); + } + + if (media >= STEP_FUTURE) { + media = STEP_FUTURE; + drm_dbg(&xe->drm, "Media GMD_ID revid value %d treated as future stepping\n", + media_gmdid_revid); + } + + step->graphics = graphics; + step->media = media; +} + +#define STEP_NAME_CASE(name) \ + case STEP_##name: \ + return #name; + +const char *xe_step_name(enum intel_step step) +{ + switch (step) { + STEP_NAME_LIST(STEP_NAME_CASE); + + default: + return "**"; + } +} +EXPORT_SYMBOL_IF_KUNIT(xe_step_name); diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h new file mode 100644 index 000000000..5a5845335 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_step.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_STEP_H_ +#define _XE_STEP_H_ + +#include + +#include "xe_step_types.h" + +enum xe_platform; +struct xe_device; + +void xe_step_platform_get(enum xe_platform platform, u8 revid, struct xe_step_info *step); + +void xe_step_pre_gmdid_get(struct xe_device *xe, struct xe_step_info *step); +void xe_step_gmdid_get(struct xe_device *xe, + u32 graphics_gmdid_revid, + u32 media_gmdid_revid, + struct xe_step_info *step); +static inline u32 xe_step_to_gmdid(enum intel_step step) { return step - STEP_A0; } + +const char *xe_step_name(enum intel_step step); + +#endif diff --git a/drivers/gpu/drm/xe/xe_step_types.h b/drivers/gpu/drm/xe/xe_step_types.h new file mode 100644 index 000000000..f60572b93 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_step_types.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_STEP_TYPES_H_ +#define _XE_STEP_TYPES_H_ + +#include + +#include + +struct xe_step_info { + u8 platform; + u8 graphics; + u8 media; + u8 basedie; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c new file mode 100644 index 000000000..4c506027f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_survivability_mode.h" +#include "xe_survivability_mode_types.h" + +#include +#include +#include + +#include "xe_configfs.h" +#include "xe_device.h" +#include "xe_heci_gsc.h" +#include "xe_i2c.h" +#include "xe_mmio.h" +#include "xe_nvm.h" +#include "xe_pcode_api.h" +#include "xe_vsec.h" + +/** + * DOC: Survivability Mode + * + * Survivability Mode is a software based workflow for recovering a system in a failed boot state + * Here system recoverability is concerned with recovering the firmware responsible for boot. + * + * Boot Survivability + * =================== + * + * Boot Survivability is implemented by loading the driver with bare minimum (no drm card) to allow + * the firmware to be flashed through mei driver and collect telemetry. The driver's probe flow is + * modified such that it enters survivability mode when pcode initialization is incomplete and boot + * status denotes a failure. + * + * Survivability mode can also be entered manually using the survivability mode attribute available + * through configfs which is beneficial in several usecases. It can be used to address scenarios + * where pcode does not detect failure or for validation purposes. It can also be used in + * In-Field-Repair (IFR) to repair a single card without impacting the other cards in a node. + * + * Use below command enable survivability mode manually:: + * + * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/survivability_mode + * + * It is the responsibility of the user to clear the mode once firmware flash is complete. + * + * Refer :ref:`xe_configfs` for more details on how to use configfs + * + * Survivability mode is indicated by the below admin-only readable sysfs entry. It + * provides information about the type of survivability mode (Boot/Runtime). + * + * .. code-block:: shell + * + * # cat /sys/bus/pci/devices//survivability_mode + * Boot + * + * Any additional debug information if present will be visible under the directory + * ``survivability_info``:: + * + * /sys/bus/pci/devices//survivability_info/ + * ├── aux_info0 + * ├── aux_info1 + * ├── aux_info2 + * ├── aux_info3 + * ├── aux_info4 + * ├── capability_info + * ├── fdo_mode + * ├── postcode_trace + * └── postcode_trace_overflow + * + * This directory has the following attributes + * + * - ``capability_info`` : Indicates Boot status and support for additional information + * + * - ``postcode_trace``, ``postcode_trace_overflow`` : Each postcode is a 8bit value and + * represents a boot failure event. When a new failure event is logged by PCODE the + * existing postcodes are shifted left. These entries provide a history of 8 postcodes. + * + * - ``aux_info`` : Some failures have additional debug information + * + * - ``fdo_mode`` : To allow recovery in scenarios where MEI itself fails, a new SPI Flash + * Descriptor Override (FDO) mode is added in v2 survivability breadcrumbs. This mode is enabled + * by PCODE and provides the ability to directly update the firmware via SPI Driver without + * any dependency on MEI. Xe KMD initializes the nvm aux driver if FDO mode is enabled. + * + * Runtime Survivability + * ===================== + * + * Certain runtime firmware errors can cause the device to enter a wedged state + * (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation. + * Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and + * is indicated by the presence of survivability mode sysfs. + * Survivability mode sysfs provides information about the type of survivability mode. + * + * .. code-block:: shell + * + * # cat /sys/bus/pci/devices//survivability_mode + * Runtime + * + * On some CSC firmware errors, PCODE sets FDO mode and the only recovery possible is through + * firmware flash using SPI driver. Userspace can check if FDO mode is set by checking the below + * sysfs entry. + * + * .. code-block:: shell + * + * # cat /sys/bus/pci/devices//survivability_info/fdo_mode + * enabled + * + * When such errors occur, userspace is notified with the drm device wedged uevent and runtime + * survivability mode. User can then initiate a firmware flash using userspace tools like fwupd + * to restore device to normal operation. + */ + +static const char * const reg_map[] = { + [CAPABILITY_INFO] = "Capability Info", + [POSTCODE_TRACE] = "Postcode trace", + [POSTCODE_TRACE_OVERFLOW] = "Postcode trace overflow", + [AUX_INFO0] = "Auxiliary Info 0", + [AUX_INFO1] = "Auxiliary Info 1", + [AUX_INFO2] = "Auxiliary Info 2", + [AUX_INFO3] = "Auxiliary Info 3", + [AUX_INFO4] = "Auxiliary Info 4", +}; + +#define FDO_INFO (MAX_SCRATCH_REG + 1) + +struct xe_survivability_attribute { + struct device_attribute attr; + u8 index; +}; + +static struct +xe_survivability_attribute *dev_attr_to_survivability_attr(struct device_attribute *attr) +{ + return container_of(attr, struct xe_survivability_attribute, attr); +} + +static void set_survivability_info(struct xe_mmio *mmio, u32 *info, int id) +{ + info[id] = xe_mmio_read32(mmio, PCODE_SCRATCH(id)); +} + +static void populate_survivability_info(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + u32 *info = survivability->info; + struct xe_mmio *mmio; + u32 id = 0, reg_value; + + mmio = xe_root_tile_mmio(xe); + set_survivability_info(mmio, info, CAPABILITY_INFO); + reg_value = info[CAPABILITY_INFO]; + + survivability->version = REG_FIELD_GET(BREADCRUMB_VERSION, reg_value); + /* FDO mode is exposed only from version 2 */ + if (survivability->version >= 2) + survivability->fdo_mode = REG_FIELD_GET(FDO_MODE, reg_value); + + if (reg_value & HISTORY_TRACKING) { + set_survivability_info(mmio, info, POSTCODE_TRACE); + + if (reg_value & OVERFLOW_SUPPORT) + set_survivability_info(mmio, info, POSTCODE_TRACE_OVERFLOW); + } + + /* Traverse the linked list of aux info registers */ + if (reg_value & AUXINFO_SUPPORT) { + for (id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value); + id >= AUX_INFO0 && id < MAX_SCRATCH_REG; + id = REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, info[id])) + set_survivability_info(mmio, info, id); + } +} + +static void log_survivability_info(struct pci_dev *pdev) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_survivability *survivability = &xe->survivability; + u32 *info = survivability->info; + int id; + + dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n", + survivability->boot_status); + for (id = 0; id < MAX_SCRATCH_REG; id++) { + if (info[id]) + dev_info(&pdev->dev, "%s: 0x%x\n", reg_map[id], info[id]); + } +} + +static int check_boot_failure(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + + return survivability->boot_status == NON_CRITICAL_FAILURE || + survivability->boot_status == CRITICAL_FAILURE; +} + +static ssize_t survivability_mode_show(struct device *dev, + struct device_attribute *attr, char *buff) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_survivability *survivability = &xe->survivability; + + return sysfs_emit(buff, "%s\n", survivability->type ? "Runtime" : "Boot"); +} + +static DEVICE_ATTR_ADMIN_RO(survivability_mode); + +static ssize_t survivability_info_show(struct device *dev, + struct device_attribute *attr, char *buff) +{ + struct xe_survivability_attribute *sa = dev_attr_to_survivability_attr(attr); + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_survivability *survivability = &xe->survivability; + u32 *info = survivability->info; + + if (sa->index == FDO_INFO) + return sysfs_emit(buff, "%s\n", str_enabled_disabled(survivability->fdo_mode)); + + return sysfs_emit(buff, "0x%x\n", info[sa->index]); +} + +#define SURVIVABILITY_ATTR_RO(name, _index) \ + struct xe_survivability_attribute attr_##name = { \ + .attr = __ATTR(name, 0400, survivability_info_show, NULL), \ + .index = _index, \ + } + +static SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO); +static SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE); +static SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW); +static SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0); +static SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1); +static SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2); +static SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3); +static SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4); +static SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO); + +static void xe_survivability_mode_fini(void *arg) +{ + struct xe_device *xe = arg; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct device *dev = &pdev->dev; + + device_remove_file(dev, &dev_attr_survivability_mode); +} + +static umode_t survivability_info_attrs_visible(struct kobject *kobj, struct attribute *attr, + int idx) +{ + struct xe_device *xe = kdev_to_xe_device(kobj_to_dev(kobj)); + struct xe_survivability *survivability = &xe->survivability; + u32 *info = survivability->info; + + /* + * Last index in survivability_info_attrs is fdo mode and is applicable only in + * version 2 of survivability mode + */ + if (idx == MAX_SCRATCH_REG && survivability->version >= 2) + return 0400; + + if (idx < MAX_SCRATCH_REG && info[idx]) + return 0400; + + return 0; +} + +/* Attributes are ordered according to enum scratch_reg */ +static struct attribute *survivability_info_attrs[] = { + &attr_capability_info.attr.attr, + &attr_postcode_trace.attr.attr, + &attr_postcode_trace_overflow.attr.attr, + &attr_aux_info0.attr.attr, + &attr_aux_info1.attr.attr, + &attr_aux_info2.attr.attr, + &attr_aux_info3.attr.attr, + &attr_aux_info4.attr.attr, + &attr_fdo_mode.attr.attr, + NULL, +}; + +static const struct attribute_group survivability_info_group = { + .name = "survivability_info", + .attrs = survivability_info_attrs, + .is_visible = survivability_info_attrs_visible, +}; + +static int create_survivability_sysfs(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct xe_device *xe = pdev_to_xe_device(pdev); + int ret; + + ret = device_create_file(dev, &dev_attr_survivability_mode); + if (ret) { + dev_warn(dev, "Failed to create survivability sysfs files\n"); + return ret; + } + + ret = devm_add_action_or_reset(xe->drm.dev, + xe_survivability_mode_fini, xe); + if (ret) + return ret; + + /* Survivability info is not required if enabled via configfs */ + if (!xe_configfs_get_survivability_mode(pdev)) { + ret = devm_device_add_group(dev, &survivability_info_group); + if (ret) + return ret; + } + + return 0; +} + +static int enable_boot_survivability_mode(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_survivability *survivability = &xe->survivability; + int ret = 0; + + ret = create_survivability_sysfs(pdev); + if (ret) + return ret; + + /* Make sure xe_heci_gsc_init() and xe_i2c_probe() are aware of survivability */ + survivability->mode = true; + + xe_heci_gsc_init(xe); + + xe_vsec_init(xe); + + if (survivability->fdo_mode) { + ret = xe_nvm_init(xe); + if (ret) + goto err; + } + + ret = xe_i2c_probe(xe); + if (ret) + goto err; + + dev_err(dev, "In Survivability Mode\n"); + + return 0; + +err: + dev_err(dev, "Failed to enable Survivability Mode\n"); + survivability->mode = false; + return ret; +} + +/** + * xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled + * @xe: xe device instance + * + * Returns true if in boot survivability mode of type, else false + */ +bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + + return survivability->mode && survivability->type == XE_SURVIVABILITY_TYPE_BOOT; +} + +/** + * xe_survivability_mode_is_requested - check if it's possible to enable survivability + * mode that was requested by firmware or userspace + * @xe: xe device instance + * + * This function reads configfs and boot status from Pcode. + * + * Return: true if platform support is available and boot status indicates + * failure or if survivability mode is requested, false otherwise. + */ +bool xe_survivability_mode_is_requested(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + u32 data; + bool survivability_mode; + + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) + return false; + + survivability_mode = xe_configfs_get_survivability_mode(pdev); + /* Enable survivability mode if set via configfs */ + if (survivability_mode) + return true; + + data = xe_mmio_read32(mmio, PCODE_SCRATCH(0)); + survivability->boot_status = REG_FIELD_GET(BOOT_STATUS, data); + + return check_boot_failure(xe); +} + +/** + * xe_survivability_mode_runtime_enable - Initialize and enable runtime survivability mode + * @xe: xe device instance + * + * Initialize survivability information and enable runtime survivability mode. + * Runtime survivability mode is enabled when certain errors cause the device to be + * in non-recoverable state. The device is declared wedged with the appropriate + * recovery method and survivability mode sysfs exposed to userspace + */ +void xe_survivability_mode_runtime_enable(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) { + dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n"); + return; + } + + populate_survivability_info(xe); + + if (create_survivability_sysfs(pdev)) + dev_err(&pdev->dev, "Failed to create survivability sysfs\n"); + + survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME; + dev_err(&pdev->dev, "Runtime Survivability mode enabled\n"); + + xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); + xe_device_declare_wedged(xe); + dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n"); +} + +/** + * xe_survivability_mode_boot_enable - Initialize and enable boot survivability mode + * @xe: xe device instance + * + * Initialize survivability information and enable boot survivability mode + * + * Return: 0 if boot survivability mode is enabled or not requested, negative error + * code otherwise. + */ +int xe_survivability_mode_boot_enable(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + if (!xe_survivability_mode_is_requested(xe)) + return 0; + + populate_survivability_info(xe); + + /* + * v2 supports survivability mode for critical errors + */ + if (survivability->version < 2 && survivability->boot_status == CRITICAL_FAILURE) { + log_survivability_info(pdev); + return -ENXIO; + } + + survivability->type = XE_SURVIVABILITY_TYPE_BOOT; + + return enable_boot_survivability_mode(pdev); +} diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h new file mode 100644 index 000000000..cd040e4d1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_survivability_mode.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SURVIVABILITY_MODE_H_ +#define _XE_SURVIVABILITY_MODE_H_ + +#include + +struct xe_device; + +int xe_survivability_mode_boot_enable(struct xe_device *xe); +void xe_survivability_mode_runtime_enable(struct xe_device *xe); +bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe); +bool xe_survivability_mode_is_requested(struct xe_device *xe); + +#endif /* _XE_SURVIVABILITY_MODE_H_ */ diff --git a/drivers/gpu/drm/xe/xe_survivability_mode_types.h b/drivers/gpu/drm/xe/xe_survivability_mode_types.h new file mode 100644 index 000000000..bd5dc1c95 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_survivability_mode_types.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SURVIVABILITY_MODE_TYPES_H_ +#define _XE_SURVIVABILITY_MODE_TYPES_H_ + +#include +#include + +enum scratch_reg { + CAPABILITY_INFO, + POSTCODE_TRACE, + POSTCODE_TRACE_OVERFLOW, + AUX_INFO0, + AUX_INFO1, + AUX_INFO2, + AUX_INFO3, + AUX_INFO4, + MAX_SCRATCH_REG, +}; + +enum xe_survivability_type { + XE_SURVIVABILITY_TYPE_BOOT, + XE_SURVIVABILITY_TYPE_RUNTIME, +}; + +/** + * struct xe_survivability: Contains survivability mode information + */ +struct xe_survivability { + /** @info: survivability debug info */ + u32 info[MAX_SCRATCH_REG]; + + /** @size: number of scratch registers */ + u32 size; + + /** @boot_status: indicates critical/non critical boot failure */ + u8 boot_status; + + /** @mode: boolean to indicate survivability mode */ + bool mode; + + /** @type: survivability type */ + enum xe_survivability_type type; + + /** @fdo_mode: indicates if FDO mode is enabled */ + bool fdo_mode; + + /** @version: breadcrumb version of survivability mode */ + u8 version; +}; + +#endif /* _XE_SURVIVABILITY_MODE_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c new file mode 100644 index 000000000..b228a737c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -0,0 +1,2085 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_exec_queue_types.h" +#include "xe_gt_stats.h" +#include "xe_migrate.h" +#include "xe_module.h" +#include "xe_pm.h" +#include "xe_pt.h" +#include "xe_svm.h" +#include "xe_tile.h" +#include "xe_tlb_inval.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vm.h" +#include "xe_vm_types.h" +#include "xe_vram_types.h" + +/* Identifies subclasses of struct drm_pagemap_peer */ +#define XE_PEER_PAGEMAP ((void *)0ul) +#define XE_PEER_VM ((void *)1ul) + +/** + * DOC: drm_pagemap reference-counting in xe: + * + * In addition to the drm_pagemap internal reference counting by its zone + * device data, the xe driver holds the following long-time references: + * + * - struct xe_pagemap: + * The xe_pagemap struct derives from struct drm_pagemap and uses its + * reference count. + * - SVM-enabled VMs: + * SVM-enabled VMs look up and keeps a reference to all xe_pagemaps on + * the same device. + * - VMAs: + * vmas keep a reference on the drm_pagemap indicated by a gpu_madvise() + * call. + * + * In addition, all drm_pagemap or xe_pagemap pointers where lifetime cannot + * be guaranteed by a vma reference under the vm lock should keep a reference. + * That includes the range->pages.dpagemap pointer. + */ + +static int xe_svm_get_pagemaps(struct xe_vm *vm); + +void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) +{ + return force_smem ? NULL : vm->svm.peer.owner; +} + +static bool xe_svm_range_in_vram(struct xe_svm_range *range) +{ + /* + * Advisory only check whether the range is currently backed by VRAM + * memory. + */ + + struct drm_gpusvm_pages_flags flags = { + /* Pairs with WRITE_ONCE in drm_gpusvm.c */ + .__flags = READ_ONCE(range->pages.flags.__flags), + }; + + return flags.has_devmem_pages; +} + +static bool xe_svm_range_has_vram_binding(struct xe_svm_range *range) +{ + /* Not reliable without notifier lock */ + return xe_svm_range_in_vram(range) && range->tile_present; +} + +static struct xe_vm *gpusvm_to_vm(struct drm_gpusvm *gpusvm) +{ + return container_of(gpusvm, struct xe_vm, svm.gpusvm); +} + +static struct xe_vm *range_to_vm(struct drm_gpusvm_range *r) +{ + return gpusvm_to_vm(r->gpusvm); +} + +#define range_debug(r__, operation__) \ + vm_dbg(&range_to_vm(&(r__)->base)->xe->drm, \ + "%s: asid=%u, gpusvm=%p, vram=%d,%d, seqno=%lu, " \ + "start=0x%014lx, end=0x%014lx, size=%lu", \ + (operation__), range_to_vm(&(r__)->base)->usm.asid, \ + (r__)->base.gpusvm, \ + xe_svm_range_in_vram((r__)) ? 1 : 0, \ + xe_svm_range_has_vram_binding((r__)) ? 1 : 0, \ + (r__)->pages.notifier_seq, \ + xe_svm_range_start((r__)), xe_svm_range_end((r__)), \ + xe_svm_range_size((r__))) + +void xe_svm_range_debug(struct xe_svm_range *range, const char *operation) +{ + range_debug(range, operation); +} + +static struct drm_gpusvm_range * +xe_svm_range_alloc(struct drm_gpusvm *gpusvm) +{ + struct xe_svm_range *range; + + range = kzalloc_obj(*range); + if (!range) + return NULL; + + INIT_LIST_HEAD(&range->garbage_collector_link); + drm_gpusvm_init_pages(&range->pages, &gpusvm_to_vm(gpusvm)->xe->drm); + xe_vm_get(gpusvm_to_vm(gpusvm)); + + return &range->base; +} + +static void xe_svm_range_free(struct drm_gpusvm_range *range) +{ + drm_gpusvm_free_pages(range->gpusvm, &(to_xe_range(range)->pages), + drm_gpusvm_range_size(range) >> PAGE_SHIFT); + xe_vm_put(range_to_vm(range)); + kfree(to_xe_range(range)); +} + +static void +xe_svm_garbage_collector_add_range(struct xe_vm *vm, struct xe_svm_range *range, + const struct mmu_notifier_range *mmu_range) +{ + struct xe_device *xe = vm->xe; + + range_debug(range, "GARBAGE COLLECTOR ADD"); + + drm_gpusvm_range_set_unmapped(&range->base, &range->pages, 1, + mmu_range); + + spin_lock(&vm->svm.garbage_collector.lock); + if (list_empty(&range->garbage_collector_link)) + list_add_tail(&range->garbage_collector_link, + &vm->svm.garbage_collector.range_list); + spin_unlock(&vm->svm.garbage_collector.lock); + + queue_work(xe->usm.pf_wq, &vm->svm.garbage_collector.work); +} + +static void xe_svm_tlb_inval_count_stats_incr(struct xe_gt *gt) +{ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT, 1); +} + +static u8 +xe_svm_range_notifier_event_begin(struct xe_vm *vm, struct drm_gpusvm_range *r, + const struct mmu_notifier_range *mmu_range, + u64 *adj_start, u64 *adj_end) +{ + struct xe_svm_range *range = to_xe_range(r); + struct xe_device *xe = vm->xe; + struct xe_tile *tile; + u8 tile_mask = 0; + u8 id; + + xe_svm_assert_in_notifier(vm); + + range_debug(range, "NOTIFIER"); + + /* Skip if already unmapped or if no binding exist */ + if (range->base.flags.unmapped || !range->tile_present) + return 0; + + range_debug(range, "NOTIFIER - EXECUTE"); + + /* Adjust invalidation to range boundaries */ + *adj_start = min(xe_svm_range_start(range), mmu_range->start); + *adj_end = max(xe_svm_range_end(range), mmu_range->end); + + /* + * XXX: Ideally would zap PTEs in one shot in xe_svm_invalidate but the + * invalidation code can't correctly cope with sparse ranges or + * invalidations spanning multiple ranges. + */ + for_each_tile(tile, xe, id) + if (xe_pt_zap_ptes_range(tile, vm, range)) { + /* + * WRITE_ONCE pairs with READ_ONCE in + * xe_vm_has_valid_gpu_mapping() + */ + WRITE_ONCE(range->tile_invalidated, + range->tile_invalidated | BIT(id)); + + if (!(tile_mask & BIT(id))) { + xe_svm_tlb_inval_count_stats_incr(tile->primary_gt); + if (tile->media_gt) + xe_svm_tlb_inval_count_stats_incr(tile->media_gt); + tile_mask |= BIT(id); + } + } + + return tile_mask; +} + +static void +xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r, + const struct mmu_notifier_range *mmu_range) +{ + struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; + + xe_svm_assert_in_notifier(vm); + + drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &(to_xe_range(r)->pages), + drm_gpusvm_range_size(r) >> PAGE_SHIFT, &ctx); + if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP) + xe_svm_garbage_collector_add_range(vm, to_xe_range(r), + mmu_range); +} + +static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start) +{ + s64 us_delta = xe_gt_stats_ktime_us_delta(start); + + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta); +} + +static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, + struct drm_gpusvm_notifier *notifier, + const struct mmu_notifier_range *mmu_range) +{ + struct xe_vm *vm = gpusvm_to_vm(gpusvm); + struct xe_tlb_inval_batch batch; + struct xe_device *xe = vm->xe; + struct drm_gpusvm_range *r, *first; + struct xe_tile *tile; + ktime_t start = xe_gt_stats_ktime_get(); + u64 adj_start = mmu_range->start, adj_end = mmu_range->end; + u8 tile_mask = 0, id; + long err; + + xe_svm_assert_in_notifier(vm); + + vm_dbg(&gpusvm_to_vm(gpusvm)->xe->drm, + "INVALIDATE: asid=%u, gpusvm=%p, seqno=%lu, start=0x%016lx, end=0x%016lx, event=%d", + vm->usm.asid, gpusvm, notifier->notifier.invalidate_seq, + mmu_range->start, mmu_range->end, mmu_range->event); + + /* Adjust invalidation to notifier boundaries */ + adj_start = max(drm_gpusvm_notifier_start(notifier), adj_start); + adj_end = min(drm_gpusvm_notifier_end(notifier), adj_end); + + first = drm_gpusvm_range_find(notifier, adj_start, adj_end); + if (!first) + return; + + /* + * PTs may be getting destroyed so not safe to touch these but PT should + * be invalidated at this point in time. Regardless we still need to + * ensure any dma mappings are unmapped in the here. + */ + if (xe_vm_is_closed(vm)) + goto range_notifier_event_end; + + /* + * XXX: Less than ideal to always wait on VM's resv slots if an + * invalidation is not required. Could walk range list twice to figure + * out if an invalidations is need, but also not ideal. + */ + err = dma_resv_wait_timeout(xe_vm_resv(vm), + DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + XE_WARN_ON(err <= 0); + + r = first; + drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) + tile_mask |= xe_svm_range_notifier_event_begin(vm, r, mmu_range, + &adj_start, + &adj_end); + if (!tile_mask) + goto range_notifier_event_end; + + xe_device_wmb(xe); + + err = xe_tlb_inval_range_tilemask_submit(xe, vm->usm.asid, adj_start, adj_end, + tile_mask, &batch); + if (!WARN_ON_ONCE(err)) + xe_tlb_inval_batch_wait(&batch); + +range_notifier_event_end: + r = first; + drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) + xe_svm_range_notifier_event_end(vm, r, mmu_range); + for_each_tile(tile, xe, id) { + if (tile_mask & BIT(id)) { + xe_svm_tlb_inval_us_stats_incr(tile->primary_gt, start); + if (tile->media_gt) + xe_svm_tlb_inval_us_stats_incr(tile->media_gt, start); + } + } +} + +static int __xe_svm_garbage_collector(struct xe_vm *vm, + struct xe_svm_range *range) +{ + struct drm_gpusvm_ctx ctx = { .in_notifier = false, }; + struct dma_fence *fence; + + range_debug(range, "GARBAGE COLLECTOR"); + + xe_vm_lock(vm, false); + fence = xe_vm_range_unbind(vm, range); + xe_vm_unlock(vm); + if (IS_ERR(fence)) + return PTR_ERR(fence); + dma_fence_put(fence); + + drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &range->pages, + drm_gpusvm_range_size(&range->base) >> PAGE_SHIFT, + &ctx); + + drm_gpusvm_range_remove(&vm->svm.gpusvm, &range->base); + + return 0; +} + +static void xe_vma_set_default_attributes(struct xe_vma *vma) +{ + struct xe_vma_mem_attr default_attr = { + .preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, + .preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES, + .pat_index = vma->attr.default_pat_index, + .atomic_access = DRM_XE_ATOMIC_UNDEFINED, + .purgeable_state = XE_MADV_PURGEABLE_WILLNEED, + }; + + xe_vma_mem_attr_copy(&vma->attr, &default_attr); +} + +static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end) +{ + struct xe_vma *vma; + bool has_default_attr; + int err; + + vma = xe_vm_find_vma_by_addr(vm, start); + if (!vma) + return -EINVAL; + + if (!(vma->gpuva.flags & XE_VMA_MADV_AUTORESET)) { + drm_dbg(&vm->xe->drm, "Skipping madvise reset for vma.\n"); + return 0; + } + + vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx", + xe_vma_start(vma), xe_vma_end(vma)); + + has_default_attr = xe_vma_has_default_mem_attrs(vma); + + if (has_default_attr) { + start = xe_vma_start(vma); + end = xe_vma_end(vma); + } else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) { + xe_vma_set_default_attributes(vma); + } + + xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end); + + if (xe_vma_start(vma) == start && xe_vma_end(vma) == end && has_default_attr) + return 0; + + vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx", start, end); + + err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start); + if (err) { + drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err)); + xe_vm_kill(vm, true); + return err; + } + + /* + * On call from xe_svm_handle_pagefault original VMA might be changed + * signal this to lookup for VMA again. + */ + return -EAGAIN; +} + +static int xe_svm_garbage_collector(struct xe_vm *vm) +{ + struct xe_svm_range *range; + u64 range_start; + u64 range_end; + int err, ret = 0; + + lockdep_assert_held_write(&vm->lock); + + if (xe_vm_is_closed_or_banned(vm)) + return -ENOENT; + + for (;;) { + spin_lock(&vm->svm.garbage_collector.lock); + range = list_first_entry_or_null(&vm->svm.garbage_collector.range_list, + typeof(*range), + garbage_collector_link); + if (!range) + break; + + range_start = xe_svm_range_start(range); + range_end = xe_svm_range_end(range); + + list_del(&range->garbage_collector_link); + spin_unlock(&vm->svm.garbage_collector.lock); + + err = __xe_svm_garbage_collector(vm, range); + if (err) { + drm_warn(&vm->xe->drm, + "Garbage collection failed: %pe\n", + ERR_PTR(err)); + xe_vm_kill(vm, true); + return err; + } + + err = xe_svm_range_set_default_attr(vm, range_start, range_end); + if (err) { + if (err == -EAGAIN) + ret = -EAGAIN; + else + return err; + } + } + spin_unlock(&vm->svm.garbage_collector.lock); + + return ret; +} + +static void xe_svm_garbage_collector_work_func(struct work_struct *w) +{ + struct xe_vm *vm = container_of(w, struct xe_vm, + svm.garbage_collector.work); + + down_write(&vm->lock); + xe_svm_garbage_collector(vm); + up_write(&vm->lock); +} + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + +static struct xe_vram_region *xe_pagemap_to_vr(struct xe_pagemap *xpagemap) +{ + return xpagemap->vr; +} + +static struct xe_pagemap *xe_page_to_pagemap(struct page *page) +{ + return container_of(page_pgmap(page), struct xe_pagemap, pagemap); +} + +static struct xe_vram_region *xe_page_to_vr(struct page *page) +{ + return xe_pagemap_to_vr(xe_page_to_pagemap(page)); +} + +static u64 xe_page_to_dpa(struct page *page) +{ + struct xe_pagemap *xpagemap = xe_page_to_pagemap(page); + struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); + u64 hpa_base = xpagemap->hpa_base; + u64 pfn = page_to_pfn(page); + u64 offset; + u64 dpa; + + xe_assert(vr->xe, is_device_private_page(page)); + xe_assert(vr->xe, (pfn << PAGE_SHIFT) >= hpa_base); + + offset = (pfn << PAGE_SHIFT) - hpa_base; + dpa = vr->dpa_base + offset; + + return dpa; +} + +static u64 xe_page_to_pcie(struct page *page) +{ + struct xe_pagemap *xpagemap = xe_page_to_pagemap(page); + struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); + + return xe_page_to_dpa(page) - vr->dpa_base + vr->io_start; +} + +enum xe_svm_copy_dir { + XE_SVM_COPY_TO_VRAM, + XE_SVM_COPY_TO_SRAM, +}; + +static void xe_svm_copy_kb_stats_incr(struct xe_gt *gt, + const enum xe_svm_copy_dir dir, + int kb) +{ + if (dir == XE_SVM_COPY_TO_VRAM) { + switch (kb) { + case 4: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB, kb); + break; + case 64: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB, kb); + break; + case 2048: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB, kb); + break; + } + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_DEVICE_COPY_KB, kb); + } else { + switch (kb) { + case 4: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB, kb); + break; + case 64: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB, kb); + break; + case 2048: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB, kb); + break; + } + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_CPU_COPY_KB, kb); + } +} + +static void xe_svm_copy_us_stats_incr(struct xe_gt *gt, + const enum xe_svm_copy_dir dir, + unsigned long npages, + ktime_t start) +{ + s64 us_delta = xe_gt_stats_ktime_us_delta(start); + + if (dir == XE_SVM_COPY_TO_VRAM) { + switch (npages) { + case 1: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US, + us_delta); + break; + case 16: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US, + us_delta); + break; + case 512: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US, + us_delta); + break; + } + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_DEVICE_COPY_US, + us_delta); + } else { + switch (npages) { + case 1: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_CPU_COPY_US, + us_delta); + break; + case 16: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_CPU_COPY_US, + us_delta); + break; + case 512: + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_CPU_COPY_US, + us_delta); + break; + } + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_CPU_COPY_US, + us_delta); + } +} + +static int xe_svm_copy(struct page **pages, + struct drm_pagemap_addr *pagemap_addr, + unsigned long npages, const enum xe_svm_copy_dir dir, + struct dma_fence *pre_migrate_fence) +{ + struct xe_vram_region *vr = NULL; + struct xe_gt *gt = NULL; + struct xe_device *xe; + struct dma_fence *fence = NULL; + unsigned long i; +#define XE_VRAM_ADDR_INVALID ~0x0ull + u64 vram_addr = XE_VRAM_ADDR_INVALID; + int err = 0, pos = 0; + bool sram = dir == XE_SVM_COPY_TO_SRAM; + ktime_t start = xe_gt_stats_ktime_get(); + + /* + * This flow is complex: it locates physically contiguous device pages, + * derives the starting physical address, and performs a single GPU copy + * to for every 8M chunk in a DMA address array. Both device pages and + * DMA addresses may be sparsely populated. If either is NULL, a copy is + * triggered based on the current search state. The last GPU copy is + * waited on to ensure all copies are complete. + */ + + for (i = 0; i < npages; ++i) { + struct page *spage = pages[i]; + struct dma_fence *__fence; + u64 __vram_addr; + bool match = false, chunk, last; + +#define XE_MIGRATE_CHUNK_SIZE SZ_8M + chunk = (i - pos) == (XE_MIGRATE_CHUNK_SIZE / PAGE_SIZE); + last = (i + 1) == npages; + + /* No CPU page and no device pages queue'd to copy */ + if (!pagemap_addr[i].addr && vram_addr == XE_VRAM_ADDR_INVALID) + continue; + + if (!vr && spage) { + vr = xe_page_to_vr(spage); + gt = xe_migrate_exec_queue(vr->migrate)->gt; + xe = vr->xe; + } + XE_WARN_ON(spage && xe_page_to_vr(spage) != vr); + + /* + * CPU page and device page valid, capture physical address on + * first device page, check if physical contiguous on subsequent + * device pages. + */ + if (pagemap_addr[i].addr && spage) { + __vram_addr = xe_page_to_dpa(spage); + if (vram_addr == XE_VRAM_ADDR_INVALID) { + vram_addr = __vram_addr; + pos = i; + } + + match = vram_addr + PAGE_SIZE * (i - pos) == __vram_addr; + /* Expected with contiguous memory */ + xe_assert(vr->xe, match); + + if (pagemap_addr[i].order) { + i += NR_PAGES(pagemap_addr[i].order) - 1; + chunk = (i - pos) == (XE_MIGRATE_CHUNK_SIZE / PAGE_SIZE); + last = (i + 1) == npages; + } + } + + /* + * Mismatched physical address, 8M copy chunk, or last page - + * trigger a copy. + */ + if (!match || chunk || last) { + /* + * Extra page for first copy if last page and matching + * physical address. + */ + int incr = (match && last) ? 1 : 0; + + if (vram_addr != XE_VRAM_ADDR_INVALID) { + xe_svm_copy_kb_stats_incr(gt, dir, + (i - pos + incr) * + (PAGE_SIZE / SZ_1K)); + if (sram) { + vm_dbg(&xe->drm, + "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", + vram_addr, + (u64)pagemap_addr[pos].addr, i - pos + incr); + __fence = xe_migrate_from_vram(vr->migrate, + i - pos + incr, + vram_addr, + &pagemap_addr[pos], + pre_migrate_fence); + } else { + vm_dbg(&xe->drm, + "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", + (u64)pagemap_addr[pos].addr, vram_addr, + i - pos + incr); + __fence = xe_migrate_to_vram(vr->migrate, + i - pos + incr, + &pagemap_addr[pos], + vram_addr, + pre_migrate_fence); + } + if (IS_ERR(__fence)) { + err = PTR_ERR(__fence); + goto err_out; + } + pre_migrate_fence = NULL; + dma_fence_put(fence); + fence = __fence; + } + + /* Setup physical address of next device page */ + if (pagemap_addr[i].addr && spage) { + vram_addr = __vram_addr; + pos = i; + } else { + vram_addr = XE_VRAM_ADDR_INVALID; + } + + /* Extra mismatched device page, copy it */ + if (!match && last && vram_addr != XE_VRAM_ADDR_INVALID) { + xe_svm_copy_kb_stats_incr(gt, dir, + (PAGE_SIZE / SZ_1K)); + if (sram) { + vm_dbg(&xe->drm, + "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", + vram_addr, (u64)pagemap_addr[pos].addr, 1); + __fence = xe_migrate_from_vram(vr->migrate, 1, + vram_addr, + &pagemap_addr[pos], + pre_migrate_fence); + } else { + vm_dbg(&xe->drm, + "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", + (u64)pagemap_addr[pos].addr, vram_addr, 1); + __fence = xe_migrate_to_vram(vr->migrate, 1, + &pagemap_addr[pos], + vram_addr, + pre_migrate_fence); + } + if (IS_ERR(__fence)) { + err = PTR_ERR(__fence); + goto err_out; + } + pre_migrate_fence = NULL; + dma_fence_put(fence); + fence = __fence; + } + } + } + +err_out: + /* Wait for all copies to complete */ + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + if (pre_migrate_fence) + dma_fence_wait(pre_migrate_fence, false); + + /* + * XXX: We can't derive the GT here (or anywhere in this functions, but + * compute always uses the primary GT so accumulate stats on the likely + * GT of the fault. + */ + if (gt) + xe_svm_copy_us_stats_incr(gt, dir, npages, start); + + return err; +#undef XE_MIGRATE_CHUNK_SIZE +#undef XE_VRAM_ADDR_INVALID +} + +static int xe_svm_copy_to_devmem(struct page **pages, + struct drm_pagemap_addr *pagemap_addr, + unsigned long npages, + struct dma_fence *pre_migrate_fence) +{ + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM, + pre_migrate_fence); +} + +static int xe_svm_copy_to_ram(struct page **pages, + struct drm_pagemap_addr *pagemap_addr, + unsigned long npages, + struct dma_fence *pre_migrate_fence) +{ + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM, + pre_migrate_fence); +} + +static struct xe_bo *to_xe_bo(struct drm_pagemap_devmem *devmem_allocation) +{ + return container_of(devmem_allocation, struct xe_bo, devmem_allocation); +} + +static void xe_svm_devmem_release(struct drm_pagemap_devmem *devmem_allocation) +{ + struct xe_bo *bo = to_xe_bo(devmem_allocation); + struct xe_device *xe = xe_bo_device(bo); + + dma_fence_put(devmem_allocation->pre_migrate_fence); + xe_bo_put_async(bo); + xe_pm_runtime_put(xe); +} + +static u64 block_offset_to_pfn(struct drm_pagemap *dpagemap, u64 offset) +{ + struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); + + return PHYS_PFN(offset + xpagemap->hpa_base); +} + +static struct gpu_buddy *vram_to_buddy(struct xe_vram_region *vram) +{ + return &vram->ttm.mm; +} + +static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocation, + unsigned long npages, unsigned long *pfn) +{ + struct xe_bo *bo = to_xe_bo(devmem_allocation); + struct ttm_resource *res = bo->ttm.resource; + struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks; + struct xe_vram_region *vr = xe_map_resource_to_region(res); + struct gpu_buddy *buddy = vram_to_buddy(vr); + struct gpu_buddy_block *block; + int j = 0; + + list_for_each_entry(block, blocks, link) { + u64 block_pfn = block_offset_to_pfn(devmem_allocation->dpagemap, + gpu_buddy_block_offset(block)); + int i; + + for (i = 0; i < gpu_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i) + pfn[j++] = block_pfn + i; + } + + return 0; +} + +static const struct drm_pagemap_devmem_ops dpagemap_devmem_ops = { + .devmem_release = xe_svm_devmem_release, + .populate_devmem_pfn = xe_svm_populate_devmem_pfn, + .copy_to_devmem = xe_svm_copy_to_devmem, + .copy_to_ram = xe_svm_copy_to_ram, +}; + +#else +static int xe_svm_get_pagemaps(struct xe_vm *vm) +{ + return 0; +} +#endif + +static const struct drm_gpusvm_ops gpusvm_ops = { + .range_alloc = xe_svm_range_alloc, + .range_free = xe_svm_range_free, + .invalidate = xe_svm_invalidate, +}; + +static const unsigned long fault_chunk_sizes[] = { + SZ_2M, + SZ_64K, + SZ_4K, +}; + +static void xe_pagemap_put(struct xe_pagemap *xpagemap) +{ + drm_pagemap_put(&xpagemap->dpagemap); +} + +static void xe_svm_put_pagemaps(struct xe_vm *vm) +{ + struct xe_device *xe = vm->xe; + struct xe_tile *tile; + int id; + + for_each_tile(tile, xe, id) { + struct xe_pagemap *xpagemap = vm->svm.pagemaps[id]; + + if (xpagemap) + xe_pagemap_put(xpagemap); + vm->svm.pagemaps[id] = NULL; + } +} + +static struct device *xe_peer_to_dev(struct drm_pagemap_peer *peer) +{ + if (peer->private == XE_PEER_PAGEMAP) + return container_of(peer, struct xe_pagemap, peer)->dpagemap.drm->dev; + + return container_of(peer, struct xe_vm, svm.peer)->xe->drm.dev; +} + +static bool xe_has_interconnect(struct drm_pagemap_peer *peer1, + struct drm_pagemap_peer *peer2) +{ + struct device *dev1 = xe_peer_to_dev(peer1); + struct device *dev2 = xe_peer_to_dev(peer2); + + if (dev1 == dev2) + return true; + + return pci_p2pdma_distance(to_pci_dev(dev1), dev2, true) >= 0; +} + +static DRM_PAGEMAP_OWNER_LIST_DEFINE(xe_owner_list); + +/** + * xe_svm_init() - SVM initialize + * @vm: The VM. + * + * Initialize SVM state which is embedded within the VM. + * + * Return: 0 on success, negative error code on error. + */ +int xe_svm_init(struct xe_vm *vm) +{ + int err; + + if (vm->flags & XE_VM_FLAG_FAULT_MODE) { + spin_lock_init(&vm->svm.garbage_collector.lock); + INIT_LIST_HEAD(&vm->svm.garbage_collector.range_list); + INIT_WORK(&vm->svm.garbage_collector.work, + xe_svm_garbage_collector_work_func); + + vm->svm.peer.private = XE_PEER_VM; + err = drm_pagemap_acquire_owner(&vm->svm.peer, &xe_owner_list, + xe_has_interconnect); + if (err) + return err; + + err = xe_svm_get_pagemaps(vm); + if (err) { + drm_pagemap_release_owner(&vm->svm.peer); + return err; + } + + err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM", + current->mm, 0, vm->size, + xe_modparam.svm_notifier_size * SZ_1M, + &gpusvm_ops, fault_chunk_sizes, + ARRAY_SIZE(fault_chunk_sizes)); + drm_gpusvm_driver_set_lock(&vm->svm.gpusvm, &vm->lock); + + if (err) { + xe_svm_put_pagemaps(vm); + drm_pagemap_release_owner(&vm->svm.peer); + return err; + } + } else { + err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", + NULL, 0, 0, 0, NULL, + NULL, 0); + } + + return err; +} + +/** + * xe_svm_close() - SVM close + * @vm: The VM. + * + * Close SVM state (i.e., stop and flush all SVM actions). + */ +void xe_svm_close(struct xe_vm *vm) +{ + xe_assert(vm->xe, xe_vm_is_closed(vm)); + disable_work_sync(&vm->svm.garbage_collector.work); + xe_svm_put_pagemaps(vm); + drm_pagemap_release_owner(&vm->svm.peer); +} + +/** + * xe_svm_fini() - SVM finalize + * @vm: The VM. + * + * Finalize SVM state which is embedded within the VM. + */ +void xe_svm_fini(struct xe_vm *vm) +{ + struct drm_gpusvm_notifier *notifier, *next; + struct drm_gpusvm_ctx ctx = { .in_notifier = false, }; + + xe_assert(vm->xe, xe_vm_is_closed(vm)); + + drm_gpusvm_for_each_notifier_safe(notifier, next, &vm->svm.gpusvm, 0, LONG_MAX) { + struct drm_gpusvm_range *range, *__next; + + drm_gpusvm_for_each_range_safe(range, __next, notifier, 0, LONG_MAX) + drm_gpusvm_unmap_pages(&vm->svm.gpusvm, + &(to_xe_range(range)->pages), + drm_gpusvm_range_size(range) >> PAGE_SHIFT, + &ctx); + } + + drm_gpusvm_fini(&vm->svm.gpusvm); +} + +static bool xe_svm_range_has_pagemap_locked(const struct xe_svm_range *range, + const struct drm_pagemap *dpagemap) +{ + return range->pages.dpagemap == dpagemap; +} + +static bool xe_svm_range_has_pagemap(struct xe_svm_range *range, + const struct drm_pagemap *dpagemap) +{ + struct xe_vm *vm = range_to_vm(&range->base); + bool ret; + + xe_svm_notifier_lock(vm); + ret = xe_svm_range_has_pagemap_locked(range, dpagemap); + xe_svm_notifier_unlock(vm); + + return ret; +} + +static bool xe_svm_range_is_valid(struct xe_svm_range *range, + struct xe_tile *tile, + bool devmem_only, + const struct drm_pagemap *dpagemap) + +{ + return (xe_vm_has_valid_gpu_mapping(tile, range->tile_present, + range->tile_invalidated) && + (!devmem_only || xe_svm_range_has_pagemap(range, dpagemap))); +} + +/** xe_svm_range_migrate_to_smem() - Move range pages from VRAM to SMEM + * @vm: xe_vm pointer + * @range: Pointer to the SVM range structure + * + * The xe_svm_range_migrate_to_smem() checks range has pages in VRAM + * and migrates them to SMEM + */ +void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range) +{ + if (xe_svm_range_in_vram(range)) + drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base); +} + +/** + * xe_svm_range_validate() - Check if the SVM range is valid + * @vm: xe_vm pointer + * @range: Pointer to the SVM range structure + * @tile_mask: Mask representing the tiles to be checked + * @dpagemap: if !%NULL, the range is expected to be present + * in device memory identified by this parameter. + * + * The xe_svm_range_validate() function checks if a range is + * valid and located in the desired memory region. + * + * Return: true if the range is valid, false otherwise + */ +bool xe_svm_range_validate(struct xe_vm *vm, + struct xe_svm_range *range, + u8 tile_mask, const struct drm_pagemap *dpagemap) +{ + bool ret; + + xe_svm_notifier_lock(vm); + + ret = (range->tile_present & ~range->tile_invalidated & tile_mask) == tile_mask; + if (dpagemap) + ret = ret && xe_svm_range_has_pagemap_locked(range, dpagemap); + else + ret = ret && !range->pages.dpagemap; + + xe_svm_notifier_unlock(vm); + + return ret; +} + +/** + * xe_svm_find_vma_start - Find start of CPU VMA + * @vm: xe_vm pointer + * @start: start address + * @end: end address + * @vma: Pointer to struct xe_vma + * + * + * This function searches for a cpu vma, within the specified + * range [start, end] in the given VM. It adjusts the range based on the + * xe_vma start and end addresses. If no cpu VMA is found, it returns ULONG_MAX. + * + * Return: The starting address of the VMA within the range, + * or ULONG_MAX if no VMA is found + */ +u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 start, u64 end, struct xe_vma *vma) +{ + return drm_gpusvm_find_vma_start(&vm->svm.gpusvm, + max(start, xe_vma_start(vma)), + min(end, xe_vma_end(vma))); +} + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) +static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, + unsigned long start, unsigned long end, + struct mm_struct *mm, + unsigned long timeslice_ms) +{ + struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); + struct drm_pagemap_migrate_details mdetails = { + .timeslice_ms = timeslice_ms, + }; + struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); + struct dma_fence *pre_migrate_fence = NULL; + struct xe_device *xe = vr->xe; + struct device *dev = xe->drm.dev; + struct xe_validation_ctx vctx; + struct drm_exec exec; + struct xe_bo *bo; + int err = 0, idx; + + if (!drm_dev_enter(&xe->drm, &idx)) + return -ENODEV; + + xe_pm_runtime_get(xe); + + xe_validation_guard(&vctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { + bo = xe_bo_create_locked(xe, NULL, NULL, end - start, + ttm_bo_type_device, + (IS_DGFX(xe) ? XE_BO_FLAG_VRAM(vr) : XE_BO_FLAG_SYSTEM) | + XE_BO_FLAG_CPU_ADDR_MIRROR, &exec); + drm_exec_retry_on_contention(&exec); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + xe_validation_retry_on_oom(&vctx, &err); + break; + } + + /* Ensure that any clearing or async eviction will complete before migration. */ + if (!dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL)) { + err = dma_resv_get_singleton(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + &pre_migrate_fence); + if (err) + dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + else if (pre_migrate_fence) + dma_fence_enable_signaling(pre_migrate_fence); + } + + drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, + &dpagemap_devmem_ops, dpagemap, end - start, + pre_migrate_fence); + + xe_bo_get(bo); + + /* Ensure the device has a pm ref while there are device pages active. */ + xe_pm_runtime_get_noresume(xe); + /* Consumes the devmem allocation ref. */ + err = drm_pagemap_migrate_to_devmem(&bo->devmem_allocation, mm, + start, end, &mdetails); + xe_bo_unlock(bo); + xe_bo_put(bo); + } + xe_pm_runtime_put(xe); + drm_dev_exit(idx); + + return err; +} +#endif + +static bool supports_4K_migration(struct xe_device *xe) +{ + if (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + return false; + + return true; +} + +/** + * xe_svm_range_needs_migrate_to_vram() - SVM range needs migrate to VRAM or not + * @range: SVM range for which migration needs to be decided + * @vma: vma which has range + * @dpagemap: The preferred struct drm_pagemap to migrate to. + * + * Return: True for range needing migration and migration is supported else false + */ +bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, + const struct drm_pagemap *dpagemap) +{ + struct xe_vm *vm = range_to_vm(&range->base); + u64 range_size = xe_svm_range_size(range); + struct drm_gpusvm_range_flags flags = { + /* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */ + .__flags = READ_ONCE(range->base.flags.__flags), + }; + + if (!flags.migrate_devmem || !dpagemap) + return false; + + xe_assert(vm->xe, IS_DGFX(vm->xe)); + + if (xe_svm_range_has_pagemap(range, dpagemap)) { + drm_dbg(&vm->xe->drm, "Range is already in VRAM\n"); + return false; + } + + if (range_size < SZ_64K && !supports_4K_migration(vm->xe)) { + drm_dbg(&vm->xe->drm, "Platform doesn't support SZ_4K range migration\n"); + return false; + } + + return true; +} + +#define DECL_SVM_RANGE_COUNT_STATS(elem, stat) \ +static void xe_svm_range_##elem##_count_stats_incr(struct xe_gt *gt, \ + struct xe_svm_range *range) \ +{ \ + switch (xe_svm_range_size(range)) { \ + case SZ_4K: \ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_##stat##_COUNT, 1); \ + break; \ + case SZ_64K: \ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_##stat##_COUNT, 1); \ + break; \ + case SZ_2M: \ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_##stat##_COUNT, 1); \ + break; \ + } \ +} \ + +DECL_SVM_RANGE_COUNT_STATS(fault, PAGEFAULT) +DECL_SVM_RANGE_COUNT_STATS(valid_fault, VALID_PAGEFAULT) +DECL_SVM_RANGE_COUNT_STATS(migrate, MIGRATE) + +#define DECL_SVM_RANGE_US_STATS(elem, stat) \ +static void xe_svm_range_##elem##_us_stats_incr(struct xe_gt *gt, \ + struct xe_svm_range *range, \ + ktime_t start) \ +{ \ + s64 us_delta = xe_gt_stats_ktime_us_delta(start); \ +\ + switch (xe_svm_range_size(range)) { \ + case SZ_4K: \ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_##stat##_US, \ + us_delta); \ + break; \ + case SZ_64K: \ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_##stat##_US, \ + us_delta); \ + break; \ + case SZ_2M: \ + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_##stat##_US, \ + us_delta); \ + break; \ + } \ +} \ + +DECL_SVM_RANGE_US_STATS(migrate, MIGRATE) +DECL_SVM_RANGE_US_STATS(get_pages, GET_PAGES) +DECL_SVM_RANGE_US_STATS(bind, BIND) +DECL_SVM_RANGE_US_STATS(fault, PAGEFAULT) + +static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, + struct xe_gt *gt, u64 fault_addr, + bool need_vram) +{ + int devmem_possible = IS_DGFX(vm->xe) && + IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); + struct drm_gpusvm_ctx ctx = { + .read_only = xe_vma_read_only(vma), + .devmem_possible = devmem_possible, + .check_pages_threshold = devmem_possible ? SZ_64K : 0, + .devmem_only = need_vram && devmem_possible, + .timeslice_ms = need_vram && devmem_possible ? + vm->xe->atomic_svm_timeslice_ms : 0, + }; + struct xe_validation_ctx vctx; + struct drm_exec exec; + struct xe_svm_range *range; + struct drm_gpusvm_range_flags range_flags; + struct dma_fence *fence; + struct drm_pagemap *dpagemap; + struct xe_tile *tile = gt_to_tile(gt); + int migrate_try_count = ctx.devmem_only ? 3 : 1; + ktime_t start = xe_gt_stats_ktime_get(), bind_start, get_pages_start; + int err; + + lockdep_assert_held_write(&vm->lock); + xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); + + xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT, 1); + +retry: + /* Always process UNMAPs first so view SVM ranges is current */ + err = xe_svm_garbage_collector(vm); + if (err) + return err; + + dpagemap = ctx.devmem_only ? xe_tile_local_pagemap(tile) : + xe_vma_resolve_pagemap(vma, tile); + ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); + range = xe_svm_range_find_or_insert(vm, fault_addr, vma, &ctx); + + if (IS_ERR(range)) + return PTR_ERR(range); + + xe_svm_range_fault_count_stats_incr(gt, range); + + /* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */ + range_flags.__flags = READ_ONCE(range->base.flags.__flags); + if (ctx.devmem_only && !range_flags.migrate_devmem) + return -EACCES; + + if (xe_svm_range_is_valid(range, tile, ctx.devmem_only, dpagemap)) { + xe_svm_range_valid_fault_count_stats_incr(gt, range); + range_debug(range, "PAGE FAULT - VALID"); + goto out; + } + + range_debug(range, "PAGE FAULT"); + + if (--migrate_try_count >= 0 && + xe_svm_range_needs_migrate_to_vram(range, vma, dpagemap)) { + ktime_t migrate_start = xe_gt_stats_ktime_get(); + + xe_svm_range_migrate_count_stats_incr(gt, range); + err = xe_svm_alloc_vram(range, &ctx, dpagemap); + xe_svm_range_migrate_us_stats_incr(gt, range, migrate_start); + ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ + if (err) { + if (migrate_try_count || !ctx.devmem_only) { + drm_dbg(&vm->xe->drm, + "VRAM allocation failed, falling back to retrying fault, asid=%u, errno=%pe\n", + vm->usm.asid, ERR_PTR(err)); + + /* + * In the devmem-only case, mixed mappings may + * be found. The get_pages function will fix + * these up to a single location, allowing the + * page fault handler to make forward progress. + */ + if (ctx.devmem_only) + goto get_pages; + else + goto retry; + } else { + drm_err(&vm->xe->drm, + "VRAM allocation failed, retry count exceeded, asid=%u, errno=%pe\n", + vm->usm.asid, ERR_PTR(err)); + return err; + } + } + } + +get_pages: + get_pages_start = xe_gt_stats_ktime_get(); + + range_debug(range, "GET PAGES"); + err = xe_svm_range_get_pages(vm, range, &ctx); + /* Corner where CPU mappings have changed */ + if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { + ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ + if (migrate_try_count > 0 || !ctx.devmem_only) { + drm_dbg(&vm->xe->drm, + "Get pages failed, falling back to retrying, asid=%u, gpusvm=%p, errno=%pe\n", + vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); + range_debug(range, "PAGE FAULT - RETRY PAGES"); + goto retry; + } else { + drm_err(&vm->xe->drm, + "Get pages failed, retry count exceeded, asid=%u, gpusvm=%p, errno=%pe\n", + vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); + } + } + if (err) { + range_debug(range, "PAGE FAULT - FAIL PAGE COLLECT"); + goto out; + } else if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) { + drm_dbg(&vm->xe->drm, "After page collect data location is %sin \"%s\".\n", + xe_svm_range_has_pagemap(range, dpagemap) ? "" : "NOT ", + dpagemap ? dpagemap->drm->unique : "System."); + } + + xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start); + range_debug(range, "PAGE FAULT - BIND"); + + bind_start = xe_gt_stats_ktime_get(); + xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) { + err = xe_vm_drm_exec_lock(vm, &exec); + drm_exec_retry_on_contention(&exec); + + xe_vm_set_validation_exec(vm, &exec); + fence = xe_vm_range_rebind(vm, vma, range, BIT(tile->id)); + xe_vm_set_validation_exec(vm, NULL); + if (IS_ERR(fence)) { + drm_exec_retry_on_contention(&exec); + err = PTR_ERR(fence); + xe_validation_retry_on_oom(&vctx, &err); + xe_svm_range_bind_us_stats_incr(gt, range, bind_start); + break; + } + } + if (err) + goto err_out; + + dma_fence_wait(fence, false); + dma_fence_put(fence); + xe_svm_range_bind_us_stats_incr(gt, range, bind_start); + +out: + xe_svm_range_fault_us_stats_incr(gt, range, start); + return 0; + +err_out: + if (err == -EAGAIN) { + ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ + range_debug(range, "PAGE FAULT - RETRY BIND"); + goto retry; + } + + return err; +} + +/** + * xe_svm_handle_pagefault() - SVM handle page fault + * @vm: The VM. + * @vma: The CPU address mirror VMA. + * @gt: The gt upon the fault occurred. + * @fault_addr: The GPU fault address. + * @atomic: The fault atomic access bit. + * + * Create GPU bindings for a SVM page fault. Optionally migrate to device + * memory. + * + * Return: 0 on success, negative error code on error. + */ +int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, + struct xe_gt *gt, u64 fault_addr, + bool atomic) +{ + int need_vram, ret; +retry: + need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); + if (need_vram < 0) + return need_vram; + + ret = __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, + need_vram ? true : false); + if (ret == -EAGAIN) { + /* + * Retry once on -EAGAIN to re-lookup the VMA, as the original VMA + * may have been split by xe_svm_range_set_default_attr. + */ + vma = xe_vm_find_vma_by_addr(vm, fault_addr); + if (!vma) + return -EINVAL; + + goto retry; + } + return ret; +} + +/** + * xe_svm_has_mapping() - SVM has mappings + * @vm: The VM. + * @start: Start address. + * @end: End address. + * + * Check if an address range has SVM mappings. + * + * Return: True if address range has a SVM mapping, False otherwise + */ +bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) +{ + return drm_gpusvm_has_mapping(&vm->svm.gpusvm, start, end); +} + +/** + * xe_svm_unmap_address_range - UNMAP SVM mappings and ranges + * @vm: The VM + * @start: start addr + * @end: end addr + * + * This function UNMAPS svm ranges if start or end address are inside them. + */ +void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpusvm_notifier *notifier, *next; + + lockdep_assert_held_write(&vm->lock); + + drm_gpusvm_for_each_notifier_safe(notifier, next, &vm->svm.gpusvm, start, end) { + struct drm_gpusvm_range *range, *__next; + + drm_gpusvm_for_each_range_safe(range, __next, notifier, start, end) { + if (start > drm_gpusvm_range_start(range) || + end < drm_gpusvm_range_end(range)) { + if (IS_DGFX(vm->xe) && xe_svm_range_in_vram(to_xe_range(range))) + drm_gpusvm_range_evict(&vm->svm.gpusvm, range); + drm_gpusvm_range_get(range); + __xe_svm_garbage_collector(vm, to_xe_range(range)); + if (!list_empty(&to_xe_range(range)->garbage_collector_link)) { + spin_lock(&vm->svm.garbage_collector.lock); + list_del(&to_xe_range(range)->garbage_collector_link); + spin_unlock(&vm->svm.garbage_collector.lock); + } + drm_gpusvm_range_put(range); + } + } + } +} + +/** + * xe_svm_bo_evict() - SVM evict BO to system memory + * @bo: BO to evict + * + * SVM evict BO to system memory. GPU SVM layer ensures all device pages + * are evicted before returning. + * + * Return: 0 on success standard error code otherwise + */ +int xe_svm_bo_evict(struct xe_bo *bo) +{ + return drm_pagemap_evict_to_ram(&bo->devmem_allocation); +} + +/** + * xe_svm_range_find_or_insert- Find or insert GPU SVM range + * @vm: xe_vm pointer + * @addr: address for which range needs to be found/inserted + * @vma: Pointer to struct xe_vma which mirrors CPU + * @ctx: GPU SVM context + * + * This function finds or inserts a newly allocated a SVM range based on the + * address. + * + * Return: Pointer to the SVM range on success, ERR_PTR() on failure. + */ +struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, + struct xe_vma *vma, struct drm_gpusvm_ctx *ctx) +{ + struct drm_gpusvm_range *r; + + r = drm_gpusvm_range_find_or_insert(&vm->svm.gpusvm, max(addr, xe_vma_start(vma)), + xe_vma_start(vma), xe_vma_end(vma), ctx); + if (IS_ERR(r)) + return ERR_CAST(r); + + return to_xe_range(r); +} + +/** + * xe_svm_range_get_pages() - Get pages for a SVM range + * @vm: Pointer to the struct xe_vm + * @range: Pointer to the xe SVM range structure + * @ctx: GPU SVM context + * + * This function gets pages for a SVM range and ensures they are mapped for + * DMA access. In case of failure with -EOPNOTSUPP, it evicts the range. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, + struct drm_gpusvm_ctx *ctx) +{ + int err = 0; + + err = drm_gpusvm_get_pages(&vm->svm.gpusvm, &range->pages, + vm->svm.gpusvm.mm, + &range->base.notifier->notifier, + drm_gpusvm_range_start(&range->base), + drm_gpusvm_range_end(&range->base), ctx); + if (err == -EOPNOTSUPP) { + range_debug(range, "PAGE FAULT - EVICT PAGES"); + drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base); + } + + return err; +} + +/** + * xe_svm_ranges_zap_ptes_in_range - clear ptes of svm ranges in input range + * @vm: Pointer to the xe_vm structure + * @start: Start of the input range + * @end: End of the input range + * + * This function removes the page table entries (PTEs) associated + * with the svm ranges within the given input start and end + * + * Return: tile_mask for which gt's need to be tlb invalidated. + */ +u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpusvm_notifier *notifier; + struct xe_svm_range *range; + u64 adj_start, adj_end; + struct xe_tile *tile; + u8 tile_mask = 0; + u8 id; + + lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && + lockdep_is_held_type(&vm->lock, 0)); + + drm_gpusvm_for_each_notifier(notifier, &vm->svm.gpusvm, start, end) { + struct drm_gpusvm_range *r = NULL; + + adj_start = max(start, drm_gpusvm_notifier_start(notifier)); + adj_end = min(end, drm_gpusvm_notifier_end(notifier)); + drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) { + range = to_xe_range(r); + for_each_tile(tile, vm->xe, id) { + if (xe_pt_zap_ptes_range(tile, vm, range)) { + tile_mask |= BIT(id); + /* + * WRITE_ONCE pairs with READ_ONCE in + * xe_vm_has_valid_gpu_mapping(). + * Must not fail after setting + * tile_invalidated and before + * TLB invalidation. + */ + WRITE_ONCE(range->tile_invalidated, + range->tile_invalidated | BIT(id)); + } + } + } + } + + return tile_mask; +} + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + +/** + * xe_vma_resolve_pagemap - Resolve the appropriate DRM pagemap for a VMA + * @vma: Pointer to the xe_vma structure containing memory attributes + * @tile: Pointer to the xe_tile structure used as fallback for VRAM mapping + * + * This function determines the correct DRM pagemap to use for a given VMA. + * It first checks if a valid devmem_fd is provided in the VMA's preferred + * location. If the devmem_fd is negative, it returns NULL, indicating no + * pagemap is available and smem to be used as preferred location. + * If the devmem_fd is equal to the default faulting + * GT identifier, it returns the VRAM pagemap associated with the tile. + * + * Future support for multi-device configurations may use drm_pagemap_from_fd() + * to resolve pagemaps from arbitrary file descriptors. + * + * Return: A pointer to the resolved drm_pagemap, or NULL if none is applicable. + */ +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) +{ + struct drm_pagemap *dpagemap = vma->attr.preferred_loc.dpagemap; + s32 fd; + + if (dpagemap) + return dpagemap; + + fd = (s32)vma->attr.preferred_loc.devmem_fd; + + if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM) + return NULL; + + if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE) + return IS_DGFX(tile_to_xe(tile)) ? xe_tile_local_pagemap(tile) : NULL; + + return NULL; +} + +/** + * xe_svm_alloc_vram()- Allocate device memory pages for range, + * migrating existing data. + * @range: SVM range + * @ctx: DRM GPU SVM context + * @dpagemap: The struct drm_pagemap representing the memory to allocate. + * + * Return: 0 on success, error code on failure. + */ +int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, + struct drm_pagemap *dpagemap) +{ + static DECLARE_RWSEM(driver_migrate_lock); + struct xe_vm *vm = range_to_vm(&range->base); + enum drm_gpusvm_scan_result migration_state; + struct xe_device *xe = vm->xe; + int err, retries = 1; + bool write_locked = false; + struct drm_gpusvm_range_flags flags = { + /* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */ + .__flags = READ_ONCE(range->base.flags.__flags), + }; + + xe_assert(range_to_vm(&range->base)->xe, flags.migrate_devmem); + range_debug(range, "ALLOCATE VRAM"); + + migration_state = drm_gpusvm_scan_mm(&range->base, + xe_svm_private_page_owner(vm, false), + dpagemap->pagemap); + + if (migration_state == DRM_GPUSVM_SCAN_EQUAL) { + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) + drm_dbg(dpagemap->drm, "Already migrated!\n"); + return 0; + } + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) + drm_dbg(&xe->drm, "Request migration to device memory on \"%s\".\n", + dpagemap->drm->unique); + + err = down_read_interruptible(&driver_migrate_lock); + if (err) + return err; + do { + err = drm_pagemap_populate_mm(dpagemap, xe_svm_range_start(range), + xe_svm_range_end(range), + range->base.gpusvm->mm, + ctx->timeslice_ms); + + if (err == -EBUSY && retries) { + if (!write_locked) { + int lock_err; + + up_read(&driver_migrate_lock); + lock_err = down_write_killable(&driver_migrate_lock); + if (lock_err) + return lock_err; + write_locked = true; + } + drm_gpusvm_range_evict(range->base.gpusvm, &range->base); + } + } while (err == -EBUSY && retries--); + if (write_locked) + up_write(&driver_migrate_lock); + else + up_read(&driver_migrate_lock); + + return err; +} + +static struct drm_pagemap_addr +xe_drm_pagemap_device_map(struct drm_pagemap *dpagemap, + struct device *dev, + struct page *page, + unsigned int order, + enum dma_data_direction dir) +{ + struct device *pgmap_dev = dpagemap->drm->dev; + enum drm_interconnect_protocol prot; + dma_addr_t addr; + + if (pgmap_dev == dev) { + addr = xe_page_to_dpa(page); + prot = XE_INTERCONNECT_VRAM; + } else { + addr = dma_map_resource(dev, + xe_page_to_pcie(page), + PAGE_SIZE << order, dir, + DMA_ATTR_SKIP_CPU_SYNC); + prot = XE_INTERCONNECT_P2P; + } + + return drm_pagemap_addr_encode(addr, prot, order, dir); +} + +static void xe_drm_pagemap_device_unmap(struct drm_pagemap *dpagemap, + struct device *dev, + const struct drm_pagemap_addr *addr) +{ + if (addr->proto != XE_INTERCONNECT_P2P) + return; + + dma_unmap_resource(dev, addr->addr, PAGE_SIZE << addr->order, + addr->dir, DMA_ATTR_SKIP_CPU_SYNC); +} + +static void xe_pagemap_destroy_work(struct work_struct *work) +{ + struct xe_pagemap *xpagemap = container_of(work, typeof(*xpagemap), destroy_work); + struct dev_pagemap *pagemap = &xpagemap->pagemap; + struct drm_device *drm = xpagemap->dpagemap.drm; + int idx; + + /* + * Only unmap / release if devm_ release hasn't run yet. + * Otherwise the devm_ callbacks have already released, or + * will do shortly. + */ + if (drm_dev_enter(drm, &idx)) { + devm_memunmap_pages(drm->dev, pagemap); + devm_release_mem_region(drm->dev, pagemap->range.start, + pagemap->range.end - pagemap->range.start + 1); + drm_dev_exit(idx); + } + + drm_pagemap_release_owner(&xpagemap->peer); + kfree(xpagemap); +} + +static void xe_pagemap_destroy(struct drm_pagemap *dpagemap, bool from_atomic_or_reclaim) +{ + struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); + struct xe_device *xe = to_xe_device(dpagemap->drm); + + if (from_atomic_or_reclaim) + queue_work(xe->destroy_wq, &xpagemap->destroy_work); + else + xe_pagemap_destroy_work(&xpagemap->destroy_work); +} + +static const struct drm_pagemap_ops xe_drm_pagemap_ops = { + .device_map = xe_drm_pagemap_device_map, + .device_unmap = xe_drm_pagemap_device_unmap, + .populate_mm = xe_drm_pagemap_populate_mm, + .destroy = xe_pagemap_destroy, +}; + +/** + * xe_pagemap_create() - Create a struct xe_pagemap object + * @xe: The xe device. + * @vr: Back-pointer to the struct xe_vram_region. + * + * Allocate and initialize a struct xe_pagemap. On successful + * return, drm_pagemap_put() on the embedded struct drm_pagemap + * should be used to unreference. + * + * Return: Pointer to a struct xe_pagemap if successful. Error pointer + * on failure. + */ +static struct xe_pagemap *xe_pagemap_create(struct xe_device *xe, struct xe_vram_region *vr) +{ + struct device *dev = xe->drm.dev; + struct xe_pagemap *xpagemap; + struct dev_pagemap *pagemap; + struct drm_pagemap *dpagemap; + struct resource *res; + void *addr; + int err; + + xpagemap = kzalloc_obj(*xpagemap); + if (!xpagemap) + return ERR_PTR(-ENOMEM); + + pagemap = &xpagemap->pagemap; + dpagemap = &xpagemap->dpagemap; + INIT_WORK(&xpagemap->destroy_work, xe_pagemap_destroy_work); + xpagemap->vr = vr; + xpagemap->peer.private = XE_PEER_PAGEMAP; + + err = drm_pagemap_init(dpagemap, pagemap, &xe->drm, &xe_drm_pagemap_ops); + if (err) + goto out_no_dpagemap; + + res = devm_request_free_mem_region(dev, &iomem_resource, + vr->usable_size); + if (IS_ERR(res)) { + err = PTR_ERR(res); + goto out_err; + } + + err = drm_pagemap_acquire_owner(&xpagemap->peer, &xe_owner_list, + xe_has_interconnect); + if (err) + goto out_no_owner; + + pagemap->type = MEMORY_DEVICE_PRIVATE; + pagemap->range.start = res->start; + pagemap->range.end = res->end; + pagemap->nr_range = 1; + pagemap->owner = xpagemap->peer.owner; + pagemap->ops = drm_pagemap_pagemap_ops_get(); + addr = devm_memremap_pages(dev, pagemap); + if (IS_ERR(addr)) { + err = PTR_ERR(addr); + goto out_no_pages; + } + xpagemap->hpa_base = res->start; + return xpagemap; + +out_no_pages: + drm_pagemap_release_owner(&xpagemap->peer); +out_no_owner: + devm_release_mem_region(dev, res->start, res->end - res->start + 1); +out_err: + drm_pagemap_put(dpagemap); + return ERR_PTR(err); + +out_no_dpagemap: + kfree(xpagemap); + return ERR_PTR(err); +} + +/** + * xe_pagemap_find_or_create() - Find or create a struct xe_pagemap + * @xe: The xe device. + * @cache: The struct xe_pagemap_cache. + * @vr: The VRAM region. + * + * Check if there is an already used xe_pagemap for this tile, and in that case, + * return it. + * If not, check if there is a cached xe_pagemap for this tile, and in that case, + * cancel its destruction, re-initialize it and return it. + * Finally if there is no cached or already used pagemap, create one and + * register it in the tile's pagemap cache. + * + * Note that this function is typically called from within an IOCTL, and waits are + * therefore carried out interruptible if possible. + * + * Return: A pointer to a struct xe_pagemap if successful, Error pointer on failure. + */ +static struct xe_pagemap * +xe_pagemap_find_or_create(struct xe_device *xe, struct drm_pagemap_cache *cache, + struct xe_vram_region *vr) +{ + struct drm_pagemap *dpagemap; + struct xe_pagemap *xpagemap; + int err; + + err = drm_pagemap_cache_lock_lookup(cache); + if (err) + return ERR_PTR(err); + + dpagemap = drm_pagemap_get_from_cache(cache); + if (IS_ERR(dpagemap)) { + xpagemap = ERR_CAST(dpagemap); + } else if (!dpagemap) { + xpagemap = xe_pagemap_create(xe, vr); + if (IS_ERR(xpagemap)) + goto out_unlock; + drm_pagemap_cache_set_pagemap(cache, &xpagemap->dpagemap); + } else { + xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); + } + +out_unlock: + drm_pagemap_cache_unlock_lookup(cache); + return xpagemap; +} + +static int xe_svm_get_pagemaps(struct xe_vm *vm) +{ + struct xe_device *xe = vm->xe; + struct xe_pagemap *xpagemap; + struct xe_tile *tile; + int id; + + for_each_tile(tile, xe, id) { + struct xe_vram_region *vr; + + if (!((BIT(id) << 1) & xe->info.mem_region_mask)) + continue; + + vr = xe_tile_to_vr(tile); + xpagemap = xe_pagemap_find_or_create(xe, vr->dpagemap_cache, vr); + if (IS_ERR(xpagemap)) + break; + vm->svm.pagemaps[id] = xpagemap; + } + + if (IS_ERR(xpagemap)) { + xe_svm_put_pagemaps(vm); + return PTR_ERR(xpagemap); + } + + return 0; +} + +/** + * xe_pagemap_shrinker_create() - Create a drm_pagemap shrinker + * @xe: The xe device + * + * Create a drm_pagemap shrinker and register with the xe device. + * + * Return: %0 on success, negative error code on failure. + */ +int xe_pagemap_shrinker_create(struct xe_device *xe) +{ + xe->usm.dpagemap_shrinker = drm_pagemap_shrinker_create_devm(&xe->drm); + return PTR_ERR_OR_ZERO(xe->usm.dpagemap_shrinker); +} + +/** + * xe_pagemap_cache_create() - Create a drm_pagemap cache + * @tile: The tile to register the cache with + * + * Create a drm_pagemap cache and register with the tile. + * + * Return: %0 on success, negative error code on failure. + */ +int xe_pagemap_cache_create(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + + if (IS_DGFX(xe)) { + struct drm_pagemap_cache *cache = + drm_pagemap_cache_create_devm(xe->usm.dpagemap_shrinker); + + if (IS_ERR(cache)) + return PTR_ERR(cache); + + tile->mem.vram->dpagemap_cache = cache; + } + + return 0; +} + +static struct drm_pagemap *xe_devmem_open(struct xe_device *xe, u32 region_instance) +{ + u32 tile_id = region_instance - 1; + struct xe_pagemap *xpagemap; + struct xe_vram_region *vr; + + if (tile_id >= xe->info.tile_count) + return ERR_PTR(-ENOENT); + + if (!((BIT(tile_id) << 1) & xe->info.mem_region_mask)) + return ERR_PTR(-ENOENT); + + vr = xe_tile_to_vr(&xe->tiles[tile_id]); + + /* Returns a reference-counted embedded struct drm_pagemap */ + xpagemap = xe_pagemap_find_or_create(xe, vr->dpagemap_cache, vr); + if (IS_ERR(xpagemap)) + return ERR_CAST(xpagemap); + + return &xpagemap->dpagemap; +} + +/** + * xe_drm_pagemap_from_fd() - Return a drm_pagemap pointer from a + * (file_descriptor, region_instance) pair. + * @fd: An fd opened against an xe device. + * @region_instance: The region instance representing the device memory + * on the opened xe device. + * + * Opens a struct drm_pagemap pointer on the + * indicated device and region_instance. + * + * Return: A reference-counted struct drm_pagemap pointer on success, + * negative error pointer on failure. + */ +struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) +{ + struct drm_pagemap *dpagemap; + struct file *file; + struct drm_file *fpriv; + struct drm_device *drm; + int idx; + + if (fd <= 0) + return ERR_PTR(-EINVAL); + + file = fget(fd); + if (!file) + return ERR_PTR(-ENOENT); + + if (!xe_is_xe_file(file)) { + dpagemap = ERR_PTR(-ENOENT); + goto out; + } + + fpriv = file->private_data; + drm = fpriv->minor->dev; + if (!drm_dev_enter(drm, &idx)) { + dpagemap = ERR_PTR(-ENODEV); + goto out; + } + + dpagemap = xe_devmem_open(to_xe_device(drm), region_instance); + drm_dev_exit(idx); +out: + fput(file); + return dpagemap; +} + +#else + +int xe_pagemap_shrinker_create(struct xe_device *xe) +{ + return 0; +} + +int xe_pagemap_cache_create(struct xe_tile *tile) +{ + return 0; +} + +int xe_svm_alloc_vram(struct xe_svm_range *range, + const struct drm_gpusvm_ctx *ctx, + struct drm_pagemap *dpagemap) +{ + return -EOPNOTSUPP; +} + +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) +{ + return NULL; +} + +struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) +{ + return ERR_PTR(-ENOENT); +} + +#endif + +/** + * xe_svm_flush() - SVM flush + * @vm: The VM. + * + * Flush all SVM actions. + */ +void xe_svm_flush(struct xe_vm *vm) +{ + if (xe_vm_in_fault_mode(vm)) + flush_work(&vm->svm.garbage_collector.work); +} diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h new file mode 100644 index 000000000..a921556d3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -0,0 +1,443 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_SVM_H_ +#define _XE_SVM_H_ + +#if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) + +#include +#include +#include + +#define XE_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER +#define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) + +struct drm_device; +struct drm_file; + +struct xe_bo; +struct xe_gt; +struct xe_device; +struct xe_vram_region; +struct xe_tile; +struct xe_vm; +struct xe_vma; +struct xe_vram_region; + +/** struct xe_svm_range - SVM range */ +struct xe_svm_range { + /** @base: base drm_gpusvm_range */ + struct drm_gpusvm_range base; + /** @pages: Page/DMA mapping state for this range (single drm_device). */ + struct drm_gpusvm_pages pages; + /** + * @garbage_collector_link: Link into VM's garbage collect SVM range + * list. Protected by VM's garbage collect lock. + */ + struct list_head garbage_collector_link; + /** + * @tile_present: Tile mask of binding is present for this range. + * Protected by GPU SVM notifier lock. + */ + u8 tile_present; + /** + * @tile_invalidated: Tile mask of binding is invalidated for this + * range. Protected by GPU SVM notifier lock. + */ + u8 tile_invalidated; +}; + +/** + * struct xe_pagemap - Manages xe device_private memory for SVM. + * @pagemap: The struct dev_pagemap providing the struct pages. + * @dpagemap: The drm_pagemap managing allocation and migration. + * @destroy_work: Handles asnynchronous destruction and caching. + * @peer: Used for pagemap owner computation. + * @hpa_base: The host physical address base for the managemd memory. + * @vr: Backpointer to the xe_vram region. + */ +struct xe_pagemap { + struct dev_pagemap pagemap; + struct drm_pagemap dpagemap; + struct work_struct destroy_work; + struct drm_pagemap_peer peer; + resource_size_t hpa_base; + struct xe_vram_region *vr; +}; + +/** + * xe_svm_range_pages_valid() - SVM range pages valid + * @range: SVM range + * + * Return: True if SVM range pages are valid, False otherwise + */ +static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range) +{ + return drm_gpusvm_pages_valid(range->base.gpusvm, &range->pages); +} + +int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr); + +int xe_svm_init(struct xe_vm *vm); + +void xe_svm_fini(struct xe_vm *vm); + +void xe_svm_close(struct xe_vm *vm); + +int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, + struct xe_gt *gt, u64 fault_addr, + bool atomic); + +bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end); + +int xe_svm_bo_evict(struct xe_bo *bo); + +void xe_svm_range_debug(struct xe_svm_range *range, const char *operation); + +int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, + struct drm_pagemap *dpagemap); + +struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, + struct xe_vma *vma, struct drm_gpusvm_ctx *ctx); + +int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, + struct drm_gpusvm_ctx *ctx); + +bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, + const struct drm_pagemap *dpagemap); + +void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range); + +bool xe_svm_range_validate(struct xe_vm *vm, + struct xe_svm_range *range, + u8 tile_mask, const struct drm_pagemap *dpagemap); + +u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma); + +void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end); + +u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end); + +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile); + +void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem); + +/** + * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping + * @range: SVM range + * + * Return: True if SVM range has a DMA mapping, False otherwise + */ +static inline bool xe_svm_range_has_dma_mapping(struct xe_svm_range *range) +{ + lockdep_assert_held(&range->base.gpusvm->notifier_lock); + return range->pages.flags.has_dma_mapping; +} + +/** + * to_xe_range - Convert a drm_gpusvm_range pointer to a xe_svm_range + * @r: Pointer to the drm_gpusvm_range structure + * + * This function takes a pointer to a drm_gpusvm_range structure and + * converts it to a pointer to the containing xe_svm_range structure. + * + * Return: Pointer to the xe_svm_range structure + */ +static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) +{ + return container_of(r, struct xe_svm_range, base); +} + +/** + * xe_svm_range_start() - SVM range start address + * @range: SVM range + * + * Return: start address of range. + */ +static inline unsigned long xe_svm_range_start(struct xe_svm_range *range) +{ + return drm_gpusvm_range_start(&range->base); +} + +/** + * xe_svm_range_end() - SVM range end address + * @range: SVM range + * + * Return: end address of range. + */ +static inline unsigned long xe_svm_range_end(struct xe_svm_range *range) +{ + return drm_gpusvm_range_end(&range->base); +} + +/** + * xe_svm_range_size() - SVM range size + * @range: SVM range + * + * Return: Size of range. + */ +static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) +{ + return drm_gpusvm_range_size(&range->base); +} + +void xe_svm_flush(struct xe_vm *vm); + +int xe_pagemap_shrinker_create(struct xe_device *xe); + +int xe_pagemap_cache_create(struct xe_tile *tile); + +struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance); + +#else +#include +#include "xe_vm.h" + +struct drm_pagemap_addr; +struct drm_gpusvm_ctx; +struct drm_gpusvm_range; +struct xe_bo; +struct xe_device; +struct xe_vm; +struct xe_vma; +struct xe_tile; +struct xe_vram_region; + +#define XE_INTERCONNECT_VRAM 1 +#define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) + +struct xe_svm_range { + struct { + struct interval_tree_node itree; + } base; + struct { + const struct drm_pagemap_addr *dma_addr; + } pages; + u32 tile_present; + u32 tile_invalidated; +}; + +static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range) +{ + return false; +} + +static inline +int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) +{ + return 0; +} + +static inline +int xe_svm_init(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_GPUSVM) + return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", + NULL, 0, 0, 0, NULL, NULL, 0); +#else + return 0; +#endif +} + +static inline +void xe_svm_fini(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_GPUSVM) + xe_assert(vm->xe, xe_vm_is_closed(vm)); + drm_gpusvm_fini(&vm->svm.gpusvm); +#endif +} + +static inline +void xe_svm_close(struct xe_vm *vm) +{ +} + +static inline +int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, + struct xe_gt *gt, u64 fault_addr, + bool atomic) +{ + return 0; +} + +static inline +bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) +{ + return false; +} + +static inline +int xe_svm_bo_evict(struct xe_bo *bo) +{ + return 0; +} + +static inline +void xe_svm_range_debug(struct xe_svm_range *range, const char *operation) +{ +} + +static inline int +xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, + struct drm_pagemap *dpagemap) +{ + return -EOPNOTSUPP; +} + +static inline +struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, + struct xe_vma *vma, struct drm_gpusvm_ctx *ctx) +{ + return ERR_PTR(-EINVAL); +} + +static inline +int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, + struct drm_gpusvm_ctx *ctx) +{ + return -EINVAL; +} + +static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) +{ + return NULL; +} + +static inline unsigned long xe_svm_range_start(struct xe_svm_range *range) +{ + return 0; +} + +static inline unsigned long xe_svm_range_end(struct xe_svm_range *range) +{ + return 0; +} + +static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) +{ + return 0; +} + +static inline +bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, + const struct drm_pagemap *dpagemap) +{ + return false; +} + +static inline +void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range) +{ +} + +static inline +bool xe_svm_range_validate(struct xe_vm *vm, + struct xe_svm_range *range, + u8 tile_mask, bool devmem_preferred) +{ + return false; +} + +static inline +u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma) +{ + return ULONG_MAX; +} + +static inline +void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) +{ +} + +static inline +u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) +{ + return 0; +} + +static inline +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) +{ + return NULL; +} + +static inline void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) +{ + return NULL; +} + +static inline void xe_svm_flush(struct xe_vm *vm) +{ +} + +static inline int xe_pagemap_shrinker_create(struct xe_device *xe) +{ + return 0; +} + +static inline int xe_pagemap_cache_create(struct xe_tile *tile) +{ + return 0; +} + +static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) +{ + return ERR_PTR(-ENOENT); +} + +#define xe_svm_range_has_dma_mapping(...) false +#endif /* CONFIG_DRM_XE_GPUSVM */ + +#if IS_ENABLED(CONFIG_DRM_GPUSVM) /* Need to support userptr without XE_GPUSVM */ +#define xe_svm_assert_in_notifier(vm__) \ + lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) + +/* + * Assert the svm notifier_lock is held. Read mode by default; write mode + * when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, because that path forces + * a userptr invalidation that ends in drm_gpusvm_unmap_pages() with + * ctx->in_notifier=true, which requires the lock held for write. + */ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) +#define xe_svm_assert_held_read_or_inject_write(vm__) \ + lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) +#else +#define xe_svm_assert_held_read_or_inject_write(vm__) \ + lockdep_assert_held_read(&(vm__)->svm.gpusvm.notifier_lock) +#endif + +#define xe_svm_notifier_lock(vm__) \ + drm_gpusvm_notifier_lock(&(vm__)->svm.gpusvm) + +#define xe_svm_notifier_lock_interruptible(vm__) \ + down_read_interruptible(&(vm__)->svm.gpusvm.notifier_lock) + +#define xe_svm_notifier_unlock(vm__) \ + drm_gpusvm_notifier_unlock(&(vm__)->svm.gpusvm) + +#else +#define xe_svm_assert_in_notifier(...) do {} while (0) + +static inline void xe_svm_assert_held_read_or_inject_write(struct xe_vm *vm) +{ +} + +static inline void xe_svm_notifier_lock(struct xe_vm *vm) +{ +} + +static inline int xe_svm_notifier_lock_interruptible(struct xe_vm *vm) +{ + return 0; +} + +static inline void xe_svm_notifier_unlock(struct xe_vm *vm) +{ +} +#endif /* CONFIG_DRM_GPUSVM */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c new file mode 100644 index 000000000..37866768d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_sync.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_macros.h" +#include "xe_sched_job_types.h" + +struct xe_user_fence { + struct xe_device *xe; + struct kref refcount; + struct dma_fence_cb cb; + struct work_struct worker; + struct mm_struct *mm; + u64 __user *addr; + u64 value; + int signalled; +}; + +static void user_fence_destroy(struct kref *kref) +{ + struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, + refcount); + + mmdrop(ufence->mm); + kfree(ufence); +} + +static void user_fence_get(struct xe_user_fence *ufence) +{ + kref_get(&ufence->refcount); +} + +static void user_fence_put(struct xe_user_fence *ufence) +{ + kref_put(&ufence->refcount, user_fence_destroy); +} + +static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, + u64 value) +{ + struct xe_user_fence *ufence; + u64 __user *ptr = u64_to_user_ptr(addr); + u64 __maybe_unused prefetch_val; + + if (get_user(prefetch_val, ptr)) + return ERR_PTR(-EFAULT); + + ufence = kzalloc_obj(*ufence); + if (!ufence) + return ERR_PTR(-ENOMEM); + + ufence->xe = xe; + kref_init(&ufence->refcount); + ufence->addr = ptr; + ufence->value = value; + ufence->mm = current->mm; + mmgrab(ufence->mm); + + return ufence; +} + +static void user_fence_worker(struct work_struct *w) +{ + struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); + + WRITE_ONCE(ufence->signalled, 1); + if (mmget_not_zero(ufence->mm)) { + kthread_use_mm(ufence->mm); + if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) + XE_WARN_ON("Copy to user failed"); + kthread_unuse_mm(ufence->mm); + mmput(ufence->mm); + } else { + drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n"); + } + + /* + * Wake up waiters only after updating the ufence state, allowing the UMD + * to safely reuse the same ufence without encountering -EBUSY errors. + */ + wake_up_all(&ufence->xe->ufence_wq); + user_fence_put(ufence); +} + +static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) +{ + INIT_WORK(&ufence->worker, user_fence_worker); + queue_work(ufence->xe->ordered_wq, &ufence->worker); + dma_fence_put(fence); +} + +static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); + + kick_ufence(ufence, fence); +} + +int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, + struct xe_sync_entry *sync, + struct drm_xe_sync __user *sync_user, + struct drm_syncobj *ufence_syncobj, + u64 ufence_timeline_value, + unsigned int flags) +{ + struct drm_xe_sync sync_in; + int err; + bool exec = flags & SYNC_PARSE_FLAG_EXEC; + bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; + bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; + bool signal; + + if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || + XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) + return -EINVAL; + + signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; + switch (sync_in.type) { + case DRM_XE_SYNC_TYPE_SYNCOBJ: + if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) + return -EOPNOTSUPP; + + if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) + return -EINVAL; + + sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); + if (XE_IOCTL_DBG(xe, !sync->syncobj)) + return -ENOENT; + + if (!signal) { + sync->fence = drm_syncobj_fence_get(sync->syncobj); + if (XE_IOCTL_DBG(xe, !sync->fence)) { + err = -EINVAL; + goto free_sync; + } + } + break; + + case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: + if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) + return -EOPNOTSUPP; + + if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) + return -EINVAL; + + sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); + if (XE_IOCTL_DBG(xe, !sync->syncobj)) + return -ENOENT; + + if (signal) { + sync->chain_fence = dma_fence_chain_alloc(); + if (!sync->chain_fence) { + err = -ENOMEM; + goto free_sync; + } + } else { + sync->fence = drm_syncobj_fence_get(sync->syncobj); + if (XE_IOCTL_DBG(xe, !sync->fence)) { + err = -EINVAL; + goto free_sync; + } + + err = dma_fence_chain_find_seqno(&sync->fence, + sync_in.timeline_value); + if (err) + goto free_sync; + } + break; + + case DRM_XE_SYNC_TYPE_USER_FENCE: + if (XE_IOCTL_DBG(xe, disallow_user_fence)) + return -EOPNOTSUPP; + + if (XE_IOCTL_DBG(xe, !signal)) + return -EOPNOTSUPP; + + if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) + return -EINVAL; + + if (exec) { + sync->addr = sync_in.addr; + } else { + sync->ufence_timeline_value = ufence_timeline_value; + sync->ufence = user_fence_create(xe, sync_in.addr, + sync_in.timeline_value); + if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) + return PTR_ERR(sync->ufence); + sync->ufence_chain_fence = dma_fence_chain_alloc(); + if (!sync->ufence_chain_fence) { + err = -ENOMEM; + goto free_sync; + } + sync->ufence_syncobj = ufence_syncobj; + } + + break; + + default: + return -EINVAL; + } + + sync->type = sync_in.type; + sync->flags = sync_in.flags; + sync->timeline_value = sync_in.timeline_value; + + return 0; + +free_sync: + xe_sync_entry_cleanup(sync); + return err; +} +ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO); + +int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) +{ + if (sync->fence) + return drm_sched_job_add_dependency(&job->drm, + dma_fence_get(sync->fence)); + + return 0; +} + +/** + * xe_sync_entry_wait() - Wait on in-sync + * @sync: Sync object + * + * If the sync is in an in-sync, wait on the sync to signal. + * + * Return: 0 on success, -ERESTARTSYS on failure (interruption) + */ +int xe_sync_entry_wait(struct xe_sync_entry *sync) +{ + return xe_sync_needs_wait(sync) ? + dma_fence_wait(sync->fence, true) : 0; +} + +/** + * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled) + * @sync: Sync object + * + * Return: True if sync needs a wait, False otherwise + */ +bool xe_sync_needs_wait(struct xe_sync_entry *sync) +{ + return sync->fence && + !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); +} + +void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) +{ + if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) + return; + + if (sync->chain_fence) { + drm_syncobj_add_point(sync->syncobj, sync->chain_fence, + fence, sync->timeline_value); + /* + * The chain's ownership is transferred to the + * timeline. + */ + sync->chain_fence = NULL; + } else if (sync->syncobj) { + drm_syncobj_replace_fence(sync->syncobj, fence); + } else if (sync->ufence) { + int err; + + drm_syncobj_add_point(sync->ufence_syncobj, + sync->ufence_chain_fence, + fence, sync->ufence_timeline_value); + sync->ufence_chain_fence = NULL; + + fence = drm_syncobj_fence_get(sync->ufence_syncobj); + user_fence_get(sync->ufence); + err = dma_fence_add_callback(fence, &sync->ufence->cb, + user_fence_cb); + if (err == -ENOENT) { + kick_ufence(sync->ufence, fence); + } else if (err) { + XE_WARN_ON("failed to add user fence"); + user_fence_put(sync->ufence); + dma_fence_put(fence); + } + } +} + +void xe_sync_entry_cleanup(struct xe_sync_entry *sync) +{ + if (sync->syncobj) + drm_syncobj_put(sync->syncobj); + dma_fence_put(sync->fence); + dma_fence_chain_free(sync->chain_fence); + dma_fence_chain_free(sync->ufence_chain_fence); + if (!IS_ERR_OR_NULL(sync->ufence)) + user_fence_put(sync->ufence); +} + +/** + * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM + * @sync: input syncs + * @num_sync: number of syncs + * @q: exec queue + * @vm: VM + * + * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create + * and return a composite fence of all in-fences + last fence. If no in-fences + * return last fence on input exec queue. Caller must drop reference to + * returned fence. + * + * Return: fence on success, ERR_PTR(-ENOMEM) on failure + */ +struct dma_fence * +xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, + struct xe_exec_queue *q, struct xe_vm *vm) +{ + struct dma_fence **fences = NULL; + struct dma_fence_array *cf = NULL; + struct dma_fence *fence; + int i, num_fence = 0, current_fence = 0; + + lockdep_assert_held(&vm->lock); + + /* Reject in fences */ + for (i = 0; i < num_sync; ++i) + if (sync[i].fence) + return ERR_PTR(-EOPNOTSUPP); + + if (q->flags & EXEC_QUEUE_FLAG_VM) { + struct xe_exec_queue *__q; + struct xe_tile *tile; + u8 id; + + for_each_tile(tile, vm->xe, id) { + num_fence++; + for_each_tlb_inval(i) + num_fence++; + } + + fences = kmalloc_objs(*fences, num_fence); + if (!fences) + return ERR_PTR(-ENOMEM); + + fences[current_fence++] = + xe_exec_queue_last_fence_get(q, vm); + for_each_tlb_inval(i) + fences[current_fence++] = + xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); + list_for_each_entry(__q, &q->multi_gt_list, + multi_gt_link) { + fences[current_fence++] = + xe_exec_queue_last_fence_get(__q, vm); + for_each_tlb_inval(i) + fences[current_fence++] = + xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i); + } + + xe_assert(vm->xe, current_fence == num_fence); + cf = dma_fence_array_create(num_fence, fences, + dma_fence_context_alloc(1), + 1); + if (!cf) + goto err_out; + + return &cf->base; + } + + fence = xe_exec_queue_last_fence_get(q, vm); + return fence; + +err_out: + while (current_fence) + dma_fence_put(fences[--current_fence]); + kfree(fences); + + return ERR_PTR(-ENOMEM); +} + +/** + * __xe_sync_ufence_get() - Get user fence from user fence + * @ufence: input user fence + * + * Get a user fence reference from user fence + * + * Return: xe_user_fence pointer with reference + */ +struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) +{ + user_fence_get(ufence); + + return ufence; +} + +/** + * xe_sync_ufence_get() - Get user fence from sync + * @sync: input sync + * + * Get a user fence reference from sync. + * + * Return: xe_user_fence pointer with reference + */ +struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) +{ + user_fence_get(sync->ufence); + + return sync->ufence; +} + +/** + * xe_sync_ufence_put() - Put user fence reference + * @ufence: user fence reference + * + */ +void xe_sync_ufence_put(struct xe_user_fence *ufence) +{ + user_fence_put(ufence); +} + +/** + * xe_sync_ufence_get_status() - Get user fence status + * @ufence: user fence + * + * Return: 1 if signalled, 0 not signalled, <0 on error + */ +int xe_sync_ufence_get_status(struct xe_user_fence *ufence) +{ + return READ_ONCE(ufence->signalled); +} diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h new file mode 100644 index 000000000..6b949194a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sync.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_SYNC_H_ +#define _XE_SYNC_H_ + +#include "xe_sync_types.h" + +struct drm_syncobj; +struct xe_device; +struct xe_exec_queue; +struct xe_file; +struct xe_sched_job; +struct xe_vm; + +#define SYNC_PARSE_FLAG_EXEC BIT(0) +#define SYNC_PARSE_FLAG_LR_MODE BIT(1) +#define SYNC_PARSE_FLAG_DISALLOW_USER_FENCE BIT(2) + +int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, + struct xe_sync_entry *sync, + struct drm_xe_sync __user *sync_user, + struct drm_syncobj *ufence_syncobj, + u64 ufence_timeline_value, + unsigned int flags); +int xe_sync_entry_add_deps(struct xe_sync_entry *sync, + struct xe_sched_job *job); +void xe_sync_entry_signal(struct xe_sync_entry *sync, + struct dma_fence *fence); +int xe_sync_entry_wait(struct xe_sync_entry *sync); +bool xe_sync_needs_wait(struct xe_sync_entry *sync); +void xe_sync_entry_cleanup(struct xe_sync_entry *sync); +struct dma_fence * +xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, + struct xe_exec_queue *q, struct xe_vm *vm); + +static inline bool xe_sync_is_ufence(struct xe_sync_entry *sync) +{ + return !!sync->ufence; +} + +struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence); +struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync); +void xe_sync_ufence_put(struct xe_user_fence *ufence); +int xe_sync_ufence_get_status(struct xe_user_fence *ufence); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sync_types.h b/drivers/gpu/drm/xe/xe_sync_types.h new file mode 100644 index 000000000..b88f1833e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sync_types.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_SYNC_TYPES_H_ +#define _XE_SYNC_TYPES_H_ + +#include + +struct drm_syncobj; +struct dma_fence; +struct dma_fence_chain; +struct drm_xe_sync; +struct user_fence; + +struct xe_sync_entry { + struct drm_syncobj *syncobj; + struct dma_fence *fence; + struct dma_fence_chain *chain_fence; + struct dma_fence_chain *ufence_chain_fence; + struct drm_syncobj *ufence_syncobj; + struct xe_user_fence *ufence; + u64 addr; + u64 timeline_value; + u64 ufence_timeline_value; + u32 type; + u32 flags; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c new file mode 100644 index 000000000..62ccc9be7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include +#include + +#include + +#include "regs/xe_irq_regs.h" +#include "regs/xe_sysctrl_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_soc_remapper.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_types.h" + +/** + * DOC: System Controller (sysctrl) + * + * System Controller (sysctrl) is a firmware-managed entity on Intel dGPUs + * responsible for selected low-level platform management functions. + * Communication between driver and System Controller is performed + * via a mailbox interface, enabling command and response exchange. + * + * This module provides initialization and support code for interacting + * with System Controller through the mailbox interface. + */ +static void sysctrl_fini(void *arg) +{ + struct xe_device *xe = arg; + struct xe_sysctrl *sc = &xe->sc; + + disable_work_sync(&sc->work); + xe->soc_remapper.set_sysctrl_region(xe, 0); +} + +static void xe_sysctrl_work(struct work_struct *work) +{ + struct xe_sysctrl *sc = container_of(work, struct xe_sysctrl, work); + struct xe_device *xe = sc_to_xe(sc); + + guard(xe_pm_runtime)(xe); + xe_sysctrl_event(sc); +} + +/** + * xe_sysctrl_init() - Initialize System Controller subsystem + * @xe: xe device instance + * + * Entry point for System Controller initialization, called from xe_device_probe. + * This function checks platform support and initializes the system controller. + * + * Return: 0 on success, error code on failure + */ +int xe_sysctrl_init(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_sysctrl *sc = &xe->sc; + int ret; + + if (!xe->info.has_soc_remapper_sysctrl) + return 0; + + if (!xe->info.has_sysctrl) + return 0; + + sc->mmio = devm_kzalloc(xe->drm.dev, sizeof(*sc->mmio), GFP_KERNEL); + if (!sc->mmio) + return -ENOMEM; + + xe_mmio_init(sc->mmio, tile, tile->mmio.regs, tile->mmio.regs_size); + sc->mmio->adj_offset = SYSCTRL_BASE; + sc->mmio->adj_limit = U32_MAX; + + ret = devm_mutex_init(xe->drm.dev, &sc->cmd_lock); + if (ret) + return ret; + + ret = devm_mutex_init(xe->drm.dev, &sc->event_lock); + if (ret) + return ret; + + xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); + INIT_WORK(&sc->work, xe_sysctrl_work); + + return devm_add_action_or_reset(xe->drm.dev, sysctrl_fini, xe); +} + +/** + * xe_sysctrl_irq_handler() - Handler for System Controller interrupts + * @xe: xe device instance + * @master_ctl: interrupt register + * + * Handle interrupts generated by System Controller. + */ +void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + struct xe_sysctrl *sc = &xe->sc; + + if (!xe->info.has_sysctrl || !sc->work.func) + return; + + if (master_ctl & SYSCTRL_IRQ) + schedule_work(&sc->work); +} + +/** + * xe_sysctrl_pm_resume() - System Controller resume handler + * @xe: xe device instance + * + * Invoked during system resume (S3/S4 to S0) and runtime resume from D3cold. + * Restores SoC remapper configuration. + */ +void xe_sysctrl_pm_resume(struct xe_device *xe) +{ + if (!xe->info.has_soc_remapper_sysctrl) + return; + + if (!xe->info.has_sysctrl) + return; + + xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h new file mode 100644 index 000000000..090dffb6d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_H_ +#define _XE_SYSCTRL_H_ + +#include + +#include "xe_device_types.h" +#include "xe_sysctrl_types.h" + +static inline struct xe_device *sc_to_xe(struct xe_sysctrl *sc) +{ + return container_of(sc, struct xe_device, sc); +} + +void xe_sysctrl_event(struct xe_sysctrl *sc); +int xe_sysctrl_init(struct xe_device *xe); +void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl); +void xe_sysctrl_pm_resume(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event.c b/drivers/gpu/drm/xe/xe_sysctrl_event.c new file mode 100644 index 000000000..da395148e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_event.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_irq.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_event_types.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +static void get_pending_event(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_command *command) +{ + struct xe_sysctrl_event_response *response = command->data_out; + struct xe_device *xe = sc_to_xe(sc); + u32 count = XE_SYSCTRL_FLOOD_LIMIT; + size_t len; + int ret; + + do { + memset(response, 0, sizeof(*response)); + + ret = xe_sysctrl_send_command(sc, command, &len); + if (ret) { + xe_err(xe, "sysctrl: failed to get pending event %d\n", ret); + return; + } + + if (len != sizeof(*response)) { + xe_err(xe, "sysctrl: unexpected event response length %zu (expected %zu)\n", + len, sizeof(*response)); + return; + } + + if (response->event == XE_SYSCTRL_EVENT_THRESHOLD_CROSSED) + xe_ras_counter_threshold_crossed(xe, response); + else + xe_warn(xe, "sysctrl: unexpected event %#x\n", response->event); + + if (!--count) { + xe_err(xe, "sysctrl: event flooding\n"); + return; + } + + xe_dbg(xe, "sysctrl: %u events pending\n", response->count); + } while (response->count); +} + +static void event_request_prepare(struct xe_device *xe, struct xe_sysctrl_app_msg_hdr *header, + struct xe_sysctrl_event_request *request) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + header->data = REG_FIELD_PREP(APP_HDR_GROUP_ID_MASK, XE_SYSCTRL_GROUP_GFSP) | + REG_FIELD_PREP(APP_HDR_COMMAND_MASK, XE_SYSCTRL_CMD_GET_PENDING_EVENT); + + request->vector = xe_device_has_msix(xe) ? XE_IRQ_DEFAULT_MSIX : 0; + request->fn = PCI_FUNC(pdev->devfn); +} + +/** + * xe_sysctrl_event() - Handler for System Controller events + * @sc: System Controller instance + * + * Handle events generated by System Controller. + */ +void xe_sysctrl_event(struct xe_sysctrl *sc) +{ + struct xe_sysctrl_mailbox_command command = {}; + struct xe_sysctrl_event_response response = {}; + struct xe_sysctrl_event_request request = {}; + struct xe_sysctrl_app_msg_hdr header = {}; + + xe_device_assert_mem_access(sc_to_xe(sc)); + event_request_prepare(sc_to_xe(sc), &header, &request); + + command.header = header; + command.data_in = &request; + command.data_in_len = sizeof(request); + command.data_out = &response; + command.data_out_len = sizeof(response); + + guard(mutex)(&sc->event_lock); + get_pending_event(sc, &command); +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event_types.h b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h new file mode 100644 index 000000000..348768ca4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_EVENT_TYPES_H_ +#define _XE_SYSCTRL_EVENT_TYPES_H_ + +#include + +#define XE_SYSCTRL_EVENT_DATA_LEN 59 + +/** + * enum xe_sysctrl_event - Events reported by System Controller + * + * @XE_SYSCTRL_EVENT_THRESHOLD_CROSSED: Error counter threshold crossed + */ +enum xe_sysctrl_event { + XE_SYSCTRL_EVENT_THRESHOLD_CROSSED = 0x01, +}; + +/** + * struct xe_sysctrl_event_request - Request structure for pending event + */ +struct xe_sysctrl_event_request { + /** @vector: MSI-X vector that was triggered */ + u32 vector; + /** @fn: Function index (0-7) of PCIe device */ + u32 fn:8; + /** @reserved: Reserved for future use */ + u32 reserved:24; + /** @reserved1: Reserved for future use */ + u32 reserved1[2]; +} __packed; + +/** + * struct xe_sysctrl_event_response - Response structure for pending event + */ +struct xe_sysctrl_event_response { + /** @count: Pending event count after this response */ + u32 count; + /** @event: Pending event type */ + u32 event; + /** @timestamp: Timestamp of most recent event */ + u64 timestamp; + /** @extended: Event has extended payload */ + u32 extended:1; + /** @reserved: Reserved for future use */ + u32 reserved:31; + /** @data: Generic event data */ + u32 data[XE_SYSCTRL_EVENT_DATA_LEN]; +} __packed; + +#endif /* _XE_SYSCTRL_EVENT_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c new file mode 100644 index 000000000..72baf1aa4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include "regs/xe_sysctrl_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +struct xe_sysctrl_mailbox_msg_hdr { + __le32 data; +} __packed; + +#define XE_SYSCTRL_HDR_GROUP_ID(hdr) \ + FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_COMMAND(hdr) \ + FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_IS_RESPONSE(hdr) \ + FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_RESULT(hdr) \ + FIELD_GET(SYSCTRL_HDR_RESULT_MASK, le32_to_cpu((hdr)->data)) + +static bool sysctrl_wait_bit_clear(struct xe_sysctrl *sc, u32 bit_mask, + unsigned int timeout_ms) +{ + int ret; + + ret = xe_mmio_wait32_not(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask, + timeout_ms * 1000, NULL, false); + + return ret == 0; +} + +static bool sysctrl_wait_bit_set(struct xe_sysctrl *sc, u32 bit_mask, + unsigned int timeout_ms) +{ + int ret; + + ret = xe_mmio_wait32(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask, + timeout_ms * 1000, NULL, false); + + return ret == 0; +} + +static int sysctrl_write_frame(struct xe_sysctrl *sc, const void *frame, + size_t len) +{ + static const struct xe_reg regs[] = { + SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3 + }; + struct xe_device *xe = sc_to_xe(sc); + u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0}; + u32 dw = DIV_ROUND_UP(len, sizeof(u32)); + u32 i; + + xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE); + + memcpy(val, frame, len); + + for (i = 0; i < dw; i++) + xe_mmio_write32(sc->mmio, regs[i], val[i]); + + return 0; +} + +static int sysctrl_read_frame(struct xe_sysctrl *sc, void *frame, + size_t len) +{ + static const struct xe_reg regs[] = { + SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3 + }; + struct xe_device *xe = sc_to_xe(sc); + u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0}; + u32 dw = DIV_ROUND_UP(len, sizeof(u32)); + u32 i; + + xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE); + + for (i = 0; i < dw; i++) + val[i] = xe_mmio_read32(sc->mmio, regs[i]); + + memcpy(frame, val, len); + + return 0; +} + +static void sysctrl_clear_response(struct xe_sysctrl *sc) +{ + xe_mmio_rmw32(sc->mmio, SYSCTRL_MB_CTRL, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, 0); +} + +static int sysctrl_prepare_command(struct xe_device *xe, + u8 group_id, u8 command, + const void *data_in, size_t data_in_len, + u8 **mbox_cmd, size_t *cmd_size) +{ + struct xe_sysctrl_mailbox_msg_hdr *hdr; + size_t size; + u8 *buffer; + + xe_assert(xe, command <= SYSCTRL_HDR_COMMAND_MAX); + + if (data_in_len > XE_SYSCTRL_MB_MAX_MESSAGE_SIZE - sizeof(*hdr)) { + xe_err(xe, "sysctrl: Input data too large: %zu bytes\n", data_in_len); + return -EINVAL; + } + + size = sizeof(*hdr) + data_in_len; + + buffer = kmalloc(size, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; + hdr->data = cpu_to_le32(FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command)); + + if (data_in && data_in_len) + memcpy(buffer + sizeof(*hdr), data_in, data_in_len); + + *mbox_cmd = buffer; + *cmd_size = size; + + return 0; +} + +static int sysctrl_send_frames(struct xe_sysctrl *sc, + const u8 *mbox_cmd, + size_t cmd_size, unsigned int timeout_ms) +{ + struct xe_device *xe = sc_to_xe(sc); + u32 ctrl_reg, total_frames, frame; + size_t bytes_sent, frame_size; + bool phase; + + total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE); + + if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { + xe_err(xe, "sysctrl: Mailbox busy\n"); + return -EBUSY; + } + + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + phase = !(ctrl_reg & SYSCTRL_FRAME_PHASE); + bytes_sent = 0; + + for (frame = 0; frame < total_frames; frame++) { + frame_size = min_t(size_t, cmd_size - bytes_sent, XE_SYSCTRL_MB_FRAME_SIZE); + + if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) { + xe_err(xe, "sysctrl: Failed to write frame %u\n", frame); + return -EIO; + } + + ctrl_reg = SYSCTRL_MB_CTRL_RUN_BUSY | + REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) | + REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) | + SYSCTRL_MB_CTRL_CMD | + (phase ? SYSCTRL_FRAME_PHASE : 0); + + xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg); + + if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { + xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame); + return -ETIMEDOUT; + } + + bytes_sent += frame_size; + } + + return 0; +} + +static int sysctrl_process_frame(struct xe_sysctrl *sc, void *out, + size_t frame_size, unsigned int timeout_ms, + bool *done) +{ + u32 curr_frame, total_frames, ctrl_reg; + struct xe_device *xe = sc_to_xe(sc); + int ret; + + if (!sysctrl_wait_bit_set(sc, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, timeout_ms)) { + xe_err(xe, "sysctrl: Response frame timeout\n"); + return -ETIMEDOUT; + } + + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + total_frames = FIELD_GET(SYSCTRL_FRAME_TOTAL_MASK, ctrl_reg); + curr_frame = FIELD_GET(SYSCTRL_FRAME_CURRENT_MASK, ctrl_reg); + + ret = sysctrl_read_frame(sc, out, frame_size); + if (ret) + return ret; + + sysctrl_clear_response(sc); + + if (curr_frame == total_frames) + *done = true; + + return 0; +} + +static int sysctrl_receive_frames(struct xe_sysctrl *sc, + const struct xe_sysctrl_mailbox_msg_hdr *req, + void *data_out, size_t data_out_len, + size_t *rdata_len, unsigned int timeout_ms) +{ + struct xe_sysctrl_mailbox_msg_hdr *hdr; + struct xe_device *xe = sc_to_xe(sc); + size_t remain = sizeof(*hdr) + data_out_len; + u8 *buffer __free(kfree) = kzalloc(remain, GFP_KERNEL); + size_t frame_size; + bool done = false; + int ret = 0; + u8 *out; + + if (!buffer) + return -ENOMEM; + + out = buffer; + while (!done && remain) { + frame_size = min_t(size_t, remain, XE_SYSCTRL_MB_FRAME_SIZE); + + ret = sysctrl_process_frame(sc, out, frame_size, timeout_ms, + &done); + if (ret) + return ret; + + remain -= frame_size; + out += frame_size; + } + + hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; + + if (!XE_SYSCTRL_HDR_IS_RESPONSE(hdr) || + XE_SYSCTRL_HDR_GROUP_ID(hdr) != XE_SYSCTRL_HDR_GROUP_ID(req) || + XE_SYSCTRL_HDR_COMMAND(hdr) != XE_SYSCTRL_HDR_COMMAND(req)) { + xe_err(xe, "sysctrl: Response header mismatch\n"); + return -EPROTO; + } + + if (XE_SYSCTRL_HDR_RESULT(hdr) != 0) { + xe_err(xe, "sysctrl: Firmware error: 0x%02lx\n", + XE_SYSCTRL_HDR_RESULT(hdr)); + return -EIO; + } + + memcpy(data_out, hdr + 1, data_out_len); + *rdata_len = out - buffer - sizeof(*hdr); + + return 0; +} + +static int sysctrl_send_command(struct xe_sysctrl *sc, + const u8 *mbox_cmd, size_t cmd_size, + void *data_out, size_t data_out_len, + size_t *rdata_len, unsigned int timeout_ms) +{ + const struct xe_sysctrl_mailbox_msg_hdr *hdr; + size_t received; + int ret; + + ret = sysctrl_send_frames(sc, mbox_cmd, cmd_size, timeout_ms); + if (ret) + return ret; + + if (!data_out || !rdata_len) + return 0; + + hdr = (const struct xe_sysctrl_mailbox_msg_hdr *)mbox_cmd; + + ret = sysctrl_receive_frames(sc, hdr, data_out, data_out_len, + &received, timeout_ms); + if (ret) + return ret; + + *rdata_len = received; + + return 0; +} + +/** + * xe_sysctrl_create_command() - Create system controller command + * @command: Sysctrl command structure + * @group_id: Command group ID + * @cmd_id: Command ID + * @request: Pointer to request buffer (can be NULL) + * @request_len: Size of request buffer + * @response: Pointer to response buffer + * @response_len: Size of response buffer + * + * Helper function to create sysctrl command to be sent via %xe_sysctrl_send_command() + */ +void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id, + void *request, size_t request_len, void *response, + size_t response_len) +{ + struct xe_sysctrl_app_msg_hdr header = {0}; + + header.data = FIELD_PREP(APP_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(APP_HDR_COMMAND_MASK, cmd_id); + + command->header = header; + command->data_in = request; + command->data_in_len = request_len; + command->data_out = response; + command->data_out_len = response_len; +} + +/** + * xe_sysctrl_send_command() - Send mailbox command to System Controller + * @sc: System Controller instance + * @cmd: Command descriptor containing request header and payload buffers + * @rdata_len: Pointer to store actual response data length + * + * Sends a mailbox command to System Controller firmware using + * System Controller mailbox and waits for a response. + * + * Request payload is provided via @cmd->data_in and @cmd->data_in_len. + * If a response is expected, @cmd->data_out must point to a buffer of + * size @cmd->data_out_len supplied by caller. + * + * On success, @rdata_len is updated with number of valid response bytes + * returned by firmware, bounded by @cmd->data_out_len. + * + * Return: 0 on success, or negative errno on failure. + */ +int xe_sysctrl_send_command(struct xe_sysctrl *sc, + struct xe_sysctrl_mailbox_command *cmd, + size_t *rdata_len) +{ + struct xe_device *xe = sc_to_xe(sc); + u8 group_id, command_code; + u8 *mbox_cmd = NULL; + size_t cmd_size = 0; + int ret; + + guard(xe_pm_runtime_noresume)(xe); + + if (!xe->info.has_sysctrl) + return -ENODEV; + + xe_assert(xe, cmd->data_in || cmd->data_out); + xe_assert(xe, !cmd->data_in || cmd->data_in_len); + xe_assert(xe, !cmd->data_out || cmd->data_out_len); + + group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header); + command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header); + + might_sleep(); + + ret = sysctrl_prepare_command(xe, group_id, command_code, + cmd->data_in, cmd->data_in_len, + &mbox_cmd, &cmd_size); + if (ret) { + xe_err(xe, "sysctrl: Failed to prepare command: %pe\n", ERR_PTR(ret)); + return ret; + } + + guard(mutex)(&sc->cmd_lock); + + ret = sysctrl_send_command(sc, mbox_cmd, cmd_size, + cmd->data_out, cmd->data_out_len, rdata_len, + XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS); + if (ret) + xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret)); + + kfree(mbox_cmd); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h new file mode 100644 index 000000000..f0d5e3d7f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_MAILBOX_H_ +#define _XE_SYSCTRL_MAILBOX_H_ + +#include +#include + +#include "abi/xe_sysctrl_abi.h" + +struct xe_sysctrl; +struct xe_sysctrl_mailbox_command; + +#define XE_SYSCTRL_APP_HDR_GROUP_ID(hdr) \ + FIELD_GET(APP_HDR_GROUP_ID_MASK, (hdr)->data) + +#define XE_SYSCTRL_APP_HDR_COMMAND(hdr) \ + FIELD_GET(APP_HDR_COMMAND_MASK, (hdr)->data) + +#define XE_SYSCTRL_APP_HDR_VERSION(hdr) \ + FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data) + +void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id, + void *request, size_t request_len, void *response, + size_t response_len); +int xe_sysctrl_send_command(struct xe_sysctrl *sc, + struct xe_sysctrl_mailbox_command *cmd, + size_t *rdata_len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h new file mode 100644 index 000000000..d0341538a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_MAILBOX_TYPES_H_ +#define _XE_SYSCTRL_MAILBOX_TYPES_H_ + +#include + +#include "abi/xe_sysctrl_abi.h" + +/** + * enum xe_sysctrl_group - System Controller command groups + * + * @XE_SYSCTRL_GROUP_GFSP: GFSP group + */ +enum xe_sysctrl_group { + XE_SYSCTRL_GROUP_GFSP = 0x01, +}; + +/** + * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group + * + * @XE_SYSCTRL_CMD_GET_SOC_ERROR: Retrieve basic error information + * @XE_SYSCTRL_CMD_GET_COUNTER: Get error counter value + * @XE_SYSCTRL_CMD_CLEAR_COUNTER: Clear error counter value + * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event + * @XE_SYSCTRL_CMD_GET_HEALTH: Retrieve gpu health + * @XE_SYSCTRL_CMD_SET_HEALTH: Set gpu health + */ +enum xe_sysctrl_gfsp_cmd { + XE_SYSCTRL_CMD_GET_SOC_ERROR = 0x01, + XE_SYSCTRL_CMD_GET_COUNTER = 0x03, + XE_SYSCTRL_CMD_CLEAR_COUNTER = 0x04, + XE_SYSCTRL_CMD_GET_PENDING_EVENT = 0x07, + XE_SYSCTRL_CMD_GET_HEALTH = 0x0B, + XE_SYSCTRL_CMD_SET_HEALTH = 0x0C, +}; + +/** + * struct xe_sysctrl_mailbox_command - System Controller mailbox command + */ +struct xe_sysctrl_mailbox_command { + /** @header: Application message header containing command information */ + struct xe_sysctrl_app_msg_hdr header; + + /** @data_in: Pointer to input payload data (can be NULL if no input data) */ + void *data_in; + + /** @data_in_len: Size of input payload in bytes (0 if no input data) */ + size_t data_in_len; + + /** @data_out: Pointer to output buffer for response data (can be NULL if no response) */ + void *data_out; + + /** @data_out_len: Size of output buffer in bytes (0 if no response expected) */ + size_t data_out_len; +}; + +/* Modify as needed */ +#define XE_SYSCTRL_FLOOD_LIMIT 16 + +#define XE_SYSCTRL_MB_FRAME_SIZE 16 +#define XE_SYSCTRL_MB_MAX_FRAMES 64 +#define XE_SYSCTRL_MB_MAX_MESSAGE_SIZE \ + (XE_SYSCTRL_MB_FRAME_SIZE * XE_SYSCTRL_MB_MAX_FRAMES) + +#define XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS 500 + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_types.h b/drivers/gpu/drm/xe/xe_sysctrl_types.h new file mode 100644 index 000000000..98c2f473f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_types.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_TYPES_H_ +#define _XE_SYSCTRL_TYPES_H_ + +#include +#include +#include + +struct xe_mmio; + +/** + * struct xe_sysctrl - System Controller driver context + * + * This structure maintains the runtime state for System Controller + * communication. All fields are initialized during xe_sysctrl_init() + * and protected appropriately for concurrent access. + */ +struct xe_sysctrl { + /** @mmio: MMIO region for system control registers */ + struct xe_mmio *mmio; + + /** @cmd_lock: Mutex protecting mailbox command operations */ + struct mutex cmd_lock; + + /** @work: Pending events worker */ + struct work_struct work; + + /** @event_lock: Mutex protecting pending events */ + struct mutex event_lock; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c new file mode 100644 index 000000000..74d925a33 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include +#include + +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_ggtt.h" +#include "xe_memirq.h" +#include "xe_migrate.h" +#include "xe_pcode.h" +#include "xe_sa.h" +#include "xe_svm.h" +#include "xe_tile.h" +#include "xe_tile_sysfs.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vram.h" +#include "xe_vram_types.h" +#include "xe_wa.h" + +/** + * DOC: Multi-tile Design + * + * Different vendors use the term "tile" a bit differently, but in the Intel + * world, a 'tile' is pretty close to what most people would think of as being + * a complete GPU. When multiple GPUs are placed behind a single PCI device, + * that's what is referred to as a "multi-tile device." In such cases, pretty + * much all hardware is replicated per-tile, although certain responsibilities + * like PCI communication, reporting of interrupts to the OS, etc. are handled + * solely by the "root tile." A multi-tile platform takes care of tying the + * tiles together in a way such that interrupt notifications from remote tiles + * are forwarded to the root tile, the per-tile vram is combined into a single + * address space, etc. + * + * In contrast, a "GT" (which officially stands for "Graphics Technology") is + * the subset of a GPU/tile that is responsible for implementing graphics + * and/or media operations. The GT is where a lot of the driver implementation + * happens since it's where the hardware engines, the execution units, and the + * GuC all reside. + * + * Historically most Intel devices were single-tile devices that contained a + * single GT. PVC is an example of an Intel platform built on a multi-tile + * design (i.e., multiple GPUs behind a single PCI device); each PVC tile only + * has a single GT. In contrast, platforms like MTL that have separate chips + * for render and media IP are still only a single logical GPU, but the + * graphics and media IP blocks are each exposed as a separate GT within that + * single GPU. This is important from a software perspective because multi-GT + * platforms like MTL only replicate a subset of the GPU hardware and behave + * differently than multi-tile platforms like PVC where nearly everything is + * replicated. + * + * Per-tile functionality (shared by all GTs within the tile): + * - Complete 4MB MMIO space (containing SGunit/SoC registers, GT + * registers, display registers, etc.) + * - Global GTT + * - VRAM (if discrete) + * - Interrupt flows + * - Migration context + * - kernel batchbuffer pool + * - Primary GT + * - Media GT (if media version >= 13) + * + * Per-GT functionality: + * - GuC + * - Hardware engines + * - Programmable hardware units (subslices, EUs) + * - GSI subset of registers (multiple copies of these registers reside + * within the complete MMIO space provided by the tile, but at different + * offsets --- 0 for render, 0x380000 for media) + * - Multicast register steering + * - TLBs to cache page table translations + * - Reset capability + * - Low-level power management (e.g., C6) + * - Clock frequency + * - MOCS and PAT programming + */ + +/** + * xe_tile_alloc - Perform per-tile memory allocation + * @tile: Tile to perform allocations for + * + * Allocates various per-tile data structures using DRM-managed allocations. + * Does not touch the hardware. + * + * Returns -ENOMEM if allocations fail, otherwise 0. + */ +static int xe_tile_alloc(struct xe_tile *tile) +{ + tile->mem.ggtt = xe_ggtt_alloc(tile); + if (!tile->mem.ggtt) + return -ENOMEM; + + tile->migrate = xe_migrate_alloc(tile); + if (!tile->migrate) + return -ENOMEM; + + return 0; +} + +/** + * xe_tile_alloc_vram - Perform per-tile VRAM structs allocation + * @tile: Tile to perform allocations for + * + * Allocates VRAM per-tile data structures using DRM-managed allocations. + * Does not touch the hardware. + * + * Returns -ENOMEM if allocations fail, otherwise 0. + */ +int xe_tile_alloc_vram(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_vram_region *vram; + + if (!IS_DGFX(xe)) + return 0; + + vram = xe_vram_region_alloc(xe, tile->id, XE_PL_VRAM0 + tile->id); + if (!vram) + return -ENOMEM; + tile->mem.vram = vram; + + /* + * If the kernel_vram is not already allocated, + * it means that tile has common VRAM region for + * kernel and user space. + */ + if (!tile->mem.kernel_vram) + tile->mem.kernel_vram = tile->mem.vram; + + return 0; +} + +/** + * xe_tile_init_early - Initialize the tile and primary GT + * @tile: Tile to initialize + * @xe: Parent Xe device + * @id: Tile ID + * + * Initializes per-tile resources that don't require any interactions with the + * hardware or any knowledge about the Graphics/Media IP version. + * + * Returns: 0 on success, negative error code on error. + */ +int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id) +{ + int err; + + tile->xe = xe; + tile->id = id; + + err = xe_tile_alloc(tile); + if (err) + return err; + + err = xe_pcode_init_early(tile); + if (err) + return err; + + return 0; +} +ALLOW_ERROR_INJECTION(xe_tile_init_early, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_tile_init_noalloc - Init tile up to the point where allocations can happen. + * @tile: The tile to initialize. + * + * This function prepares the tile to allow memory allocations to VRAM, but is + * not allowed to allocate memory itself. This state is useful for display + * readout, because the inherited display framebuffer will otherwise be + * overwritten as it is usually put at the start of VRAM. + * + * Note that since this is tile initialization, it should not perform any + * GT-specific operations, and thus does not need to hold GT forcewake. + * + * Returns: 0 on success, negative error code on error. + */ +int xe_tile_init_noalloc(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + int err; + + xe_wa_apply_tile_workarounds(tile); + + err = xe_pagemap_cache_create(tile); + if (err) + return err; + + if (IS_DGFX(xe) && !ttm_resource_manager_used(&tile->mem.vram->ttm.manager)) { + err = xe_ttm_vram_mgr_init(xe, tile->mem.vram); + if (err) + return err; + + xe->info.mem_region_mask |= BIT(tile->mem.vram->id) << 1; + } + + return xe_tile_sysfs_init(tile); +} + +int xe_tile_init(struct xe_tile *tile) +{ + int err; + + err = xe_memirq_init(&tile->memirq); + if (err) + return err; + + tile->mem.kernel_bb_pool = xe_sa_bo_manager_init(tile, SZ_1M, 16); + if (IS_ERR(tile->mem.kernel_bb_pool)) + return PTR_ERR(tile->mem.kernel_bb_pool); + + /* Optimistically anticipate at most 256 TLB fences with PRL */ + tile->mem.reclaim_pool = xe_sa_bo_manager_init(tile, SZ_1M, XE_PAGE_RECLAIM_LIST_MAX_SIZE); + if (IS_ERR(tile->mem.reclaim_pool)) + return PTR_ERR(tile->mem.reclaim_pool); + + return 0; +} +void xe_tile_migrate_wait(struct xe_tile *tile) +{ + xe_migrate_wait(tile->migrate); +} + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) +/** + * xe_tile_local_pagemap() - Return a pointer to the tile's local drm_pagemap if any + * @tile: The tile. + * + * Return: A pointer to the tile's local drm_pagemap, or NULL if local pagemap + * support has been compiled out. + */ +struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) +{ + struct drm_pagemap *dpagemap = + drm_pagemap_get_from_cache_if_active(xe_tile_to_vr(tile)->dpagemap_cache); + + if (dpagemap) { + xe_assert(tile_to_xe(tile), kref_read(&dpagemap->ref) >= 2); + drm_pagemap_put(dpagemap); + } + + return dpagemap; +} +#endif + diff --git a/drivers/gpu/drm/xe/xe_tile.h b/drivers/gpu/drm/xe/xe_tile.h new file mode 100644 index 000000000..a2a2c0e93 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_TILE_H_ +#define _XE_TILE_H_ + +#include "xe_tile_types.h" + +struct xe_device; +struct xe_pagemap; + +int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id); +int xe_tile_init_noalloc(struct xe_tile *tile); +int xe_tile_init(struct xe_tile *tile); + +int xe_tile_alloc_vram(struct xe_tile *tile); + +void xe_tile_migrate_wait(struct xe_tile *tile); + +static inline bool xe_tile_is_root(struct xe_tile *tile) +{ + return tile->id == 0; +} + +/** + * xe_tile_to_vr() - Return the struct xe_vram_region pointer from a + * struct xe_tile pointer + * @tile: Pointer to the struct xe_tile. + * + * Return: Pointer to the struct xe_vram_region embedded in *@tile. + */ +static inline struct xe_vram_region *xe_tile_to_vr(struct xe_tile *tile) +{ + return tile->mem.vram; +} + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) +struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile); +#else +static inline struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) +{ + return NULL; +} +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_debugfs.c b/drivers/gpu/drm/xe/xe_tile_debugfs.c new file mode 100644 index 000000000..5df2f461b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_debugfs.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "xe_ggtt.h" +#include "xe_pm.h" +#include "xe_sa.h" +#include "xe_tile_debugfs.h" + +static struct xe_tile *node_to_tile(struct drm_info_node *node) +{ + return node->dent->d_parent->d_inode->i_private; +} + +/** + * xe_tile_debugfs_simple_show() - A show callback for struct drm_info_list + * @m: the &seq_file + * @data: data used by the drm debugfs helpers + * + * This callback can be used in struct drm_info_list to describe debugfs + * files that are &xe_tile specific. + * + * It is assumed that those debugfs files will be created on directory entry + * which struct dentry d_inode->i_private points to &xe_tile. + * + * /sys/kernel/debug/dri/0/ + * ├── tile0/ # tile = dentry->d_inode->i_private + * │ │ ├── id # tile = dentry->d_parent->d_inode->i_private + * + * This function assumes that &m->private will be set to the &struct + * drm_info_node corresponding to the instance of the info on a given &struct + * drm_minor (see struct drm_info_list.show for details). + * + * This function also assumes that struct drm_info_list.data will point to the + * function code that will actually print a file content:: + * + * int (*print)(struct xe_tile *, struct drm_printer *) + * + * Example:: + * + * int tile_id(struct xe_tile *tile, struct drm_printer *p) + * { + * drm_printf(p, "%u\n", tile->id); + * return 0; + * } + * + * static const struct drm_info_list info[] = { + * { name = "id", .show = tile_debugfs_simple_show, .data = tile_id }, + * }; + * + * dir = debugfs_create_dir("tile0", parent); + * dir->d_inode->i_private = tile; + * drm_debugfs_create_files(info, ARRAY_SIZE(info), dir, minor); + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_tile_debugfs_simple_show(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + struct drm_info_node *node = m->private; + struct xe_tile *tile = node_to_tile(node); + int (*print)(struct xe_tile *, struct drm_printer *) = node->info_ent->data; + + return print(tile, &p); +} + +/** + * xe_tile_debugfs_show_with_rpm() - A show callback for struct drm_info_list + * @m: the &seq_file + * @data: data used by the drm debugfs helpers + * + * Similar to tile_debugfs_simple_show() but implicitly takes a RPM ref. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_tile_debugfs_show_with_rpm(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct xe_tile *tile = node_to_tile(node); + struct xe_device *xe = tile_to_xe(tile); + + guard(xe_pm_runtime)(xe); + return xe_tile_debugfs_simple_show(m, data); +} + +static int ggtt(struct xe_tile *tile, struct drm_printer *p) +{ + return xe_ggtt_dump(tile->mem.ggtt, p); +} + +static int sa_info(struct xe_tile *tile, struct drm_printer *p) +{ + drm_suballoc_dump_debug_info(&tile->mem.kernel_bb_pool->base, p, + xe_sa_manager_gpu_addr(tile->mem.kernel_bb_pool)); + + return 0; +} + +/* only for debugfs files which can be safely used on the VF */ +static const struct drm_info_list vf_safe_debugfs_list[] = { + { "ggtt", .show = xe_tile_debugfs_show_with_rpm, .data = ggtt }, + { "sa_info", .show = xe_tile_debugfs_show_with_rpm, .data = sa_info }, +}; + +static void tile_debugfs_create_vram_mm(struct xe_tile *tile) +{ + if (tile->mem.vram) + ttm_resource_manager_create_debugfs(&tile->mem.vram->ttm.manager, tile->debugfs, + "vram_mm"); +} + +/** + * xe_tile_debugfs_register - Register tile's debugfs attributes + * @tile: the &xe_tile to register + * + * Create debugfs sub-directory with a name that includes a tile ID and + * then creates set of debugfs files (attributes) specific to this tile. + */ +void xe_tile_debugfs_register(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct drm_minor *minor = xe->drm.primary; + struct dentry *root = minor->debugfs_root; + char name[8]; + + snprintf(name, sizeof(name), "tile%u", tile->id); + tile->debugfs = debugfs_create_dir(name, root); + if (IS_ERR(tile->debugfs)) + return; + + /* + * Store the xe_tile pointer as private data of the tile/ directory + * node so other tile specific attributes under that directory may + * refer to it by looking at its parent node private data. + */ + tile->debugfs->d_inode->i_private = tile; + + drm_debugfs_create_files(vf_safe_debugfs_list, + ARRAY_SIZE(vf_safe_debugfs_list), + tile->debugfs, minor); + + tile_debugfs_create_vram_mm(tile); +} diff --git a/drivers/gpu/drm/xe/xe_tile_debugfs.h b/drivers/gpu/drm/xe/xe_tile_debugfs.h new file mode 100644 index 000000000..4429c2254 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_debugfs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TILE_DEBUGFS_H_ +#define _XE_TILE_DEBUGFS_H_ + +struct seq_file; +struct xe_tile; + +void xe_tile_debugfs_register(struct xe_tile *tile); +int xe_tile_debugfs_simple_show(struct seq_file *m, void *data); +int xe_tile_debugfs_show_with_rpm(struct seq_file *m, void *data); + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_printk.h b/drivers/gpu/drm/xe/xe_tile_printk.h new file mode 100644 index 000000000..738433a76 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_printk.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TILE_PRINTK_H_ +#define _XE_TILE_PRINTK_H_ + +#include "xe_printk.h" + +#define __XE_TILE_PRINTK_FMT(_tile, _fmt, _args...) "Tile%u: " _fmt, (_tile)->id, ##_args + +#define xe_tile_printk(_tile, _level, _fmt, ...) \ + xe_printk((_tile)->xe, _level, __XE_TILE_PRINTK_FMT((_tile), _fmt, ##__VA_ARGS__)) + +#define xe_tile_err(_tile, _fmt, ...) \ + xe_tile_printk((_tile), err, _fmt, ##__VA_ARGS__) + +#define xe_tile_err_once(_tile, _fmt, ...) \ + xe_tile_printk((_tile), err_once, _fmt, ##__VA_ARGS__) + +#define xe_tile_err_ratelimited(_tile, _fmt, ...) \ + xe_tile_printk((_tile), err_ratelimited, _fmt, ##__VA_ARGS__) + +#define xe_tile_warn(_tile, _fmt, ...) \ + xe_tile_printk((_tile), warn, _fmt, ##__VA_ARGS__) + +#define xe_tile_notice(_tile, _fmt, ...) \ + xe_tile_printk((_tile), notice, _fmt, ##__VA_ARGS__) + +#define xe_tile_info(_tile, _fmt, ...) \ + xe_tile_printk((_tile), info, _fmt, ##__VA_ARGS__) + +#define xe_tile_dbg(_tile, _fmt, ...) \ + xe_tile_printk((_tile), dbg, _fmt, ##__VA_ARGS__) + +#define xe_tile_WARN_type(_tile, _type, _condition, _fmt, ...) \ + xe_WARN##_type((_tile)->xe, _condition, _fmt, ## __VA_ARGS__) + +#define xe_tile_WARN(_tile, _condition, _fmt, ...) \ + xe_tile_WARN_type((_tile),, _condition, __XE_TILE_PRINTK_FMT((_tile), _fmt, ##__VA_ARGS__)) + +#define xe_tile_WARN_ONCE(_tile, _condition, _fmt, ...) \ + xe_tile_WARN_type((_tile), _ONCE, _condition, __XE_TILE_PRINTK_FMT((_tile), _fmt, ##__VA_ARGS__)) + +#define xe_tile_WARN_ON(_tile, _condition) \ + xe_tile_WARN((_tile), _condition, "%s(%s)", "WARN_ON", __stringify(_condition)) + +#define xe_tile_WARN_ON_ONCE(_tile, _condition) \ + xe_tile_WARN_ONCE((_tile), _condition, "%s(%s)", "WARN_ON_ONCE", __stringify(_condition)) + +static inline void __xe_tile_printfn_err(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_tile *tile = p->arg; + + xe_tile_err(tile, "%pV", vaf); +} + +static inline void __xe_tile_printfn_info(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_tile *tile = p->arg; + + xe_tile_info(tile, "%pV", vaf); +} + +static inline void __xe_tile_printfn_dbg(struct drm_printer *p, struct va_format *vaf) +{ + struct xe_tile *tile = p->arg; + struct drm_printer dbg; + + /* + * The original xe_tile_dbg() callsite annotations are useless here, + * redirect to the tweaked xe_dbg_printer() instead. + */ + dbg = xe_dbg_printer(tile->xe); + dbg.origin = p->origin; + + drm_printf(&dbg, __XE_TILE_PRINTK_FMT(tile, "%pV", vaf)); +} + +/** + * xe_tile_err_printer - Construct a &drm_printer that outputs to xe_tile_err() + * @tile: the &xe_tile pointer to use in xe_tile_err() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_tile_err_printer(struct xe_tile *tile) +{ + struct drm_printer p = { + .printfn = __xe_tile_printfn_err, + .arg = tile, + }; + return p; +} + +/** + * xe_tile_info_printer - Construct a &drm_printer that outputs to xe_tile_info() + * @tile: the &xe_tile pointer to use in xe_tile_info() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_tile_info_printer(struct xe_tile *tile) +{ + struct drm_printer p = { + .printfn = __xe_tile_printfn_info, + .arg = tile, + }; + return p; +} + +/** + * xe_tile_dbg_printer - Construct a &drm_printer that outputs like xe_tile_dbg() + * @tile: the &xe_tile pointer to use in xe_tile_dbg() + * + * Return: The &drm_printer object. + */ +static inline struct drm_printer xe_tile_dbg_printer(struct xe_tile *tile) +{ + struct drm_printer p = { + .printfn = __xe_tile_printfn_dbg, + .arg = tile, + .origin = (const void *)_THIS_IP_, + }; + return p; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c new file mode 100644 index 000000000..7f97db2f8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_debugfs.h" +#include "xe_pm.h" +#include "xe_tile_debugfs.h" +#include "xe_tile_sriov_pf_debugfs.h" +#include "xe_sriov.h" +#include "xe_sriov_pf.h" +#include "xe_sriov_pf_provision.h" + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov # d_inode->i_private = (xe_device*) + * │ ├── pf # d_inode->i_private = (xe_device*) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ ├── tile1 + * │ │ : : + * │ ├── vf1 # d_inode->i_private = VFID(1) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ ├── tile1 + * │ │ : : + * │ ├── vfN # d_inode->i_private = VFID(N) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ ├── tile1 + * : : : : + */ + +static void *extract_priv(struct dentry *d) +{ + return d->d_inode->i_private; +} + +__maybe_unused +static struct xe_tile *extract_tile(struct dentry *d) +{ + return extract_priv(d); +} + +static struct xe_device *extract_xe(struct dentry *d) +{ + return extract_priv(d->d_parent->d_parent); +} + +__maybe_unused +static unsigned int extract_vfid(struct dentry *d) +{ + void *pp = extract_priv(d->d_parent); + + return pp == extract_xe(d) ? PFID : (uintptr_t)pp; +} + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * : ├── tile0 + * : ├── ggtt_available + * ├── ggtt_provisioned + */ + +static int pf_config_print_available_ggtt(struct xe_tile *tile, struct drm_printer *p) +{ + return xe_gt_sriov_pf_config_print_available_ggtt(tile->primary_gt, p); +} + +static int pf_config_print_ggtt(struct xe_tile *tile, struct drm_printer *p) +{ + return xe_gt_sriov_pf_config_print_ggtt(tile->primary_gt, p); +} + +static const struct drm_info_list pf_ggtt_info[] = { + { + "ggtt_available", + .show = xe_tile_debugfs_simple_show, + .data = pf_config_print_available_ggtt, + }, + { + "ggtt_provisioned", + .show = xe_tile_debugfs_simple_show, + .data = pf_config_print_ggtt, + }, +}; + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * : ├── pf + * : ├── tile0 + * : ├── vram_provisioned + */ + +static int pf_config_print_vram(struct xe_tile *tile, struct drm_printer *p) +{ + return xe_gt_sriov_pf_config_print_lmem(tile->primary_gt, p); +} + +static const struct drm_info_list pf_vram_info[] = { + { + "vram_provisioned", + .show = xe_tile_debugfs_simple_show, + .data = pf_config_print_vram, + }, +}; + +/* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * │ ├── pf + * │ │ ├── tile0 + * │ │ │ ├── ggtt_spare + * │ │ │ ├── vram_spare + * │ │ ├── tile1 + * │ │ : : + * │ ├── vf1 + * │ : ├── tile0 + * │ │ ├── ggtt_quota + * │ │ ├── vram_quota + * │ ├── tile1 + * │ : : + */ + +#define DEFINE_SRIOV_TILE_CONFIG_DEBUGFS_ATTRIBUTE(NAME, CONFIG, TYPE, FORMAT) \ + \ +static int NAME##_set(void *data, u64 val) \ +{ \ + struct xe_tile *tile = extract_tile(data); \ + unsigned int vfid = extract_vfid(data); \ + struct xe_gt *gt = tile->primary_gt; \ + struct xe_device *xe = tile->xe; \ + int err; \ + \ + if (val > (TYPE)~0ull) \ + return -EOVERFLOW; \ + \ + guard(xe_pm_runtime)(xe); \ + err = xe_sriov_pf_wait_ready(xe) ?: \ + xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ + if (!err) \ + xe_sriov_pf_provision_set_custom_mode(xe); \ + \ + return err; \ +} \ + \ +static int NAME##_get(void *data, u64 *val) \ +{ \ + struct xe_tile *tile = extract_tile(data); \ + unsigned int vfid = extract_vfid(data); \ + struct xe_gt *gt = tile->primary_gt; \ + \ + *val = xe_gt_sriov_pf_config_get_##CONFIG(gt, vfid); \ + return 0; \ +} \ + \ +DEFINE_DEBUGFS_ATTRIBUTE(NAME##_fops, NAME##_get, NAME##_set, FORMAT) + +DEFINE_SRIOV_TILE_CONFIG_DEBUGFS_ATTRIBUTE(ggtt, ggtt, u64, "%llu\n"); +DEFINE_SRIOV_TILE_CONFIG_DEBUGFS_ATTRIBUTE(vram, lmem, u64, "%llu\n"); + +static void pf_add_config_attrs(struct xe_tile *tile, struct dentry *dent, unsigned int vfid) +{ + struct xe_device *xe = tile->xe; + + xe_tile_assert(tile, tile == extract_tile(dent)); + xe_tile_assert(tile, vfid == extract_vfid(dent)); + + debugfs_create_file_unsafe(vfid ? "ggtt_quota" : "ggtt_spare", + 0644, dent, dent, &ggtt_fops); + if (IS_DGFX(xe)) + debugfs_create_file_unsafe(vfid ? "vram_quota" : "vram_spare", + xe_device_has_lmtt(xe) ? 0644 : 0444, + dent, dent, &vram_fops); +} + +static void pf_populate_tile(struct xe_tile *tile, struct dentry *dent, unsigned int vfid) +{ + struct xe_device *xe = tile->xe; + struct drm_minor *minor = xe->drm.primary; + struct xe_gt *gt; + unsigned int id; + + pf_add_config_attrs(tile, dent, vfid); + + if (!vfid) { + drm_debugfs_create_files(pf_ggtt_info, + ARRAY_SIZE(pf_ggtt_info), + dent, minor); + if (IS_DGFX(xe)) + drm_debugfs_create_files(pf_vram_info, + ARRAY_SIZE(pf_vram_info), + dent, minor); + } + + for_each_gt_on_tile(gt, tile, id) + xe_gt_sriov_pf_debugfs_populate(gt, dent, vfid); +} + +/** + * xe_tile_sriov_pf_debugfs_populate() - Populate SR-IOV debugfs tree with tile files. + * @tile: the &xe_tile to register + * @parent: the parent &dentry that represents the SR-IOV @vfid function + * @vfid: the VF identifier + * + * Add to the @parent directory new debugfs directory that will represent a @tile and + * populate it with files that are related to the SR-IOV @vfid function. + * + * This function can only be called on PF. + */ +void xe_tile_sriov_pf_debugfs_populate(struct xe_tile *tile, struct dentry *parent, + unsigned int vfid) +{ + struct xe_device *xe = tile->xe; + struct dentry *dent; + char name[10]; /* should be enough up to "tile%u\0" for 2^16 - 1 */ + + xe_tile_assert(tile, IS_SRIOV_PF(xe)); + xe_tile_assert(tile, extract_priv(parent->d_parent) == xe); + xe_tile_assert(tile, extract_priv(parent) == tile->xe || + (uintptr_t)extract_priv(parent) == vfid); + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── sriov + * │ ├── pf # parent, d_inode->i_private = (xe_device*) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ ├── tile1 + * │ │ : : + * │ ├── vf1 # parent, d_inode->i_private = VFID(1) + * │ │ ├── tile0 # d_inode->i_private = (xe_tile*) + * │ │ ├── tile1 + * : : : : + */ + snprintf(name, sizeof(name), "tile%u", tile->id); + dent = debugfs_create_dir(name, parent); + if (IS_ERR(dent)) + return; + dent->d_inode->i_private = tile; + + xe_tile_assert(tile, extract_tile(dent) == tile); + xe_tile_assert(tile, extract_vfid(dent) == vfid); + xe_tile_assert(tile, extract_xe(dent) == xe); + + pf_populate_tile(tile, dent, vfid); +} diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.h b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.h new file mode 100644 index 000000000..55d179c44 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TILE_SRIOV_PF_DEBUGFS_H_ +#define _XE_TILE_SRIOV_PF_DEBUGFS_H_ + +struct dentry; +struct xe_tile; + +void xe_tile_sriov_pf_debugfs_populate(struct xe_tile *tile, struct dentry *parent, + unsigned int vfid); + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_printk.h b/drivers/gpu/drm/xe/xe_tile_sriov_printk.h new file mode 100644 index 000000000..683235128 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sriov_printk.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TILE_SRIOV_PRINTK_H_ +#define _XE_TILE_SRIOV_PRINTK_H_ + +#include "xe_tile_printk.h" +#include "xe_sriov_printk.h" + +#define __XE_TILE_SRIOV_PRINTK_FMT(_tile, _fmt, ...) \ + __XE_TILE_PRINTK_FMT((_tile), _fmt, ##__VA_ARGS__) + +#define xe_tile_sriov_printk(_tile, _level, _fmt, ...) \ + xe_sriov_##_level((_tile)->xe, __XE_TILE_SRIOV_PRINTK_FMT((_tile), _fmt, ##__VA_ARGS__)) + +#define xe_tile_sriov_err(_tile, _fmt, ...) \ + xe_tile_sriov_printk(_tile, err, _fmt, ##__VA_ARGS__) + +#define xe_tile_sriov_notice(_tile, _fmt, ...) \ + xe_tile_sriov_printk(_tile, notice, _fmt, ##__VA_ARGS__) + +#define xe_tile_sriov_info(_tile, _fmt, ...) \ + xe_tile_sriov_printk(_tile, info, _fmt, ##__VA_ARGS__) + +#define xe_tile_sriov_dbg(_tile, _fmt, ...) \ + xe_tile_sriov_printk(_tile, dbg, _fmt, ##__VA_ARGS__) + +#define xe_tile_sriov_dbg_verbose(_tile, _fmt, ...) \ + xe_tile_sriov_printk(_tile, dbg_verbose, _fmt, ##__VA_ARGS__) + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_vf.c b/drivers/gpu/drm/xe/xe_tile_sriov_vf.c new file mode 100644 index 000000000..24293521e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sriov_vf.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "regs/xe_gtt_defs.h" + +#include "xe_assert.h" +#include "xe_ggtt.h" +#include "xe_sriov.h" +#include "xe_sriov_printk.h" +#include "xe_tile_sriov_vf.h" +#include "xe_wopcm.h" + +/** + * DOC: GGTT nodes shifting during VF post-migration recovery + * + * The first fixup applied to the VF KMD structures as part of post-migration + * recovery is shifting nodes within &xe_ggtt instance. The nodes are moved + * from range previously assigned to this VF, into newly provisioned area. + * + * Below is a GGTT layout of example VF, with a certain address range assigned to + * said VF, and inaccessible areas above and below: + * + * 0 4GiB + * |<--------------------------- Total GGTT size ----------------------------->| + * WOPCM GUC_TOP + * |<-------------- Area mappable by xe_ggtt instance ---------------->| + * + * +---+---------------------------------+----------+----------------------+---+ + * |\\\|/////////////////////////////////| VF mem |//////////////////////|\\\| + * +---+---------------------------------+----------+----------------------+---+ + * + * Hardware enforced access rules before migration: + * + * |<------- inaccessible for VF ------->||<-- inaccessible for VF ->| + * + * After the migration, GGTT area assigned to the VF might have shifted, either + * to lower or to higher address. But we expect the total size and extra areas to + * be identical, as migration can only happen between matching platforms. + * Below is an example of GGTT layout of the VF after migration. Content of the + * GGTT for VF has been moved to a new area, and we receive its address from GuC: + * + * +---+----------------------+----------+---------------------------------+---+ + * |\\\|//////////////////////| VF mem |/////////////////////////////////|\\\| + * +---+----------------------+----------+---------------------------------+---+ + * + * Hardware enforced access rules after migration: + * + * |<- inaccessible for VF -->||<------- inaccessible for VF ------->| + * + * So the VF has a new slice of GGTT assigned, and during migration process, the + * memory content was copied to that new area. But the &xe_ggtt nodes are still + * tracking allocations using the old addresses. The nodes within VF owned area + * have to be shifted, and the start offset for GGTT adjusted. + * + * Due to use of GPU profiles, we do not expect the old and new GGTT areas to + * overlap; but our node shifting will fix addresses properly regardless. + */ + +/** + * xe_tile_sriov_vf_lmem - VF LMEM configuration. + * @tile: the &xe_tile + * + * This function is for VF use only. + * + * Return: size of the LMEM assigned to VF. + */ +u64 xe_tile_sriov_vf_lmem(struct xe_tile *tile) +{ + struct xe_tile_sriov_vf_selfconfig *config = &tile->sriov.vf.self_config; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + + return config->lmem_size; +} + +/** + * xe_tile_sriov_vf_lmem_store - Store VF LMEM configuration + * @tile: the &xe_tile + * @lmem_size: VF LMEM size to store + * + * This function is for VF use only. + */ +void xe_tile_sriov_vf_lmem_store(struct xe_tile *tile, u64 lmem_size) +{ + struct xe_tile_sriov_vf_selfconfig *config = &tile->sriov.vf.self_config; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + + config->lmem_size = lmem_size; +} + +/** + * xe_tile_sriov_vf_ggtt - VF GGTT configuration. + * @tile: the &xe_tile + * + * This function is for VF use only. + * + * Return: size of the GGTT assigned to VF. + */ +u64 xe_tile_sriov_vf_ggtt(struct xe_tile *tile) +{ + struct xe_tile_sriov_vf_selfconfig *config = &tile->sriov.vf.self_config; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + + return config->ggtt_size; +} + +/** + * xe_tile_sriov_vf_ggtt_store - Store VF GGTT configuration + * @tile: the &xe_tile + * @ggtt_size: VF GGTT size to store + * + * This function is for VF use only. + */ +void xe_tile_sriov_vf_ggtt_store(struct xe_tile *tile, u64 ggtt_size) +{ + struct xe_tile_sriov_vf_selfconfig *config = &tile->sriov.vf.self_config; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + + config->ggtt_size = ggtt_size; +} + +/** + * xe_tile_sriov_vf_ggtt_base - VF GGTT base configuration. + * @tile: the &xe_tile + * + * This function is for VF use only. + * + * Return: base of the GGTT assigned to VF. + */ +u64 xe_tile_sriov_vf_ggtt_base(struct xe_tile *tile) +{ + struct xe_tile_sriov_vf_selfconfig *config = &tile->sriov.vf.self_config; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + + return READ_ONCE(config->ggtt_base); +} + +/** + * xe_tile_sriov_vf_ggtt_base_store - Store VF GGTT base configuration + * @tile: the &xe_tile + * @ggtt_base: VF GGTT base to store + * + * This function is for VF use only. + */ +void xe_tile_sriov_vf_ggtt_base_store(struct xe_tile *tile, u64 ggtt_base) +{ + struct xe_tile_sriov_vf_selfconfig *config = &tile->sriov.vf.self_config; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + + WRITE_ONCE(config->ggtt_base, ggtt_base); +} diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_vf.h b/drivers/gpu/drm/xe/xe_tile_sriov_vf.h new file mode 100644 index 000000000..f2bbc4fc5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sriov_vf.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TILE_SRIOV_VF_H_ +#define _XE_TILE_SRIOV_VF_H_ + +#include + +struct xe_tile; + +u64 xe_tile_sriov_vf_ggtt(struct xe_tile *tile); +void xe_tile_sriov_vf_ggtt_store(struct xe_tile *tile, u64 ggtt_size); +u64 xe_tile_sriov_vf_ggtt_base(struct xe_tile *tile); +void xe_tile_sriov_vf_ggtt_base_store(struct xe_tile *tile, u64 ggtt_size); +u64 xe_tile_sriov_vf_lmem(struct xe_tile *tile); +void xe_tile_sriov_vf_lmem_store(struct xe_tile *tile, u64 lmem_size); + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_tile_sriov_vf_types.h new file mode 100644 index 000000000..4807ca516 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sriov_vf_types.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TILE_SRIOV_VF_TYPES_H_ +#define _XE_TILE_SRIOV_VF_TYPES_H_ + +#include + +/** + * struct xe_tile_sriov_vf_selfconfig - VF configuration data. + */ +struct xe_tile_sriov_vf_selfconfig { + /** @ggtt_base: assigned base offset of the GGTT region. */ + u64 ggtt_base; + /** @ggtt_size: assigned size of the GGTT region. */ + u64 ggtt_size; + /** @lmem_size: assigned size of the LMEM. */ + u64 lmem_size; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_sysfs.c b/drivers/gpu/drm/xe/xe_tile_sysfs.c new file mode 100644 index 000000000..510aa0ac4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sysfs.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_pm.h" +#include "xe_tile_sysfs.h" +#include "xe_vram_freq.h" + +static void xe_tile_sysfs_kobj_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static const struct kobj_type xe_tile_sysfs_kobj_type = { + .release = xe_tile_sysfs_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, +}; + +static void tile_sysfs_fini(void *arg) +{ + struct xe_tile *tile = arg; + + kobject_put(tile->sysfs); +} + +int xe_tile_sysfs_init(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct device *dev = xe->drm.dev; + struct kobj_tile *kt; + int err; + + kt = kzalloc_obj(*kt); + if (!kt) + return -ENOMEM; + + kobject_init(&kt->base, &xe_tile_sysfs_kobj_type); + kt->tile = tile; + + err = kobject_add(&kt->base, &dev->kobj, "tile%d", tile->id); + if (err) + goto err_object; + + tile->sysfs = &kt->base; + + err = xe_vram_freq_sysfs_init(tile); + if (err) + goto err_object; + + return devm_add_action_or_reset(xe->drm.dev, tile_sysfs_fini, tile); + +err_object: + kobject_put(&kt->base); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_tile_sysfs.h b/drivers/gpu/drm/xe/xe_tile_sysfs.h new file mode 100644 index 000000000..54a2ba8ba --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sysfs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_TILE_SYSFS_H_ +#define _XE_TILE_SYSFS_H_ + +#include "xe_tile_sysfs_types.h" + +int xe_tile_sysfs_init(struct xe_tile *tile); + +static inline struct xe_tile * +kobj_to_tile(struct kobject *kobj) +{ + return container_of(kobj, struct kobj_tile, base)->tile; +} + +#endif /* _XE_TILE_SYSFS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_tile_sysfs_types.h b/drivers/gpu/drm/xe/xe_tile_sysfs_types.h new file mode 100644 index 000000000..75906ba11 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_sysfs_types.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_TILE_SYSFS_TYPES_H_ +#define _XE_TILE_SYSFS_TYPES_H_ + +#include + +struct xe_tile; + +/** + * struct kobj_tile - A tile's kobject struct that connects the kobject + * and the TILE + * + * When dealing with multiple TILEs, this struct helps to understand which + * TILE needs to be addressed on a given sysfs call. + */ +struct kobj_tile { + /** @base: The actual kobject */ + struct kobject base; + /** @tile: A pointer to the tile itself */ + struct xe_tile *tile; +}; + +#endif /* _XE_TILE_SYSFS_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h new file mode 100644 index 000000000..0048100cc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tile_types.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022-2026 Intel Corporation + */ + +#ifndef _XE_TILE_TYPES_H_ +#define _XE_TILE_TYPES_H_ + +#include +#include + +#include "xe_lmtt_types.h" +#include "xe_memirq_types.h" +#include "xe_mert.h" +#include "xe_mmio_types.h" +#include "xe_tile_sriov_vf_types.h" + +#define tile_to_xe(tile__) \ + _Generic(tile__, \ + const struct xe_tile * : (const struct xe_device *)((tile__)->xe), \ + struct xe_tile * : (tile__)->xe) + +/** + * struct xe_tile - hardware tile structure + * + * From a driver perspective, a "tile" is effectively a complete GPU, containing + * an SGunit, 1-2 GTs, and (for discrete platforms) VRAM. + * + * Multi-tile platforms effectively bundle multiple GPUs behind a single PCI + * device and designate one "root" tile as being responsible for external PCI + * communication. PCI BAR0 exposes the GGTT and MMIO register space for each + * tile in a stacked layout, and PCI BAR2 exposes the local memory associated + * with each tile similarly. Device-wide interrupts can be enabled/disabled + * at the root tile, and the MSTR_TILE_INTR register will report which tiles + * have interrupts that need servicing. + */ +struct xe_tile { + /** @xe: Backpointer to tile's PCI device */ + struct xe_device *xe; + + /** @id: ID of the tile */ + u8 id; + + /** + * @primary_gt: Primary GT + */ + struct xe_gt *primary_gt; + + /** + * @media_gt: Media GT + * + * Only present on devices with media version >= 13. + */ + struct xe_gt *media_gt; + + /** + * @mmio: MMIO info for a tile. + * + * Each tile has its own 16MB space in BAR0, laid out as: + * * 0-4MB: registers + * * 4MB-8MB: reserved + * * 8MB-16MB: global GTT + */ + struct xe_mmio mmio; + + /** @mem: memory management info for tile */ + struct { + /** + * @mem.kernel_vram: kernel-dedicated VRAM info for tile. + * + * Although VRAM is associated with a specific tile, it can + * still be accessed by all tiles' GTs. + */ + struct xe_vram_region *kernel_vram; + + /** + * @mem.vram: general purpose VRAM info for tile. + * + * Although VRAM is associated with a specific tile, it can + * still be accessed by all tiles' GTs. + */ + struct xe_vram_region *vram; + + /** @mem.ggtt: Global graphics translation table */ + struct xe_ggtt *ggtt; + + /** + * @mem.kernel_bb_pool: Pool from which batchbuffers are allocated. + * + * Media GT shares a pool with its primary GT. + */ + struct xe_sa_manager *kernel_bb_pool; + + /** + * @mem.reclaim_pool: Pool for PRLs allocated. + * + * Only main GT has page reclaim list allocations. + */ + struct xe_sa_manager *reclaim_pool; + } mem; + + /** @sriov: tile level virtualization data */ + union { + struct { + /** @sriov.pf.lmtt: Local Memory Translation Table. */ + struct xe_lmtt lmtt; + } pf; + struct { + /** @sriov.vf.self_config: VF configuration data */ + struct xe_tile_sriov_vf_selfconfig self_config; + } vf; + } sriov; + + /** @memirq: Memory Based Interrupts. */ + struct xe_memirq memirq; + + /** @csc_hw_error_work: worker to report CSC HW errors */ + struct work_struct csc_hw_error_work; + + /** @pcode: tile's PCODE */ + struct { + /** @pcode.lock: protecting tile's PCODE mailbox data */ + struct mutex lock; + } pcode; + + /** @migrate: Migration helper for vram blits and clearing */ + struct xe_migrate *migrate; + + /** @sysfs: sysfs' kobj used by xe_tile_sysfs */ + struct kobject *sysfs; + + /** @debugfs: debugfs directory associated with this tile */ + struct dentry *debugfs; + + /** @mert: MERT-related data */ + struct xe_mert mert; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c new file mode 100644 index 000000000..bbd21d393 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gt_stats.h" +#include "xe_gt_types.h" +#include "xe_guc_ct.h" +#include "xe_guc_tlb_inval.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_tlb_inval.h" +#include "xe_trace.h" + +/** + * DOC: Xe TLB invalidation + * + * Xe TLB invalidation is implemented in two layers. The first is the frontend + * API, which provides an interface for TLB invalidations to the driver code. + * The frontend handles seqno assignment, synchronization (fences), and the + * timeout mechanism. The frontend is implemented via an embedded structure + * xe_tlb_inval that includes a set of ops hooking into the backend. The backend + * interacts with the hardware (or firmware) to perform the actual invalidation. + */ + +#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS + +static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence) +{ + if (WARN_ON_ONCE(!fence->tlb_inval)) + return; + + xe_pm_runtime_put(fence->tlb_inval->xe); + fence->tlb_inval = NULL; /* fini() should be called once */ +} + +static void +xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence) +{ + struct xe_tlb_inval *tlb_inval = fence->tlb_inval; + bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags); + + lockdep_assert_held(&fence->tlb_inval->pending_lock); + + list_del(&fence->link); + if (list_empty(&tlb_inval->pending_fences)) + cancel_delayed_work(&tlb_inval->fence_tdr); + trace_xe_tlb_inval_fence_signal(fence->tlb_inval->xe, fence); + xe_tlb_inval_fence_fini(fence); + dma_fence_signal(&fence->base); + if (!stack) + dma_fence_put(&fence->base); +} + +static void +xe_tlb_inval_fence_signal_unlocked(struct xe_tlb_inval_fence *fence) +{ + struct xe_tlb_inval *tlb_inval = fence->tlb_inval; + + spin_lock_irq(&tlb_inval->pending_lock); + xe_tlb_inval_fence_signal(fence); + spin_unlock_irq(&tlb_inval->pending_lock); +} + +static void xe_tlb_inval_fence_timeout(struct work_struct *work) +{ + struct xe_tlb_inval *tlb_inval = container_of(work, struct xe_tlb_inval, + fence_tdr.work); + struct xe_device *xe = tlb_inval->xe; + struct xe_tlb_inval_fence *fence, *next; + long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval); + + tlb_inval->ops->flush(tlb_inval); + + spin_lock_irq(&tlb_inval->pending_lock); + list_for_each_entry_safe(fence, next, + &tlb_inval->pending_fences, link) { + s64 since_inval_ms = ktime_ms_delta(ktime_get(), + fence->inval_time); + + if (msecs_to_jiffies(since_inval_ms) < timeout_delay) + break; + + trace_xe_tlb_inval_fence_timeout(xe, fence); + drm_err(&xe->drm, + "TLB invalidation fence timeout, seqno=%d recv=%d", + fence->seqno, tlb_inval->seqno_recv); + + fence->base.error = -ETIME; + xe_tlb_inval_fence_signal(fence); + } + if (!list_empty(&tlb_inval->pending_fences)) + queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr, + timeout_delay); + spin_unlock_irq(&tlb_inval->pending_lock); +} + +/** + * tlb_inval_fini - Clean up TLB invalidation state + * @drm: @drm_device + * @arg: pointer to struct @xe_tlb_inval + * + * Cancel pending fence workers and clean up any additional + * TLB invalidation state. + */ +static void tlb_inval_fini(struct drm_device *drm, void *arg) +{ + struct xe_tlb_inval *tlb_inval = arg; + + xe_tlb_inval_reset(tlb_inval); +} + +static void primelockdep(struct xe_tlb_inval *tlb_inval) +{ + if (!IS_ENABLED(CONFIG_LOCKDEP)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&tlb_inval->seqno_lock); + fs_reclaim_release(GFP_KERNEL); +} + +/** + * xe_gt_tlb_inval_init_early() - Initialize TLB invalidation state + * @gt: GT structure + * + * Initialize TLB invalidation state, purely software initialization, should + * be called once during driver load. + * + * Return: 0 on success, negative error code on error. + */ +int xe_gt_tlb_inval_init_early(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_tlb_inval *tlb_inval = >->tlb_inval; + int err; + + tlb_inval->xe = xe; + tlb_inval->seqno = 1; + INIT_LIST_HEAD(&tlb_inval->pending_fences); + spin_lock_init(&tlb_inval->pending_lock); + spin_lock_init(&tlb_inval->lock); + INIT_DELAYED_WORK(&tlb_inval->fence_tdr, xe_tlb_inval_fence_timeout); + + err = drmm_mutex_init(&xe->drm, &tlb_inval->seqno_lock); + if (err) + return err; + + primelockdep(tlb_inval); + + tlb_inval->job_wq = drmm_alloc_ordered_workqueue(&xe->drm, + "gt-tbl-inval-job-wq", + WQ_MEM_RECLAIM); + if (IS_ERR(tlb_inval->job_wq)) + return PTR_ERR(tlb_inval->job_wq); + + tlb_inval->timeout_wq = gt->ordered_wq; + if (IS_ERR(tlb_inval->timeout_wq)) + return PTR_ERR(tlb_inval->timeout_wq); + + /* XXX: Blindly setting up backend to GuC */ + xe_guc_tlb_inval_init_early(>->uc.guc, tlb_inval); + + return drmm_add_action_or_reset(&xe->drm, tlb_inval_fini, tlb_inval); +} + +/** + * xe_tlb_inval_reset() - TLB invalidation reset + * @tlb_inval: TLB invalidation client + * + * Signal any pending invalidation fences, should be called during a GT reset + */ +void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval) +{ + struct xe_tlb_inval_fence *fence, *next; + int pending_seqno; + + /* + * we can get here before the backends are even initialized if we're + * wedging very early, in which case there are not going to be any + * pendind fences so we can bail immediately. + */ + if (!tlb_inval->ops->initialized(tlb_inval)) + return; + + /* + * Backend is already disabled at this point. No new TLB requests can + * appear. + */ + + mutex_lock(&tlb_inval->seqno_lock); + spin_lock_irq(&tlb_inval->pending_lock); + cancel_delayed_work(&tlb_inval->fence_tdr); + /* + * We might have various kworkers waiting for TLB flushes to complete + * which are not tracked with an explicit TLB fence, however at this + * stage that will never happen since the backend is already disabled, + * so make sure we signal them here under the assumption that we have + * completed a full GT reset. + */ + if (tlb_inval->seqno == 1) + pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1; + else + pending_seqno = tlb_inval->seqno - 1; + WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno); + + list_for_each_entry_safe(fence, next, + &tlb_inval->pending_fences, link) + xe_tlb_inval_fence_signal(fence); + spin_unlock_irq(&tlb_inval->pending_lock); + mutex_unlock(&tlb_inval->seqno_lock); +} + +/** + * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout + * @tlb_inval: TLB invalidation client + * + * Reset the TLB invalidation timeout timer. + */ +static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval) +{ + lockdep_assert_held(&tlb_inval->pending_lock); + + mod_delayed_work(system_wq, &tlb_inval->fence_tdr, + tlb_inval->ops->timeout_delay(tlb_inval)); +} + +static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno) +{ + int seqno_recv = READ_ONCE(tlb_inval->seqno_recv); + + lockdep_assert_held(&tlb_inval->pending_lock); + + if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2)) + return false; + + if (seqno - seqno_recv > (TLB_INVALIDATION_SEQNO_MAX / 2)) + return true; + + return seqno_recv >= seqno; +} + +static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence) +{ + struct xe_tlb_inval *tlb_inval = fence->tlb_inval; + + fence->seqno = tlb_inval->seqno; + trace_xe_tlb_inval_fence_send(tlb_inval->xe, fence); + + spin_lock_irq(&tlb_inval->pending_lock); + fence->inval_time = ktime_get(); + list_add_tail(&fence->link, &tlb_inval->pending_fences); + + if (list_is_singular(&tlb_inval->pending_fences)) + queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr, + tlb_inval->ops->timeout_delay(tlb_inval)); + spin_unlock_irq(&tlb_inval->pending_lock); + + tlb_inval->seqno = (tlb_inval->seqno + 1) % + TLB_INVALIDATION_SEQNO_MAX; + if (!tlb_inval->seqno) + tlb_inval->seqno = 1; +} + +#define xe_tlb_inval_issue(__tlb_inval, __fence, op, args...) \ +({ \ + int __ret; \ + \ + xe_assert((__tlb_inval)->xe, (__tlb_inval)->ops); \ + xe_assert((__tlb_inval)->xe, (__fence)); \ + \ + mutex_lock(&(__tlb_inval)->seqno_lock); \ + xe_tlb_inval_fence_prep((__fence)); \ + __ret = op((__tlb_inval), (__fence)->seqno, ##args); \ + if (__ret < 0) \ + xe_tlb_inval_fence_signal_unlocked((__fence)); \ + mutex_unlock(&(__tlb_inval)->seqno_lock); \ + \ + __ret == -ECANCELED ? 0 : __ret; \ +}) + +/** + * xe_tlb_inval_all() - Issue a TLB invalidation for all TLBs + * @tlb_inval: TLB invalidation client + * @fence: invalidation fence which will be signal on TLB invalidation + * completion + * + * Issue a TLB invalidation for all TLBs. Completion of TLB is asynchronous and + * caller can use the invalidation fence to wait for completion. + * + * Return: 0 on success, negative error code on error + */ +int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence) +{ + return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->all); +} + +/** + * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT + * @tlb_inval: TLB invalidation client + * + * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and + * caller can use the invalidation fence to wait for completion. + * + * Return: 0 on success, negative error code on error + */ +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) +{ + struct xe_tlb_inval_fence fence, *fence_ptr = &fence; + int ret; + + xe_tlb_inval_fence_init(tlb_inval, fence_ptr, true); + ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, tlb_inval->ops->ggtt); + xe_tlb_inval_fence_wait(fence_ptr); + + return ret; +} + +/** + * xe_tlb_inval_range() - Issue a TLB invalidation for an address range + * @tlb_inval: TLB invalidation client + * @fence: invalidation fence which will be signal on TLB invalidation + * completion + * @start: start address + * @end: end address + * @asid: address space id + * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush + * + * Issue a range based TLB invalidation if supported, if not fallback to a full + * TLB invalidation. Completion of TLB is asynchronous and caller can use + * the invalidation fence to wait for completion. + * + * Return: Negative error code on error, 0 on success + */ +int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, u64 start, u64 end, + u32 asid, struct drm_suballoc *prl_sa) +{ + return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt, + start, end, asid, prl_sa); +} + +/** + * xe_tlb_inval_vm() - Issue a TLB invalidation for a VM + * @tlb_inval: TLB invalidation client + * @vm: VM to invalidate + * + * Invalidate entire VM's address space + */ +void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm) +{ + struct xe_tlb_inval_fence fence; + u64 range = 1ull << vm->xe->info.va_bits; + + xe_tlb_inval_fence_init(tlb_inval, &fence, true); + xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL); + xe_tlb_inval_fence_wait(&fence); +} + +/** + * xe_tlb_inval_done_handler() - TLB invalidation done handler + * @tlb_inval: TLB invalidation client + * @seqno: seqno of invalidation that is done + * + * Update recv seqno, signal any TLB invalidation fences, and restart TDR + */ +void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno) +{ + struct xe_device *xe = tlb_inval->xe; + struct xe_tlb_inval_fence *fence, *next; + unsigned long flags; + + /* + * This can also be run both directly from the IRQ handler and also in + * process_g2h_msg(). Only one may process any individual CT message, + * however the order they are processed here could result in skipping a + * seqno. To handle that we just process all the seqnos from the last + * seqno_recv up to and including the one in msg[0]. The delta should be + * very small so there shouldn't be much of pending_fences we actually + * need to iterate over here. + * + * From GuC POV we expect the seqnos to always appear in-order, so if we + * see something later in the timeline we can be sure that anything + * appearing earlier has already signalled, just that we have yet to + * officially process the CT message like if racing against + * process_g2h_msg(). + */ + spin_lock_irqsave(&tlb_inval->pending_lock, flags); + if (seqno == TLB_INVALIDATION_SEQNO_INVALID) { + xe_tlb_inval_reset_timeout(tlb_inval); + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); + return; + } + + if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) { + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); + return; + } + + WRITE_ONCE(tlb_inval->seqno_recv, seqno); + + list_for_each_entry_safe(fence, next, + &tlb_inval->pending_fences, link) { + trace_xe_tlb_inval_fence_recv(xe, fence); + + if (!xe_tlb_inval_seqno_past(tlb_inval, fence->seqno)) + break; + + xe_tlb_inval_fence_signal(fence); + } + + if (!list_empty(&tlb_inval->pending_fences)) + mod_delayed_work(tlb_inval->timeout_wq, + &tlb_inval->fence_tdr, + tlb_inval->ops->timeout_delay(tlb_inval)); + else + cancel_delayed_work(&tlb_inval->fence_tdr); + + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); +} + +static const char * +xe_inval_fence_get_driver_name(struct dma_fence *dma_fence) +{ + return "xe"; +} + +static const char * +xe_inval_fence_get_timeline_name(struct dma_fence *dma_fence) +{ + return "tlb_inval_fence"; +} + +static const struct dma_fence_ops inval_fence_ops = { + .get_driver_name = xe_inval_fence_get_driver_name, + .get_timeline_name = xe_inval_fence_get_timeline_name, +}; + +/** + * xe_tlb_inval_fence_init() - Initialize TLB invalidation fence + * @tlb_inval: TLB invalidation client + * @fence: TLB invalidation fence to initialize + * @stack: fence is stack variable + * + * Initialize TLB invalidation fence for use. xe_tlb_inval_fence_fini + * will be automatically called when fence is signalled (all fences must signal), + * even on error. + */ +void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, + bool stack) +{ + xe_pm_runtime_get_noresume(tlb_inval->xe); + + spin_lock_irq(&tlb_inval->lock); + dma_fence_init(&fence->base, &inval_fence_ops, &tlb_inval->lock, + dma_fence_context_alloc(1), 1); + spin_unlock_irq(&tlb_inval->lock); + INIT_LIST_HEAD(&fence->link); + if (stack) + set_bit(FENCE_STACK_BIT, &fence->base.flags); + else + dma_fence_get(&fence->base); + fence->tlb_inval = tlb_inval; +} + +/** + * xe_tlb_inval_idle() - Initialize TLB invalidation is idle + * @tlb_inval: TLB invalidation client + * + * Check the TLB invalidation seqno to determine if it is idle (i.e., no TLB + * invalidations are in flight). Expected to be called in the backend after the + * fence has been added to the pending list, and takes this into account. + * + * Return: True if TLB invalidation client is idle, False otherwise + */ +bool xe_tlb_inval_idle(struct xe_tlb_inval *tlb_inval) +{ + lockdep_assert_held(&tlb_inval->seqno_lock); + + guard(spinlock_irq)(&tlb_inval->pending_lock); + return list_is_singular(&tlb_inval->pending_fences); +} + +/** + * xe_tlb_inval_batch_wait() - Wait for all fences in a TLB invalidation batch + * @batch: Batch of TLB invalidation fences to wait on + * + * Waits for every fence in @batch to signal, then resets @batch so it can be + * reused for a subsequent invalidation. + */ +void xe_tlb_inval_batch_wait(struct xe_tlb_inval_batch *batch) +{ + struct xe_tlb_inval_fence *fence = &batch->fence[0]; + unsigned int i; + + for (i = 0; i < batch->num_fences; ++i) + xe_tlb_inval_fence_wait(fence++); + + batch->num_fences = 0; +} + +/** + * xe_tlb_inval_range_tilemask_submit() - Submit TLB invalidations for an + * address range on a tile mask + * @xe: The xe device + * @asid: Address space ID + * @start: start address + * @end: end address + * @tile_mask: mask for which gt's issue tlb invalidation + * @batch: Batch of tlb invalidate fences + * + * Issue a range based TLB invalidation for gt's in tilemask + * If the function returns an error, there is no need to call + * xe_tlb_inval_batch_wait() on @batch. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid, + u64 start, u64 end, u8 tile_mask, + struct xe_tlb_inval_batch *batch) +{ + struct xe_tlb_inval_fence *fence = &batch->fence[0]; + struct xe_tile *tile; + u32 fence_id = 0; + u8 id; + int err = 0; + + batch->num_fences = 0; + if (!tile_mask) + return 0; + + for_each_tile(tile, xe, id) { + if (!(tile_mask & BIT(id))) + continue; + + xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval, + &fence[fence_id], true); + + err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval, + &fence[fence_id], start, end, + asid, NULL); + if (err) + goto wait; + ++fence_id; + + if (!tile->media_gt) + continue; + + xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval, + &fence[fence_id], true); + + err = xe_tlb_inval_range(&tile->media_gt->tlb_inval, + &fence[fence_id], start, end, + asid, NULL); + if (err) + goto wait; + ++fence_id; + } + +wait: + batch->num_fences = fence_id; + if (err) + xe_tlb_inval_batch_wait(batch); + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h new file mode 100644 index 000000000..a76b7823a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TLB_INVAL_H_ +#define _XE_TLB_INVAL_H_ + +#include + +#include "xe_tlb_inval_types.h" + +struct xe_gt; +struct xe_guc; +struct xe_vm; + +int xe_gt_tlb_inval_init_early(struct xe_gt *gt); + +void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval); +int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence); +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); +void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm); +int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, + u64 start, u64 end, u32 asid, struct drm_suballoc *prl_sa); + +void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, + bool stack); + +/** + * xe_tlb_inval_fence_wait() - TLB invalidiation fence wait + * @fence: TLB invalidation fence to wait on + * + * Wait on a TLB invalidiation fence until it signals, non interruptible + */ +static inline void +xe_tlb_inval_fence_wait(struct xe_tlb_inval_fence *fence) +{ + dma_fence_wait(&fence->base, false); +} + +void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno); + +bool xe_tlb_inval_idle(struct xe_tlb_inval *tlb_inval); + +int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid, + u64 start, u64 end, u8 tile_mask, + struct xe_tlb_inval_batch *batch); + +void xe_tlb_inval_batch_wait(struct xe_tlb_inval_batch *batch); + +#endif /* _XE_TLB_INVAL_ */ diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c new file mode 100644 index 000000000..04d21015c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_assert.h" +#include "xe_dep_job_types.h" +#include "xe_dep_scheduler.h" +#include "xe_exec_queue.h" +#include "xe_gt_printk.h" +#include "xe_gt_types.h" +#include "xe_page_reclaim.h" +#include "xe_tlb_inval.h" +#include "xe_tlb_inval_job.h" +#include "xe_migrate.h" +#include "xe_pm.h" +#include "xe_vm.h" + +/** struct xe_tlb_inval_job - TLB invalidation job */ +struct xe_tlb_inval_job { + /** @dep: base generic dependency Xe job */ + struct xe_dep_job dep; + /** @tlb_inval: TLB invalidation client */ + struct xe_tlb_inval *tlb_inval; + /** @q: exec queue issuing the invalidate */ + struct xe_exec_queue *q; + /** @vm: VM which TLB invalidation is being issued for */ + struct xe_vm *vm; + /** @prl: Embedded copy of page reclaim list */ + struct xe_page_reclaim_list prl; + /** @refcount: ref count of this job */ + struct kref refcount; + /** + * @fence: dma fence to indicate completion. 1 way relationship - job + * can safely reference fence, fence cannot safely reference job. + */ + struct dma_fence *fence; + /** @start: Start address to invalidate */ + u64 start; + /** @end: End address to invalidate */ + u64 end; + /** @type: GT type */ + int type; + /** @fence_armed: Fence has been armed */ + bool fence_armed; +}; + +static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job) +{ + struct xe_tlb_inval_job *job = + container_of(dep_job, typeof(*job), dep); + struct xe_tlb_inval_fence *ifence = + container_of(job->fence, typeof(*ifence), base); + struct drm_suballoc *prl_sa = NULL; + + if (xe_page_reclaim_list_valid(&job->prl)) { + prl_sa = xe_page_reclaim_create_prl_bo(job->tlb_inval, &job->prl, ifence); + if (IS_ERR(prl_sa)) + prl_sa = NULL; /* Indicate fall back PPC flush with NULL */ + } + + xe_tlb_inval_range(job->tlb_inval, ifence, job->start, + job->end, job->vm->usm.asid, prl_sa); + + return job->fence; +} + +static void xe_tlb_inval_job_free(struct xe_dep_job *dep_job) +{ + struct xe_tlb_inval_job *job = + container_of(dep_job, typeof(*job), dep); + + /* Pairs with get in xe_tlb_inval_job_push */ + xe_tlb_inval_job_put(job); +} + +static const struct xe_dep_job_ops dep_job_ops = { + .run_job = xe_tlb_inval_job_run, + .free_job = xe_tlb_inval_job_free, +}; + +/** + * xe_tlb_inval_job_create() - TLB invalidation job create + * @q: exec queue issuing the invalidate + * @tlb_inval: TLB invalidation client + * @dep_scheduler: Dependency scheduler for job + * @vm: VM which TLB invalidation is being issued for + * @start: Start address to invalidate + * @end: End address to invalidate + * @type: GT type + * + * Create a TLB invalidation job and initialize internal fields. The caller is + * responsible for releasing the creation reference. + * + * Return: TLB invalidation job object on success, ERR_PTR failure + */ +struct xe_tlb_inval_job * +xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, + struct xe_dep_scheduler *dep_scheduler, + struct xe_vm *vm, u64 start, u64 end, int type) +{ + struct xe_tlb_inval_job *job; + struct drm_sched_entity *entity = + xe_dep_scheduler_entity(dep_scheduler); + struct xe_tlb_inval_fence *ifence; + int err; + + xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + + job = kmalloc_obj(*job); + if (!job) + return ERR_PTR(-ENOMEM); + + job->q = q; + job->vm = vm; + job->tlb_inval = tlb_inval; + job->start = start; + job->end = end; + job->fence_armed = false; + xe_page_reclaim_list_init(&job->prl); + job->dep.ops = &dep_job_ops; + job->type = type; + kref_init(&job->refcount); + xe_exec_queue_get(q); /* Pairs with put in xe_tlb_inval_job_destroy */ + xe_vm_get(vm); /* Pairs with put in xe_tlb_inval_job_destroy */ + + ifence = kmalloc_obj(*ifence); + if (!ifence) { + err = -ENOMEM; + goto err_job; + } + job->fence = &ifence->base; + + err = drm_sched_job_init(&job->dep.drm, entity, 1, NULL, + q->xef ? q->xef->drm->client_id : 0); + if (err) + goto err_fence; + + /* Pairs with put in xe_tlb_inval_job_destroy */ + xe_pm_runtime_get_noresume(gt_to_xe(q->gt)); + + return job; + +err_fence: + kfree(ifence); +err_job: + xe_vm_put(vm); + xe_exec_queue_put(q); + kfree(job); + + return ERR_PTR(err); +} + +/** + * xe_tlb_inval_job_add_page_reclaim() - Embed PRL into a TLB job + * @job: TLB invalidation job that may trigger reclamation + * @prl: Page reclaim list populated during unbind + * + * Copies @prl into the job and takes an extra reference to the entry page so + * ownership can transfer to the TLB fence when the job is pushed. + */ +void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job, + struct xe_page_reclaim_list *prl) +{ + struct xe_device *xe = gt_to_xe(job->q->gt); + + xe_gt_WARN_ON(job->q->gt, !xe->info.has_page_reclaim_hw_assist); + job->prl = *prl; + /* Pair with put in job_destroy */ + xe_page_reclaim_entries_get(job->prl.entries); +} + +static void xe_tlb_inval_job_destroy(struct kref *ref) +{ + struct xe_tlb_inval_job *job = container_of(ref, typeof(*job), + refcount); + struct xe_tlb_inval_fence *ifence = + container_of(job->fence, typeof(*ifence), base); + struct xe_exec_queue *q = job->q; + struct xe_device *xe = gt_to_xe(q->gt); + struct xe_vm *vm = job->vm; + + /* BO creation retains a copy (if used), so no longer needed */ + xe_page_reclaim_entries_put(job->prl.entries); + + if (!job->fence_armed) + kfree(ifence); + else + /* Ref from xe_tlb_inval_fence_init */ + dma_fence_put(job->fence); + + drm_sched_job_cleanup(&job->dep.drm); + kfree(job); + xe_vm_put(vm); /* Pairs with get from xe_tlb_inval_job_create */ + xe_exec_queue_put(q); /* Pairs with get from xe_tlb_inval_job_create */ + xe_pm_runtime_put(xe); /* Pairs with get from xe_tlb_inval_job_create */ +} + +/** + * xe_tlb_inval_job_alloc_dep() - TLB invalidation job alloc dependency + * @job: TLB invalidation job to alloc dependency for + * + * Allocate storage for a dependency in the TLB invalidation fence. This + * function should be called at most once per job and must be paired with + * xe_tlb_inval_job_push being called with a real fence. + * + * Return: 0 on success, -errno on failure + */ +int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job) +{ + xe_assert(gt_to_xe(job->q->gt), !xa_load(&job->dep.drm.dependencies, 0)); + might_alloc(GFP_KERNEL); + + return drm_sched_job_add_dependency(&job->dep.drm, + dma_fence_get_stub()); +} + +/** + * xe_tlb_inval_job_push() - TLB invalidation job push + * @job: TLB invalidation job to push + * @m: The migration object being used + * @fence: Dependency for TLB invalidation job + * + * Pushes a TLB invalidation job for execution, using @fence as a dependency. + * Storage for @fence must be preallocated with xe_tlb_inval_job_alloc_dep + * prior to this call if @fence is not signaled. Takes a reference to the job’s + * finished fence, which the caller is responsible for releasing, and return it + * to the caller. This function is safe to be called in the path of reclaim. + * + * Return: Job's finished fence on success, cannot fail + */ +struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, + struct xe_migrate *m, + struct dma_fence *fence) +{ + struct xe_tlb_inval_fence *ifence = + container_of(job->fence, typeof(*ifence), base); + + if (!dma_fence_is_signaled(fence)) { + void *ptr; + + /* + * Can be in path of reclaim, hence the preallocation of fence + * storage in xe_tlb_inval_job_alloc_dep. Verify caller did + * this correctly. + */ + xe_assert(gt_to_xe(job->q->gt), + xa_load(&job->dep.drm.dependencies, 0) == + dma_fence_get_stub()); + + dma_fence_get(fence); /* ref released once dependency processed by scheduler */ + ptr = xa_store(&job->dep.drm.dependencies, 0, fence, + GFP_ATOMIC); + xe_assert(gt_to_xe(job->q->gt), !xa_is_err(ptr)); + } + + xe_tlb_inval_job_get(job); /* Pairs with put in free_job */ + job->fence_armed = true; + + /* + * We need the migration lock to protect the job's seqno and the spsc + * queue, only taken on migration queue, user queues protected dma-resv + * VM lock. + */ + xe_migrate_job_lock(m, job->q); + + /* Creation ref pairs with put in xe_tlb_inval_job_destroy */ + xe_tlb_inval_fence_init(job->tlb_inval, ifence, false); + dma_fence_get(job->fence); /* Pairs with put in DRM scheduler */ + + drm_sched_job_arm(&job->dep.drm); + /* + * caller ref, get must be done before job push as it could immediately + * signal and free. + */ + dma_fence_get(&job->dep.drm.s_fence->finished); + drm_sched_entity_push_job(&job->dep.drm); + + /* Let the upper layers fish this out */ + xe_exec_queue_tlb_inval_last_fence_set(job->q, job->vm, + &job->dep.drm.s_fence->finished, + job->type); + + xe_migrate_job_unlock(m, job->q); + + /* + * Not using job->fence, as it has its own dma-fence context, which does + * not allow TLB invalidation fences on the same queue, GT tuple to + * be squashed in dma-resv/DRM scheduler. Instead, we use the DRM scheduler + * context and job's finished fence, which enables squashing. + */ + return &job->dep.drm.s_fence->finished; +} + +/** + * xe_tlb_inval_job_get() - Get a reference to TLB invalidation job + * @job: TLB invalidation job object + * + * Increment the TLB invalidation job's reference count + */ +void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job) +{ + kref_get(&job->refcount); +} + +/** + * xe_tlb_inval_job_put() - Put a reference to TLB invalidation job + * @job: TLB invalidation job object + * + * Decrement the TLB invalidation job's reference count, call + * xe_tlb_inval_job_destroy when reference count == 0. Skips decrement if + * input @job is NULL or IS_ERR. + */ +void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job) +{ + if (!IS_ERR_OR_NULL(job)) + kref_put(&job->refcount, xe_tlb_inval_job_destroy); +} diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_tlb_inval_job.h new file mode 100644 index 000000000..03d6e21cd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TLB_INVAL_JOB_H_ +#define _XE_TLB_INVAL_JOB_H_ + +#include + +struct dma_fence; +struct xe_dep_scheduler; +struct xe_exec_queue; +struct xe_migrate; +struct xe_page_reclaim_list; +struct xe_tlb_inval; +struct xe_tlb_inval_job; +struct xe_vm; + +struct xe_tlb_inval_job * +xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, + struct xe_dep_scheduler *dep_scheduler, + struct xe_vm *vm, u64 start, u64 end, int type); + +void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job, + struct xe_page_reclaim_list *prl); + +int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job); + +struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, + struct xe_migrate *m, + struct dma_fence *fence); + +void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job); + +void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job); + +#endif diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h new file mode 100644 index 000000000..3d1797d18 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_TLB_INVAL_TYPES_H_ +#define _XE_TLB_INVAL_TYPES_H_ + +#include +#include + +#include "xe_device_types.h" + +struct drm_suballoc; +struct xe_tlb_inval; + +/** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */ +struct xe_tlb_inval_ops { + /** + * @all: Invalidate all TLBs + * @tlb_inval: TLB invalidation client + * @seqno: Seqno of TLB invalidation + * + * Return 0 on success, -ECANCELED if backend is mid-reset, error on + * failure + */ + int (*all)(struct xe_tlb_inval *tlb_inval, u32 seqno); + + /** + * @ggtt: Invalidate global translation TLBs + * @tlb_inval: TLB invalidation client + * @seqno: Seqno of TLB invalidation + * + * Return 0 on success, -ECANCELED if backend is mid-reset, error on + * failure + */ + int (*ggtt)(struct xe_tlb_inval *tlb_inval, u32 seqno); + + /** + * @ppgtt: Invalidate per-process translation TLBs + * @tlb_inval: TLB invalidation client + * @seqno: Seqno of TLB invalidation + * @start: Start address + * @end: End address + * @asid: Address space ID + * @prl_sa: Suballocation for page reclaim list + * + * Return 0 on success, -ECANCELED if backend is mid-reset, error on + * failure + */ + int (*ppgtt)(struct xe_tlb_inval *tlb_inval, u32 seqno, u64 start, + u64 end, u32 asid, struct drm_suballoc *prl_sa); + + /** + * @initialized: Backend is initialized + * @tlb_inval: TLB invalidation client + * + * Return: True if back is initialized, False otherwise + */ + bool (*initialized)(struct xe_tlb_inval *tlb_inval); + + /** + * @flush: Flush pending TLB invalidations + * @tlb_inval: TLB invalidation client + */ + void (*flush)(struct xe_tlb_inval *tlb_inval); + + /** + * @timeout_delay: Timeout delay for TLB invalidation + * @tlb_inval: TLB invalidation client + * + * Return: Timeout delay for TLB invalidation in jiffies + */ + long (*timeout_delay)(struct xe_tlb_inval *tlb_inval); +}; + +/** struct xe_tlb_inval - TLB invalidation client (frontend) */ +struct xe_tlb_inval { + /** @private: Backend private pointer */ + void *private; + /** @xe: Pointer to Xe device */ + struct xe_device *xe; + /** @ops: TLB invalidation ops */ + const struct xe_tlb_inval_ops *ops; + /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ +#define TLB_INVALIDATION_SEQNO_MAX 0x100000 +#define TLB_INVALIDATION_SEQNO_INVALID TLB_INVALIDATION_SEQNO_MAX + int seqno; + /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ + struct mutex seqno_lock; + /** + * @seqno_recv: last received TLB invalidation seqno, protected by + * CT lock + */ + int seqno_recv; + /** + * @pending_fences: list of pending fences waiting TLB invaliations, + * protected CT lock + */ + struct list_head pending_fences; + /** + * @pending_lock: protects @pending_fences and updating @seqno_recv. + */ + spinlock_t pending_lock; + /** + * @fence_tdr: schedules a delayed call to xe_tlb_fence_timeout after + * the timeout interval is over. + */ + struct delayed_work fence_tdr; + /** @job_wq: schedules TLB invalidation jobs */ + struct workqueue_struct *job_wq; + /** @tlb_inval.lock: protects TLB invalidation fences */ + spinlock_t lock; + /** @timeout_wq: schedules TLB invalidation fence timeouts */ + struct workqueue_struct *timeout_wq; +}; + +/** + * struct xe_tlb_inval_fence - TLB invalidation fence + * + * Optionally passed to xe_tlb_inval* functions and will be signaled upon TLB + * invalidation completion. + */ +struct xe_tlb_inval_fence { + /** @base: dma fence base */ + struct dma_fence base; + /** @tlb_inval: TLB invalidation client which fence belong to */ + struct xe_tlb_inval *tlb_inval; + /** @link: link into list of pending tlb fences */ + struct list_head link; + /** @seqno: seqno of TLB invalidation to signal fence one */ + int seqno; + /** @inval_time: time of TLB invalidation */ + ktime_t inval_time; +}; + +/** + * struct xe_tlb_inval_batch - Batch of TLB invalidation fences + * + * Holds one fence per GT covered by a TLB invalidation request. + */ +struct xe_tlb_inval_batch { + /** @fence: per-GT TLB invalidation fences */ + struct xe_tlb_inval_fence fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; + /** @num_fences: number of valid entries in @fence */ + unsigned int num_fences; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_trace.c b/drivers/gpu/drm/xe/xe_trace.c new file mode 100644 index 000000000..2527c556b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __CHECKER__ +#define CREATE_TRACE_POINTS +#include "xe_trace.h" +#endif diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h new file mode 100644 index 000000000..2fe8f89a1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -0,0 +1,520 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright © 2022 Intel Corporation + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xe + +#if !defined(_XE_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _XE_TRACE_H_ + +#include +#include + +#include "xe_exec_queue_types.h" +#include "xe_exec_queue.h" +#include "xe_gpu_scheduler_types.h" +#include "xe_gt_types.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_sched_job.h" +#include "xe_tlb_inval_types.h" +#include "xe_vm.h" + +#define __dev_name_xe(xe) dev_name((xe)->drm.dev) +#define __dev_name_tile(tile) __dev_name_xe(tile_to_xe((tile))) +#define __dev_name_gt(gt) __dev_name_xe(gt_to_xe((gt))) +#define __dev_name_eq(q) __dev_name_gt((q)->gt) + +DECLARE_EVENT_CLASS(xe_tlb_inval_fence, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), + TP_ARGS(xe, fence), + + TP_STRUCT__entry( + __string(dev, __dev_name_xe(xe)) + __field(struct xe_tlb_inval_fence *, fence) + __field(int, seqno) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->fence = fence; + __entry->seqno = fence->seqno; + ), + + TP_printk("dev=%s, fence=%p, seqno=%d", + __get_str(dev), __entry->fence, __entry->seqno) +); + +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_send, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), + TP_ARGS(xe, fence) +); + +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_recv, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), + TP_ARGS(xe, fence) +); + +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_signal, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), + TP_ARGS(xe, fence) +); + +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_timeout, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), + TP_ARGS(xe, fence) +); + +DECLARE_EVENT_CLASS(xe_exec_queue, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q), + + TP_STRUCT__entry( + __string(dev, __dev_name_eq(q)) + __field(enum xe_engine_class, class) + __field(u32, logical_mask) + __field(u8, gt_id) + __field(u16, width) + __field(u16, guc_id) + __field(u32, guc_state) + __field(u32, flags) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->class = q->class; + __entry->logical_mask = q->logical_mask; + __entry->gt_id = q->gt->info.id; + __entry->width = q->width; + __entry->guc_id = q->guc->id; + __entry->guc_state = atomic_read(&q->guc->state); + __entry->flags = q->flags; + ), + + TP_printk("dev=%s, %d:0x%x, gt=%d, width=%d, guc_id=%d, guc_state=0x%x, flags=0x%x", + __get_str(dev), __entry->class, __entry->logical_mask, + __entry->gt_id, __entry->width, __entry->guc_id, + __entry->guc_state, __entry->flags) +); + +DECLARE_EVENT_CLASS(xe_exec_queue_multi_queue, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q), + + TP_STRUCT__entry( + __string(dev, __dev_name_eq(q)) + __field(enum xe_engine_class, class) + __field(u32, logical_mask) + __field(u8, gt_id) + __field(u16, width) + __field(u32, guc_id) + __field(u32, guc_state) + __field(u32, flags) + __field(u32, primary) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->class = q->class; + __entry->logical_mask = q->logical_mask; + __entry->gt_id = q->gt->info.id; + __entry->width = q->width; + __entry->guc_id = q->guc->id; + __entry->guc_state = atomic_read(&q->guc->state); + __entry->flags = q->flags; + __entry->primary = xe_exec_queue_multi_queue_primary(q)->guc->id; + ), + + TP_printk("dev=%s, %d:0x%x, gt=%d, width=%d guc_id=%d, guc_state=0x%x, flags=0x%x, primary=%d", + __get_str(dev), __entry->class, __entry->logical_mask, + __entry->gt_id, __entry->width, __entry->guc_id, + __entry->guc_state, __entry->flags, + __entry->primary) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue_multi_queue, xe_exec_queue_create_multi_queue, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_submit, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_scheduling_enable, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_scheduling_disable, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_scheduling_done, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_register, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_deregister, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_deregister_done, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_close, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_kill, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cleanup_entity, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_destroy, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_reset, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_guc_uncorrectable_error, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cgp_context_error, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_resubmit, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + +DECLARE_EVENT_CLASS(xe_sched_job, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job), + + TP_STRUCT__entry( + __string(dev, __dev_name_eq(job->q)) + __field(u32, seqno) + __field(u32, lrc_seqno) + __field(u8, gt_id) + __field(u16, guc_id) + __field(u32, guc_state) + __field(u32, flags) + __field(int, error) + __field(struct dma_fence *, fence) + __field(u64, batch_addr) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->seqno = xe_sched_job_seqno(job); + __entry->lrc_seqno = xe_sched_job_lrc_seqno(job); + __entry->gt_id = job->q->gt->info.id; + __entry->guc_id = job->q->guc->id; + __entry->guc_state = + atomic_read(&job->q->guc->state); + __entry->flags = job->q->flags; + __entry->error = job->fence ? job->fence->error : 0; + __entry->fence = job->fence; + __entry->batch_addr = (u64)job->ptrs[0].batch_addr; + ), + + TP_printk("dev=%s, fence=%p, seqno=%u, lrc_seqno=%u, gt=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d", + __get_str(dev), __entry->fence, __entry->seqno, + __entry->lrc_seqno, __entry->gt_id, __entry->guc_id, + __entry->batch_addr, __entry->guc_state, + __entry->flags, __entry->error) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_create, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_exec, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_run, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_free, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_timedout, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_set_error, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_ban, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DECLARE_EVENT_CLASS(xe_sched_msg, + TP_PROTO(struct xe_sched_msg *msg), + TP_ARGS(msg), + + TP_STRUCT__entry( + __string(dev, __dev_name_eq(((struct xe_exec_queue *)msg->private_data))) + __field(u32, opcode) + __field(u16, guc_id) + __field(u8, gt_id) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->opcode = msg->opcode; + __entry->guc_id = + ((struct xe_exec_queue *)msg->private_data)->guc->id; + __entry->gt_id = + ((struct xe_exec_queue *)msg->private_data)->gt->info.id; + ), + + TP_printk("dev=%s, gt=%u guc_id=%d, opcode=%u", __get_str(dev), __entry->gt_id, __entry->guc_id, + __entry->opcode) +); + +DEFINE_EVENT(xe_sched_msg, xe_sched_msg_add, + TP_PROTO(struct xe_sched_msg *msg), + TP_ARGS(msg) +); + +DEFINE_EVENT(xe_sched_msg, xe_sched_msg_recv, + TP_PROTO(struct xe_sched_msg *msg), + TP_ARGS(msg) +); + +DECLARE_EVENT_CLASS(xe_hw_fence, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence), + + TP_STRUCT__entry( + __string(dev, __dev_name_xe(fence->xe)) + __field(u64, ctx) + __field(u32, seqno) + __field(struct xe_hw_fence *, fence) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->ctx = fence->dma.context; + __entry->seqno = fence->dma.seqno; + __entry->fence = fence; + ), + + TP_printk("dev=%s, ctx=0x%016llx, fence=%p, seqno=%u", + __get_str(dev), __entry->ctx, __entry->fence, __entry->seqno) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_create, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_signal, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_try_signal, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +TRACE_EVENT(xe_reg_rw, + TP_PROTO(struct xe_mmio *mmio, bool write, u32 reg, u64 val, int len), + + TP_ARGS(mmio, write, reg, val, len), + + TP_STRUCT__entry( + __string(dev, __dev_name_tile(mmio->tile)) + __field(u64, val) + __field(u32, reg) + __field(u16, write) + __field(u16, len) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->val = val; + __entry->reg = reg; + __entry->write = write; + __entry->len = len; + ), + + TP_printk("dev=%s, %s reg=0x%x, len=%d, val=(0x%x, 0x%x)", + __get_str(dev), __entry->write ? "write" : "read", + __entry->reg, __entry->len, + (u32)(__entry->val & 0xffffffff), + (u32)(__entry->val >> 32)) +); + +DECLARE_EVENT_CLASS(xe_pm_runtime, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller), + + TP_STRUCT__entry( + __string(dev, __dev_name_xe(xe)) + __field(void *, caller) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->caller = caller; + ), + + TP_printk("dev=%s caller_function=%pS", __get_str(dev), __entry->caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_get, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_put, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_resume, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_suspend, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_resume, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_suspend, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_get_ioctl, + TP_PROTO(struct xe_device *xe, void *caller), + TP_ARGS(xe, caller) +); + +TRACE_EVENT(xe_eu_stall_data_read, + TP_PROTO(u8 slice, u8 subslice, + u32 read_ptr, u32 write_ptr, + size_t read_size, size_t total_size), + TP_ARGS(slice, subslice, + read_ptr, write_ptr, + read_size, total_size), + + TP_STRUCT__entry(__field(u8, slice) + __field(u8, subslice) + __field(u32, read_ptr) + __field(u32, write_ptr) + __field(size_t, read_size) + __field(size_t, total_size) + ), + + TP_fast_assign(__entry->slice = slice; + __entry->subslice = subslice; + __entry->read_ptr = read_ptr; + __entry->write_ptr = write_ptr; + __entry->read_size = read_size; + __entry->total_size = total_size; + ), + + TP_printk("slice: %u subslice: %u read ptr: 0x%x write ptr: 0x%x read size: %zu total read size: %zu", + __entry->slice, __entry->subslice, + __entry->read_ptr, __entry->write_ptr, + __entry->read_size, __entry->total_size) +); + +TRACE_EVENT(xe_exec_queue_reach_max_job_count, + TP_PROTO(struct xe_exec_queue *q, int max_cnt), + TP_ARGS(q, max_cnt), + + TP_STRUCT__entry(__string(dev, __dev_name_eq(q)) + __field(enum xe_engine_class, class) + __field(u32, logical_mask) + __field(u16, guc_id) + __field(int, max_cnt) + ), + + TP_fast_assign(__assign_str(dev); + __entry->class = q->class; + __entry->logical_mask = q->logical_mask; + __entry->guc_id = q->guc->id; + __entry->max_cnt = max_cnt; + ), + + TP_printk("dev=%s, job count exceeded the maximum limit (%d) per exec queue. engine_class=0x%x, logical_mask=0x%x, guc_id=%d", + __get_str(dev), __entry->max_cnt, + __entry->class, __entry->logical_mask, __entry->guc_id) +); + +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/xe +#define TRACE_INCLUDE_FILE xe_trace +#include diff --git a/drivers/gpu/drm/xe/xe_trace_bo.c b/drivers/gpu/drm/xe/xe_trace_bo.c new file mode 100644 index 000000000..6d5e66ce4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace_bo.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef __CHECKER__ +#define CREATE_TRACE_POINTS +#include "xe_trace_bo.h" +#endif diff --git a/drivers/gpu/drm/xe/xe_trace_bo.h b/drivers/gpu/drm/xe/xe_trace_bo.h new file mode 100644 index 000000000..86323cf3b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace_bo.h @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright © 2024 Intel Corporation + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xe + +#if !defined(_XE_TRACE_BO_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _XE_TRACE_BO_H_ + +#include +#include + +#include "xe_bo.h" +#include "xe_bo_types.h" +#include "xe_vm.h" + +#define __dev_name_bo(bo) dev_name(xe_bo_device(bo)->drm.dev) +#define __dev_name_vm(vm) dev_name((vm)->xe->drm.dev) +#define __dev_name_vma(vma) __dev_name_vm(xe_vma_vm(vma)) + +DECLARE_EVENT_CLASS(xe_bo, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo), + + TP_STRUCT__entry( + __string(dev, __dev_name_bo(bo)) + __field(size_t, size) + __field(u32, flags) + __field(struct xe_vm *, vm) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->size = xe_bo_size(bo); + __entry->flags = bo->flags; + __entry->vm = bo->vm; + ), + + TP_printk("dev=%s, size=%zu, flags=0x%02x, vm=%p", + __get_str(dev), __entry->size, + __entry->flags, __entry->vm) +); + +DEFINE_EVENT(xe_bo, xe_bo_cpu_fault, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo) +); + +DEFINE_EVENT(xe_bo, xe_bo_validate, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo) +); + +DEFINE_EVENT(xe_bo, xe_bo_create, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo) +); + +TRACE_EVENT(xe_bo_move, + TP_PROTO(struct xe_bo *bo, uint32_t new_placement, uint32_t old_placement, + bool move_lacks_source), + TP_ARGS(bo, new_placement, old_placement, move_lacks_source), + TP_STRUCT__entry( + __field(struct xe_bo *, bo) + __field(size_t, size) + __string(new_placement_name, xe_mem_type_to_name[new_placement]) + __string(old_placement_name, xe_mem_type_to_name[old_placement]) + __string(device_id, __dev_name_bo(bo)) + __field(bool, move_lacks_source) + ), + + TP_fast_assign( + __entry->bo = bo; + __entry->size = xe_bo_size(bo); + __assign_str(new_placement_name); + __assign_str(old_placement_name); + __assign_str(device_id); + __entry->move_lacks_source = move_lacks_source; + ), + TP_printk("move_lacks_source:%s, migrate object %p [size %zu] from %s to %s device_id:%s", + __entry->move_lacks_source ? "yes" : "no", __entry->bo, __entry->size, + __get_str(old_placement_name), + __get_str(new_placement_name), __get_str(device_id)) +); + +DECLARE_EVENT_CLASS(xe_vma, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma), + + TP_STRUCT__entry( + __string(dev, __dev_name_vma(vma)) + __field(struct xe_vma *, vma) + __field(struct xe_vm *, vm) + __field(u32, asid) + __field(u64, start) + __field(u64, end) + __field(u64, ptr) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->vma = vma; + __entry->vm = xe_vma_vm(vma); + __entry->asid = xe_vma_vm(vma)->usm.asid; + __entry->start = xe_vma_start(vma); + __entry->end = xe_vma_end(vma) - 1; + __entry->ptr = xe_vma_userptr(vma); + ), + + TP_printk("dev=%s, vma=%p, vm=%p, asid=0x%05x, start=0x%012llx, end=0x%012llx, userptr=0x%012llx", + __get_str(dev), __entry->vma, __entry->vm, + __entry->asid, __entry->start, + __entry->end, __entry->ptr) +) + +DEFINE_EVENT(xe_vma, xe_vma_flush, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_pagefault, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_acc, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_bind, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_pf_bind, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_unbind, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_rebind_worker, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_rebind_exec, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_rebind_worker, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_rebind_exec, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_invalidate, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_invalidate, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_evict, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_invalidate_complete, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DECLARE_EVENT_CLASS(xe_vm, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm), + + TP_STRUCT__entry( + __string(dev, __dev_name_vm(vm)) + __field(struct xe_vm *, vm) + __field(u32, asid) + __field(u32, flags) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->vm = vm; + __entry->asid = vm->usm.asid; + __entry->flags = vm->flags; + ), + + TP_printk("dev=%s, vm=%p, asid=0x%05x, vm flags=0x%05x", + __get_str(dev), __entry->vm, __entry->asid, + __entry->flags) +); + +DEFINE_EVENT(xe_vm, xe_vm_kill, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_create, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_free, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_cpu_bind, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_restart, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_enter, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_retry, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_exit, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_ops_fail, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/xe +#define TRACE_INCLUDE_FILE xe_trace_bo +#include diff --git a/drivers/gpu/drm/xe/xe_trace_guc.c b/drivers/gpu/drm/xe/xe_trace_guc.c new file mode 100644 index 000000000..fcdf6888f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace_guc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef __CHECKER__ +#define CREATE_TRACE_POINTS +#include "xe_trace_guc.h" +#endif diff --git a/drivers/gpu/drm/xe/xe_trace_guc.h b/drivers/gpu/drm/xe/xe_trace_guc.h new file mode 100644 index 000000000..78949db9c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace_guc.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright © 2024 Intel Corporation + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xe + +#if !defined(_XE_TRACE_GUC_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _XE_TRACE_GUC_H_ + +#include +#include + +#include "xe_device_types.h" +#include "xe_guc_exec_queue_types.h" +#include "xe_guc_engine_activity_types.h" + +#define __dev_name_xe(xe) dev_name((xe)->drm.dev) + +DECLARE_EVENT_CLASS(xe_guc_ct_flow_control, + TP_PROTO(struct xe_device *xe, u32 _head, u32 _tail, u32 size, u32 space, u32 len), + TP_ARGS(xe, _head, _tail, size, space, len), + + TP_STRUCT__entry( + __string(dev, __dev_name_xe(xe)) + __field(u32, _head) + __field(u32, _tail) + __field(u32, size) + __field(u32, space) + __field(u32, len) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->_head = _head; + __entry->_tail = _tail; + __entry->size = size; + __entry->space = space; + __entry->len = len; + ), + + TP_printk("h2g flow control: dev=%s, head=%u, tail=%u, size=%u, space=%u, len=%u", + __get_str(dev), __entry->_head, __entry->_tail, __entry->size, + __entry->space, __entry->len) +); + +DEFINE_EVENT(xe_guc_ct_flow_control, xe_guc_ct_h2g_flow_control, + TP_PROTO(struct xe_device *xe, u32 _head, u32 _tail, u32 size, u32 space, u32 len), + TP_ARGS(xe, _head, _tail, size, space, len) +); + +DEFINE_EVENT_PRINT(xe_guc_ct_flow_control, xe_guc_ct_g2h_flow_control, + TP_PROTO(struct xe_device *xe, u32 _head, u32 _tail, u32 size, u32 space, u32 len), + TP_ARGS(xe, _head, _tail, size, space, len), + + TP_printk("g2h flow control: dev=%s, head=%u, tail=%u, size=%u, space=%u, len=%u", + __get_str(dev), __entry->_head, __entry->_tail, __entry->size, + __entry->space, __entry->len) +); + +DECLARE_EVENT_CLASS(xe_guc_ctb, + TP_PROTO(struct xe_device *xe, u8 gt_id, u32 action, u32 len, u32 _head, u32 tail), + TP_ARGS(xe, gt_id, action, len, _head, tail), + + TP_STRUCT__entry( + __string(dev, __dev_name_xe(xe)) + __field(u8, gt_id) + __field(u32, action) + __field(u32, len) + __field(u32, tail) + __field(u32, _head) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->gt_id = gt_id; + __entry->action = action; + __entry->len = len; + __entry->tail = tail; + __entry->_head = _head; + ), + + TP_printk("H2G CTB: dev=%s, gt%d: action=0x%x, len=%d, tail=%d, head=%d\n", + __get_str(dev), __entry->gt_id, __entry->action, __entry->len, + __entry->tail, __entry->_head) +); + +DEFINE_EVENT(xe_guc_ctb, xe_guc_ctb_h2g, + TP_PROTO(struct xe_device *xe, u8 gt_id, u32 action, u32 len, u32 _head, u32 tail), + TP_ARGS(xe, gt_id, action, len, _head, tail) +); + +DEFINE_EVENT_PRINT(xe_guc_ctb, xe_guc_ctb_g2h, + TP_PROTO(struct xe_device *xe, u8 gt_id, u32 action, u32 len, u32 _head, u32 tail), + TP_ARGS(xe, gt_id, action, len, _head, tail), + + TP_printk("G2H CTB: dev=%s, gt%d: action=0x%x, len=%d, tail=%d, head=%d\n", + __get_str(dev), __entry->gt_id, __entry->action, __entry->len, + __entry->tail, __entry->_head) + +); + +TRACE_EVENT(xe_guc_engine_activity, + TP_PROTO(struct xe_device *xe, struct engine_activity *ea, const char *name, + u16 instance), + TP_ARGS(xe, ea, name, instance), + + TP_STRUCT__entry( + __string(dev, __dev_name_xe(xe)) + __string(name, name) + __field(u32, global_change_num) + __field(u32, guc_tsc_frequency_hz) + __field(u32, lag_latency_usec) + __field(u16, instance) + __field(u16, change_num) + __field(u16, quanta_ratio) + __field(u32, last_update_tick) + __field(u64, active_ticks) + __field(u64, active) + __field(u64, total) + __field(u64, quanta) + __field(u64, last_cpu_ts) + ), + + TP_fast_assign( + __assign_str(dev); + __assign_str(name); + __entry->global_change_num = ea->metadata.global_change_num; + __entry->guc_tsc_frequency_hz = ea->metadata.guc_tsc_frequency_hz; + __entry->lag_latency_usec = ea->metadata.lag_latency_usec; + __entry->instance = instance; + __entry->change_num = ea->activity.change_num; + __entry->quanta_ratio = ea->activity.quanta_ratio; + __entry->last_update_tick = ea->activity.last_update_tick; + __entry->active_ticks = ea->activity.active_ticks; + __entry->active = ea->active; + __entry->total = ea->total; + __entry->quanta = ea->quanta; + __entry->last_cpu_ts = ea->last_cpu_ts; + ), + + TP_printk("dev=%s engine %s:%d Active=%llu, quanta=%llu, last_cpu_ts=%llu\n" + "Activity metadata: global_change_num=%u, guc_tsc_frequency_hz=%u lag_latency_usec=%u\n" + "Activity data: change_num=%u, quanta_ratio=0x%x, last_update_tick=%u, active_ticks=%llu\n", + __get_str(dev), __get_str(name), __entry->instance, + (__entry->active + __entry->total), __entry->quanta, __entry->last_cpu_ts, + __entry->global_change_num, __entry->guc_tsc_frequency_hz, + __entry->lag_latency_usec, __entry->change_num, __entry->quanta_ratio, + __entry->last_update_tick, __entry->active_ticks) +); +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/xe +#define TRACE_INCLUDE_FILE xe_trace_guc +#include diff --git a/drivers/gpu/drm/xe/xe_trace_lrc.c b/drivers/gpu/drm/xe/xe_trace_lrc.c new file mode 100644 index 000000000..ab9b7e297 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace_lrc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef __CHECKER__ +#define CREATE_TRACE_POINTS +#include "xe_trace_lrc.h" +#endif diff --git a/drivers/gpu/drm/xe/xe_trace_lrc.h b/drivers/gpu/drm/xe/xe_trace_lrc.h new file mode 100644 index 000000000..5c4cfa0c1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace_lrc.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright © 2024 Intel Corporation + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xe + +#if !defined(_XE_TRACE_LRC_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _XE_TRACE_LRC_H_ + +#include +#include + +#include "xe_exec_queue_types.h" +#include "xe_gt_types.h" +#include "xe_lrc.h" +#include "xe_lrc_types.h" + +#define __dev_name_lrc(lrc) dev_name(gt_to_xe((lrc)->fence_ctx.gt)->drm.dev) + +TRACE_EVENT(xe_lrc_update_timestamp, + TP_PROTO(struct xe_lrc *lrc, uint64_t old), + TP_ARGS(lrc, old), + TP_STRUCT__entry( + __field(struct xe_lrc *, lrc) + __field(u64, old) + __field(u64, new) + __string(name, lrc->fence_ctx.name) + __string(device_id, __dev_name_lrc(lrc)) + ), + + TP_fast_assign( + __entry->lrc = lrc; + __entry->old = old; + __entry->new = lrc->ctx_timestamp; + __assign_str(name); + __assign_str(device_id); + ), + TP_printk("lrc=:%p lrc->name=%s old=%llu new=%llu device_id:%s", + __entry->lrc, __get_str(name), + __entry->old, __entry->new, + __get_str(device_id)) +); + +TRACE_EVENT(xe_lrc_update_queue_timestamp, + TP_PROTO(struct xe_lrc *lrc, uint64_t old), + TP_ARGS(lrc, old), + TP_STRUCT__entry( + __field(struct xe_lrc *, lrc) + __field(struct xe_lrc *, primary_lrc) + __field(u64, old) + __field(u64, new) + __string(name, lrc->fence_ctx.name) + __string(device_id, __dev_name_lrc(lrc)) + ), + + TP_fast_assign( + __entry->lrc = lrc; + __entry->primary_lrc = lrc->multi_queue.primary_lrc; + __entry->old = old; + __entry->new = lrc->queue_timestamp; + __assign_str(name); + __assign_str(device_id); + ), + TP_printk("lrc=%p primary_lrc=%p lrc->name=%s old=%llu new=%llu device_id:%s", + __entry->lrc, __entry->primary_lrc, __get_str(name), + __entry->old, __entry->new, + __get_str(device_id)) +); + +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/xe +#define TRACE_INCLUDE_FILE xe_trace_lrc +#include diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c new file mode 100644 index 000000000..60f39efb0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2023 Intel Corporation + * Copyright (C) 2021-2022 Red Hat + */ + +#include + +#include +#include +#include + +#include + +#include "regs/xe_bars.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt_printk.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_vram.h" +#include "xe_wa.h" + +/** + * xe_ttm_stolen_cpu_access_needs_ggtt() - If we can't directly CPU access + * stolen, can we then fallback to mapping through the GGTT. + * @xe: xe device + * + * Some older integrated platforms don't support reliable CPU access for stolen, + * however on such hardware we can always use the mappable part of the GGTT for + * CPU access. Check if that's the case for this device. + */ +bool xe_ttm_stolen_cpu_access_needs_ggtt(struct xe_device *xe) +{ + return GRAPHICS_VERx100(xe) < 1270 && !IS_DGFX(xe); +} + +static u32 get_wopcm_size(struct xe_device *xe) +{ + u32 wopcm_size; + u64 val; + + val = xe_mmio_read64_2x32(xe_root_tile_mmio(xe), STOLEN_RESERVED); + val = REG_FIELD_GET64(WOPCM_SIZE_MASK, val); + + switch (val) { + case 0x5 ... 0x6: + val--; + fallthrough; + case 0x0 ... 0x3: + wopcm_size = (1U << val) * SZ_1M; + break; + default: + WARN(1, "Missing case wopcm_size=%llx\n", val); + wopcm_size = 0; + } + + return wopcm_size; +} + +static u64 detect_lmembar_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) +{ + struct xe_vram_region *tile_vram = xe_device_get_root_tile(xe)->mem.vram; + resource_size_t tile_io_start = xe_vram_region_io_start(tile_vram); + struct xe_mmio *mmio = xe_root_tile_mmio(xe); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + u64 stolen_size, wopcm_size; + u64 tile_offset; + u64 tile_size; + + tile_offset = tile_io_start - xe_vram_region_io_start(xe->mem.vram); + tile_size = xe_vram_region_actual_physical_size(tile_vram); + + /* Use DSM base address instead for stolen memory */ + mgr->stolen_base = (xe_mmio_read64_2x32(mmio, DSMBASE) & BDSM_MASK) - tile_offset; + if (drm_WARN_ON(&xe->drm, tile_size < mgr->stolen_base)) + return 0; + + /* Carve out the top of DSM as it contains the reserved WOPCM region */ + wopcm_size = get_wopcm_size(xe); + if (drm_WARN_ON(&xe->drm, !wopcm_size)) + return 0; + + stolen_size = tile_size - mgr->stolen_base; + + xe_assert(xe, stolen_size >= wopcm_size); + stolen_size -= wopcm_size; + + /* Verify usage fits in the actual resource available */ + if (mgr->stolen_base + stolen_size <= pci_resource_len(pdev, LMEM_BAR)) + mgr->io_base = tile_io_start + mgr->stolen_base; + + /* + * There may be few KB of platform dependent reserved memory at the end + * of vram which is not part of the DSM. Such reserved memory portion is + * always less then DSM granularity so align down the stolen_size to DSM + * granularity to accommodate such reserve vram portion. + */ + return ALIGN_DOWN(stolen_size, SZ_1M); +} + +static u32 detect_lmembar_integrated(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct xe_gt *media_gt = xe_device_get_root_tile(xe)->media_gt; + u32 stolen_size, wopcm_size; + u32 ggc, gms; + + ggc = xe_mmio_read32(xe_root_tile_mmio(xe), GGC); + + /* + * Check GGMS: it should be fixed 0x3 (8MB), which corresponds to the + * GTT size + */ + if (drm_WARN_ON(&xe->drm, (ggc & GGMS_MASK) != GGMS_MASK)) + return 0; + + /* + * Graphics >= 1270 uses the offset to the GSMBASE as address in the + * PTEs, together with the DM flag being set. Previously there was no + * such flag so the address was the io_base. + * + * DSMBASE = GSMBASE + 8MB + */ + mgr->stolen_base = SZ_8M; + mgr->io_base = pci_resource_start(pdev, 2) + mgr->stolen_base; + + /* return valid GMS value, -EIO if invalid */ + gms = REG_FIELD_GET(GMS_MASK, ggc); + switch (gms) { + case 0x0 ... 0x04: + stolen_size = gms * 32 * SZ_1M; + break; + case 0xf0 ... 0xfe: + stolen_size = (gms - 0xf0 + 1) * 4 * SZ_1M; + break; + default: + return 0; + } + + /* Carve out the top of DSM as it contains the reserved WOPCM region */ + wopcm_size = get_wopcm_size(xe); + if (drm_WARN_ON(&xe->drm, !wopcm_size)) + return 0; + + stolen_size -= wopcm_size; + + if (media_gt && XE_GT_WA(media_gt, 14019821291)) { + u64 gscpsmi_base = xe_mmio_read64_2x32(&media_gt->mmio, GSCPSMI_BASE) + & ~GENMASK_ULL(5, 0); + + /* + * This workaround is primarily implemented by the BIOS. We + * just need to figure out whether the BIOS has applied the + * workaround (meaning the programmed address falls within + * the DSM) and, if so, reserve that part of the DSM to + * prevent accidental reuse. The DSM location should be just + * below the WOPCM. + */ + if (gscpsmi_base >= mgr->io_base && + gscpsmi_base < mgr->io_base + stolen_size) { + xe_gt_dbg(media_gt, + "Reserving %llu bytes of DSM for Wa_14019821291\n", + mgr->io_base + stolen_size - gscpsmi_base); + stolen_size = gscpsmi_base - mgr->io_base; + } + } + + if (drm_WARN_ON(&xe->drm, stolen_size + SZ_8M > pci_resource_len(pdev, 2))) + return 0; + + return stolen_size; +} + +extern struct resource intel_graphics_stolen_res; + +static u64 detect_stolen(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) +{ +#ifdef CONFIG_X86 + /* Map into GGTT */ + mgr->io_base = pci_resource_start(to_pci_dev(xe->drm.dev), 2); + + /* Stolen memory is x86 only */ + mgr->stolen_base = intel_graphics_stolen_res.start; + return resource_size(&intel_graphics_stolen_res); +#else + return 0; +#endif +} + +static void xe_ttm_stolen_mgr_fini(struct drm_device *dev, void *arg) +{ + struct xe_device *xe = to_xe_device(dev); + + ttm_range_man_fini_nocheck(&xe->ttm, XE_PL_STOLEN); +} + +int xe_ttm_stolen_mgr_init(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct xe_ttm_stolen_mgr *mgr; + u64 stolen_size, io_size; + int ret; + + mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL); + if (!mgr) + return -ENOMEM; + + if (IS_SRIOV_VF(xe)) + stolen_size = 0; + else if (IS_DGFX(xe)) + stolen_size = detect_lmembar_dgfx(xe, mgr); + else if (GRAPHICS_VERx100(xe) >= 1270) + stolen_size = detect_lmembar_integrated(xe, mgr); + else + stolen_size = detect_stolen(xe, mgr); + + if (!stolen_size) { + drm_dbg_kms(&xe->drm, "No stolen memory support\n"); + return 0; + } + + /* + * We don't try to attempt partial visible support for stolen vram, + * since stolen is always at the end of vram, and the BAR size is pretty + * much always 256M, with small-bar. + */ + io_size = 0; + if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe)) + io_size = stolen_size; + + ret = ttm_range_man_init_nocheck(&xe->ttm, XE_PL_STOLEN, false, + stolen_size >> PAGE_SHIFT); + if (ret) + return ret; + + xe->mem.stolen_mgr = mgr; + + drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n", + stolen_size); + + if (io_size) + mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, io_size); + + return drmm_add_action_or_reset(&xe->drm, xe_ttm_stolen_mgr_fini, mgr); +} + +u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; + + XE_WARN_ON(!mgr->io_base); + + if (xe_ttm_stolen_cpu_access_needs_ggtt(xe)) + return mgr->io_base + xe_bo_ggtt_addr(bo) + offset; + + /* Range allocator: res->start is in pages. */ + return mgr->io_base + (bo->ttm.resource->start << PAGE_SHIFT) + offset; +} + +static int __xe_ttm_stolen_io_mem_reserve_lmembar(struct xe_device *xe, + struct xe_ttm_stolen_mgr *mgr, + struct ttm_resource *mem) +{ + if (!mgr->io_base) + return -EIO; + + /* Range allocator always produces contiguous allocations. */ + mem->bus.offset = mem->start << PAGE_SHIFT; + + drm_WARN_ON(&xe->drm, !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)); + + if (mem->placement & TTM_PL_FLAG_CONTIGUOUS && mgr->mapping) + mem->bus.addr = (u8 __force *)mgr->mapping + mem->bus.offset; + + mem->bus.offset += mgr->io_base; + mem->bus.is_iomem = true; + mem->bus.caching = ttm_write_combined; + + return 0; +} + +static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe, + struct xe_ttm_stolen_mgr *mgr, + struct ttm_resource *mem) +{ +#ifdef CONFIG_X86 + struct xe_bo *bo = ttm_to_xe_bo(mem->bo); + + XE_WARN_ON(IS_DGFX(xe)); + + /* XXX: Require BO to be mapped to GGTT? */ + if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_FLAG_GGTT))) + return -EIO; + + /* GGTT is always contiguously mapped */ + mem->bus.offset = xe_bo_ggtt_addr(bo) + mgr->io_base; + + mem->bus.is_iomem = true; + mem->bus.caching = ttm_write_combined; + + return 0; +#else + /* How is it even possible to get here without gen12 stolen? */ + drm_WARN_ON(&xe->drm, 1); + return -EIO; +#endif +} + +int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) +{ + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; + + if (!mgr || !mgr->io_base) + return -EIO; + + if (xe_ttm_stolen_cpu_access_needs_ggtt(xe)) + return __xe_ttm_stolen_io_mem_reserve_stolen(xe, mgr, mem); + else + return __xe_ttm_stolen_io_mem_reserve_lmembar(xe, mgr, mem); +} + +u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe) +{ + return xe->mem.stolen_mgr->stolen_base; +} diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h new file mode 100644 index 000000000..0675106d5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTM_STOLEN_MGR_H_ +#define _XE_TTM_STOLEN_MGR_H_ + +#include + +struct ttm_resource; +struct xe_bo; +struct xe_device; + +/** + * struct xe_ttm_stolen_mgr - Xe TTM stolen memory manager + */ +struct xe_ttm_stolen_mgr { + /** @io_base: PCI base offset for CPU I/O access */ + resource_size_t io_base; + /** @stolen_base: GPU base offset */ + resource_size_t stolen_base; + /** @mapping: I/O memory mapping for CPU access */ + void __iomem *mapping; +}; + +int xe_ttm_stolen_mgr_init(struct xe_device *xe); +int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem); +bool xe_ttm_stolen_cpu_access_needs_ggtt(struct xe_device *xe); +u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset); +u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c b/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c new file mode 100644 index 000000000..9d77ba605 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2022 Intel Corporation + * Copyright (C) 2021-2022 Red Hat + */ + +#include "xe_ttm_sys_mgr.h" + +#include + +#include +#include +#include + +#include "xe_bo.h" + +struct xe_ttm_sys_node { + struct ttm_buffer_object *tbo; + struct ttm_range_mgr_node base; +}; + +static inline struct xe_ttm_sys_node * +to_xe_ttm_sys_node(struct ttm_resource *res) +{ + return container_of(res, struct xe_ttm_sys_node, base.base); +} + +static int xe_ttm_sys_mgr_new(struct ttm_resource_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct xe_ttm_sys_node *node; + int r; + + node = kzalloc_flex(*node, base.mm_nodes, 1); + if (!node) + return -ENOMEM; + + node->tbo = tbo; + ttm_resource_init(tbo, place, &node->base.base); + + if (!(place->flags & TTM_PL_FLAG_TEMPORARY) && + ttm_resource_manager_usage(man) > (man->size << PAGE_SHIFT)) { + r = -ENOSPC; + goto err_fini; + } + + node->base.mm_nodes[0].start = 0; + node->base.mm_nodes[0].size = PFN_UP(node->base.base.size); + node->base.base.start = XE_BO_INVALID_OFFSET; + + *res = &node->base.base; + + return 0; + +err_fini: + ttm_resource_fini(man, &node->base.base); + kfree(node); + return r; +} + +static void xe_ttm_sys_mgr_del(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct xe_ttm_sys_node *node = to_xe_ttm_sys_node(res); + + ttm_resource_fini(man, res); + kfree(node); +} + +static void xe_ttm_sys_mgr_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + /* + * This function is called by debugfs entry and would require + * pm_runtime_{get,put} wrappers around any operation. + */ +} + +static const struct ttm_resource_manager_func xe_ttm_sys_mgr_func = { + .alloc = xe_ttm_sys_mgr_new, + .free = xe_ttm_sys_mgr_del, + .debug = xe_ttm_sys_mgr_debug +}; + +static void xe_ttm_sys_mgr_fini(struct drm_device *drm, void *arg) +{ + struct xe_device *xe = (struct xe_device *)arg; + struct ttm_resource_manager *man = &xe->mem.sys_mgr; + int err; + + ttm_resource_manager_set_used(man, false); + + err = ttm_resource_manager_evict_all(&xe->ttm, man); + if (err) + return; + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(&xe->ttm, XE_PL_TT, NULL); +} + +int xe_ttm_sys_mgr_init(struct xe_device *xe) +{ + struct ttm_resource_manager *man = &xe->mem.sys_mgr; + struct sysinfo si; + u64 gtt_size; + + si_meminfo(&si); + /* Potentially restrict amount of TT memory here. */ + gtt_size = (u64)si.totalram * si.mem_unit; + + man->use_tt = true; + man->func = &xe_ttm_sys_mgr_func; + ttm_resource_manager_init(man, &xe->ttm, gtt_size >> PAGE_SHIFT); + ttm_set_driver_manager(&xe->ttm, XE_PL_TT, man); + ttm_resource_manager_set_used(man, true); + return drmm_add_action_or_reset(&xe->drm, xe_ttm_sys_mgr_fini, xe); +} diff --git a/drivers/gpu/drm/xe/xe_ttm_sys_mgr.h b/drivers/gpu/drm/xe/xe_ttm_sys_mgr.h new file mode 100644 index 000000000..e8f5cd395 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_sys_mgr.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_TTM_SYS_MGR_H_ +#define _XE_TTM_SYS_MGR_H_ + +struct xe_device; + +int xe_ttm_sys_mgr_init(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c new file mode 100644 index 000000000..05911904c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2022 Intel Corporation + * Copyright (C) 2021-2022 Red Hat + */ + +#include + +#include +#include +#include + +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_pm.h" +#include "xe_res_cursor.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vram_types.h" + +static inline struct gpu_buddy_block * +xe_ttm_vram_mgr_first_block(struct list_head *list) +{ + return list_first_entry_or_null(list, struct gpu_buddy_block, link); +} + +static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm, + struct list_head *head) +{ + struct gpu_buddy_block *block; + u64 start, size; + + block = xe_ttm_vram_mgr_first_block(head); + if (!block) + return false; + + while (head != block->link.next) { + start = gpu_buddy_block_offset(block); + size = gpu_buddy_block_size(mm, block); + + block = list_entry(block->link.next, struct gpu_buddy_block, + link); + if (start + size != gpu_buddy_block_offset(block)) + return false; + } + + return true; +} + +static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + struct xe_ttm_vram_mgr_resource *vres; + struct gpu_buddy *mm = &mgr->mm; + u64 size, min_page_size; + unsigned long lpfn; + int err; + + lpfn = place->lpfn; + if (!lpfn || lpfn > man->size >> PAGE_SHIFT) + lpfn = man->size >> PAGE_SHIFT; + + if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn)) + return -E2BIG; /* don't trigger eviction for the impossible */ + + vres = kzalloc_obj(*vres); + if (!vres) + return -ENOMEM; + + ttm_resource_init(tbo, place, &vres->base); + + /* bail out quickly if there's likely not enough VRAM for this BO */ + if (ttm_resource_manager_usage(man) > man->size) { + err = -ENOSPC; + goto error_fini; + } + + INIT_LIST_HEAD(&vres->blocks); + + if (place->flags & TTM_PL_FLAG_TOPDOWN) + vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) + vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; + + if (place->fpfn || lpfn != man->size >> PAGE_SHIFT) + vres->flags |= GPU_BUDDY_RANGE_ALLOCATION; + + if (WARN_ON(!vres->base.size)) { + err = -EINVAL; + goto error_fini; + } + size = vres->base.size; + + min_page_size = mgr->default_page_size; + if (tbo->page_alignment) + min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT; + + if (WARN_ON(min_page_size < mm->chunk_size)) { + err = -EINVAL; + goto error_fini; + } + + if (WARN_ON(!IS_ALIGNED(size, min_page_size))) { + err = -EINVAL; + goto error_fini; + } + + mutex_lock(&mgr->lock); + if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) { + err = -ENOSPC; + goto error_unlock; + } + + err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, + (u64)lpfn << PAGE_SHIFT, size, + min_page_size, &vres->blocks, vres->flags); + if (err) + goto error_unlock; + + if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { + vres->used_visible_size = size; + } else { + struct gpu_buddy_block *block; + + list_for_each_entry(block, &vres->blocks, link) { + u64 start = gpu_buddy_block_offset(block); + + if (start < mgr->visible_size) { + u64 end = start + gpu_buddy_block_size(mm, block); + + vres->used_visible_size += + min(end, mgr->visible_size) - start; + } + } + } + + mgr->visible_avail -= vres->used_visible_size; + mutex_unlock(&mgr->lock); + + if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && + xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks)) + vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; + + /* + * For some kernel objects we still rely on the start when io mapping + * the object. + */ + if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) { + struct gpu_buddy_block *block = list_first_entry(&vres->blocks, + typeof(*block), + link); + + vres->base.start = gpu_buddy_block_offset(block) >> PAGE_SHIFT; + } else { + vres->base.start = XE_BO_INVALID_OFFSET; + } + + *res = &vres->base; + return 0; +error_unlock: + mutex_unlock(&mgr->lock); +error_fini: + ttm_resource_fini(man, &vres->base); + kfree(vres); + + return err; +} + +static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct xe_ttm_vram_mgr_resource *vres = + to_xe_ttm_vram_mgr_resource(res); + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + struct gpu_buddy *mm = &mgr->mm; + + mutex_lock(&mgr->lock); + gpu_buddy_free_list(mm, &vres->blocks, 0); + mgr->visible_avail += vres->used_visible_size; + mutex_unlock(&mgr->lock); + + ttm_resource_fini(man, res); + + kfree(vres); +} + +static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + struct gpu_buddy *mm = &mgr->mm; + + mutex_lock(&mgr->lock); + drm_printf(printer, "default_page_size: %lluKiB\n", + mgr->default_page_size >> 10); + drm_printf(printer, "visible_avail: %lluMiB\n", + (u64)mgr->visible_avail >> 20); + drm_printf(printer, "visible_size: %lluMiB\n", + (u64)mgr->visible_size >> 20); + + drm_buddy_print(mm, printer); + mutex_unlock(&mgr->lock); + drm_printf(printer, "man size:%llu\n", man->size); +} + +static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + struct xe_ttm_vram_mgr_resource *vres = + to_xe_ttm_vram_mgr_resource(res); + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; + + if (!place->fpfn && !place->lpfn) + return true; + + if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) + return vres->used_visible_size > 0; + + list_for_each_entry(block, &vres->blocks, link) { + unsigned long fpfn = + gpu_buddy_block_offset(block) >> PAGE_SHIFT; + unsigned long lpfn = fpfn + + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); + + if (place->fpfn < lpfn && place->lpfn > fpfn) + return true; + } + + return false; +} + +static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + struct xe_ttm_vram_mgr_resource *vres = + to_xe_ttm_vram_mgr_resource(res); + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; + + if (!place->fpfn && !place->lpfn) + return true; + + if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) + return vres->used_visible_size == size; + + list_for_each_entry(block, &vres->blocks, link) { + unsigned long fpfn = + gpu_buddy_block_offset(block) >> PAGE_SHIFT; + unsigned long lpfn = fpfn + + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); + + if (fpfn < place->fpfn || lpfn > place->lpfn) + return false; + } + + return true; +} + +static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = { + .alloc = xe_ttm_vram_mgr_new, + .free = xe_ttm_vram_mgr_del, + .intersects = xe_ttm_vram_mgr_intersects, + .compatible = xe_ttm_vram_mgr_compatible, + .debug = xe_ttm_vram_mgr_debug +}; + +static const struct dmem_cgroup_ops xe_ttm_vram_mgr_dmem_ops; + +static int xe_ttm_vram_mgr_dmem_reclaim(struct dmem_cgroup_pool_state *pool, + u64 target_bytes, void *priv) +{ + struct ttm_resource_manager *man = priv; + struct xe_device *xe = ttm_to_xe_device(man->bdev); + int ret, idx; + + if (!drm_dev_enter(&xe->drm, &idx)) + return -ENODEV; + + { + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); + if (ret >= 0) + ret = ttm_resource_manager_dmem_reclaim(pool, target_bytes, priv); + } + + drm_dev_exit(idx); + return ret; +} + +static const struct dmem_cgroup_ops xe_ttm_vram_mgr_dmem_ops = { + .reclaim = xe_ttm_vram_mgr_dmem_reclaim, +}; + +static void xe_ttm_vram_mgr_set_unused(struct drm_device *dev, void *arg) +{ + struct ttm_resource_manager *man = arg; + + ttm_resource_manager_set_used(man, false); +} + +static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_ttm_vram_mgr *mgr = arg; + struct ttm_resource_manager *man = &mgr->manager; + + if (ttm_resource_manager_evict_all(&xe->ttm, man)) + return; + + WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size); + + gpu_buddy_fini(&mgr->mm); + + ttm_resource_manager_cleanup(&mgr->manager); + + ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); +} + +int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, + u32 mem_type, u64 size, u64 io_size, + u64 default_page_size) +{ + struct ttm_resource_manager *man = &mgr->manager; + struct dmem_cgroup_region *cg; + const char *name; + int err; + + man->func = &xe_ttm_vram_mgr_func; + mgr->mem_type = mem_type; + err = drmm_mutex_init(&xe->drm, &mgr->lock); + if (err) + return err; + mgr->default_page_size = default_page_size; + mgr->visible_size = io_size; + mgr->visible_avail = io_size; + + ttm_resource_manager_init(man, &xe->ttm, size); + err = gpu_buddy_init(&mgr->mm, man->size, default_page_size); + if (err) + return err; + + gpu_buddy_driver_set_lock(&mgr->mm, &mgr->lock); + ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); + + /* + * Register the fini action before the cgroup region so that devres + * LIFO teardown runs unregister_region before manager teardown + * (draining any in-flight reclaim callbacks) and the manager fini second. + */ + err = drmm_add_action_or_reset(&xe->drm, xe_ttm_vram_mgr_fini, mgr); + if (err) + return err; + + name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; + cg = drmm_cgroup_register_region(&xe->drm, name, + &(struct dmem_cgroup_init){ + .size = size, + .ops = &xe_ttm_vram_mgr_dmem_ops, + .reclaim_priv = man, + }); + if (IS_ERR(cg)) + return PTR_ERR(cg); + + ttm_resource_manager_set_dmem_region(man, cg); + ttm_resource_manager_set_used(man, true); + + return drmm_add_action_or_reset(&xe->drm, xe_ttm_vram_mgr_set_unused, man); +} + +/** + * xe_ttm_vram_mgr_init - initialize TTM VRAM region + * @xe: pointer to Xe device + * @vram: pointer to xe_vram_region that contains the memory region attributes + * + * Initialize the Xe TTM for given @vram region using the given parameters. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram) +{ + return __xe_ttm_vram_mgr_init(xe, &vram->ttm, vram->placement, + xe_vram_region_usable_size(vram), + xe_vram_region_io_size(vram), + PAGE_SIZE); +} + +int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, + struct ttm_resource *res, + u64 offset, u64 length, + struct device *dev, + enum dma_data_direction dir, + struct sg_table **sgt) +{ + struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0]; + struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); + struct xe_res_cursor cursor; + struct scatterlist *sg; + int num_entries = 0; + int i, r; + + if (vres->used_visible_size < res->size) + return -EOPNOTSUPP; + + *sgt = kmalloc_obj(**sgt); + if (!*sgt) + return -ENOMEM; + + /* Determine the number of GPU_BUDDY blocks to export */ + xe_res_first(res, offset, length, &cursor); + while (cursor.remaining) { + num_entries++; + /* Limit maximum size to 2GiB due to SG table limitations. */ + xe_res_next(&cursor, min_t(u64, cursor.size, SZ_2G)); + } + + r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); + if (r) + goto error_free; + + /* Initialize scatterlist nodes of sg_table */ + for_each_sgtable_sg((*sgt), sg, i) + sg->length = 0; + + /* + * Walk down GPU_BUDDY blocks to populate scatterlist nodes + * @note: Use iterator api to get first the GPU_BUDDY block + * and the number of bytes from it. Access the following + * GPU_BUDDY block(s) if more buffer needs to exported + */ + xe_res_first(res, offset, length, &cursor); + for_each_sgtable_sg((*sgt), sg, i) { + phys_addr_t phys = cursor.start + xe_vram_region_io_start(tile->mem.vram); + size_t size = min_t(u64, cursor.size, SZ_2G); + dma_addr_t addr; + + addr = dma_map_resource(dev, phys, size, dir, + DMA_ATTR_SKIP_CPU_SYNC); + r = dma_mapping_error(dev, addr); + if (r) + goto error_unmap; + + sg_set_page(sg, NULL, size, 0); + sg_dma_address(sg) = addr; + sg_dma_len(sg) = size; + + xe_res_next(&cursor, size); + } + + return 0; + +error_unmap: + for_each_sgtable_sg((*sgt), sg, i) { + if (!sg->length) + continue; + + dma_unmap_resource(dev, sg->dma_address, + sg->length, dir, + DMA_ATTR_SKIP_CPU_SYNC); + } + sg_free_table(*sgt); + +error_free: + kfree(*sgt); + return r; +} + +void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, + struct sg_table *sgt) +{ + struct scatterlist *sg; + int i; + + for_each_sgtable_sg(sgt, sg, i) + dma_unmap_resource(dev, sg->dma_address, + sg->length, dir, + DMA_ATTR_SKIP_CPU_SYNC); + sg_free_table(sgt); + kfree(sgt); +} + +u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + + return mgr->visible_size; +} + +void xe_ttm_vram_get_used(struct ttm_resource_manager *man, + u64 *used, u64 *used_visible) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + + mutex_lock(&mgr->lock); + *used = mgr->mm.size - mgr->mm.avail; + *used_visible = mgr->visible_size - mgr->visible_avail; + mutex_unlock(&mgr->lock); +} + +u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + u64 avail; + + mutex_lock(&mgr->lock); + avail = mgr->mm.avail; + mutex_unlock(&mgr->lock); + + return avail; +} diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h new file mode 100644 index 000000000..87b7fae5e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTM_VRAM_MGR_H_ +#define _XE_TTM_VRAM_MGR_H_ + +#include "xe_ttm_vram_mgr_types.h" + +enum dma_data_direction; +struct xe_device; +struct xe_tile; +struct xe_vram_region; + +int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, + u32 mem_type, u64 size, u64 io_size, + u64 default_page_size); +int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram); +int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, + struct ttm_resource *res, + u64 offset, u64 length, + struct device *dev, + enum dma_data_direction dir, + struct sg_table **sgt); +void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, + struct sg_table *sgt); + +u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man); +u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man); +void xe_ttm_vram_get_used(struct ttm_resource_manager *man, + u64 *used, u64 *used_visible); + +static inline struct xe_ttm_vram_mgr_resource * +to_xe_ttm_vram_mgr_resource(struct ttm_resource *res) +{ + return container_of(res, struct xe_ttm_vram_mgr_resource, base); +} + +static inline struct xe_ttm_vram_mgr * +to_xe_ttm_vram_mgr(struct ttm_resource_manager *man) +{ + return container_of(man, struct xe_ttm_vram_mgr, manager); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h new file mode 100644 index 000000000..9106da056 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTM_VRAM_MGR_TYPES_H_ +#define _XE_TTM_VRAM_MGR_TYPES_H_ + +#include +#include + +/** + * struct xe_ttm_vram_mgr - Xe TTM VRAM manager + * + * Manages placement of TTM resource in VRAM. + */ +struct xe_ttm_vram_mgr { + /** @manager: Base TTM resource manager */ + struct ttm_resource_manager manager; + /** @mm: DRM buddy allocator which manages the VRAM */ + struct gpu_buddy mm; + /** @visible_size: Proped size of the CPU visible portion */ + u64 visible_size; + /** @visible_avail: CPU visible portion still unallocated */ + u64 visible_avail; + /** @default_page_size: default page size */ + u64 default_page_size; + /** @lock: protects allocations of VRAM */ + struct mutex lock; + /** @mem_type: The TTM memory type */ + u32 mem_type; +}; + +/** + * struct xe_ttm_vram_mgr_resource - Xe TTM VRAM resource + */ +struct xe_ttm_vram_mgr_resource { + /** @base: Base TTM resource */ + struct ttm_resource base; + /** @blocks: list of DRM buddy blocks */ + struct list_head blocks; + /** @used_visible_size: How many CPU visible bytes this resource is using */ + u64 used_visible_size; + /** @flags: flags associated with the resource */ + unsigned long flags; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c new file mode 100644 index 000000000..bcec40ca2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_tuning.h" + +#include + +#include +#include + +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "xe_gt_types.h" +#include "xe_platform_types.h" +#include "xe_rtp.h" +#include "xe_sriov.h" + +#undef XE_REG_MCR +#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) + +VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR( + { XE_RTP_NAME("Tuning: Blend Fill Caching Optimization Disable"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS)) + }, + { XE_RTP_NAME("Tuning: 32B Access Enable"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(XEHP_SQCM, EN_32B_ACCESS)) + }, + + /* Xe2 */ + + { XE_RTP_NAME("Tuning: L3 cache"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499)), + XE_RTP_ACTIONS(FIELD_SET(XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK, + REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f))) + }, + { XE_RTP_NAME("Tuning: L3 cache - media"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(2000, 3499)), + XE_RTP_ACTIONS(FIELD_SET(XE2LPM_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK, + REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f))) + }, + { XE_RTP_NAME("Tuning: Compression Overfetch"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499), + FUNC(xe_rtp_match_has_flat_ccs)), + XE_RTP_ACTIONS(CLR(CCCHKNREG1, ENCOMPPERFFIX), + SET(CCCHKNREG1, L3CMPCTRL)) + }, + { XE_RTP_NAME("Tuning: Compression Overfetch - media"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(CLR(XE2LPM_CCCHKNREG1, ENCOMPPERFFIX), + SET(XE2LPM_CCCHKNREG1, L3CMPCTRL)) + }, + { XE_RTP_NAME("Tuning: Enable compressible partial write overfetch in L3"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499)), + XE_RTP_ACTIONS(SET(L3SQCREG3, COMPPWOVERFETCHEN)) + }, + { XE_RTP_NAME("Tuning: Enable compressible partial write overfetch in L3 - media"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(SET(XE2LPM_L3SQCREG3, COMPPWOVERFETCHEN)) + }, + { XE_RTP_NAME("Tuning: L2 Overfetch Compressible Only"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_has_flat_ccs)), + XE_RTP_ACTIONS(SET(L3SQCREG2, + COMPMEMRD256BOVRFETCHEN)) + }, + { XE_RTP_NAME("Tuning: L2 Overfetch Compressible Only - media"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_has_flat_ccs)), + XE_RTP_ACTIONS(SET(XE2LPM_L3SQCREG2, + COMPMEMRD256BOVRFETCHEN)) + }, + { XE_RTP_NAME("Tuning: Stateless compression control"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(FIELD_SET(STATELESS_COMPRESSION_CTRL, UNIFIED_COMPRESSION_FORMAT, + REG_FIELD_PREP(UNIFIED_COMPRESSION_FORMAT, 0))) + }, + { XE_RTP_NAME("Tuning: Stateless compression control - media"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1301, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(FIELD_SET(STATELESS_COMPRESSION_CTRL, UNIFIED_COMPRESSION_FORMAT, + REG_FIELD_PREP(UNIFIED_COMPRESSION_FORMAT, 0))) + }, + { XE_RTP_NAME("Tuning: L3 RW flush all Cache"), + XE_RTP_RULES(GRAPHICS_VERSION(2004)), + XE_RTP_ACTIONS(SET(SCRATCH3_LBCF, RWFLUSHALLEN)) + }, + { XE_RTP_NAME("Tuning: L3 RW flush all cache - media"), + XE_RTP_RULES(MEDIA_VERSION(2000)), + XE_RTP_ACTIONS(SET(XE2LPM_SCRATCH3_LBCF, RWFLUSHALLEN)) + }, + + /* Xe3p */ + + { XE_RTP_NAME("Tuning: Set STLB Bank Hash Mode to 4KB"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED), + IS_INTEGRATED), + XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE, + BANK_HASH_4KB_MODE)) + }, +); +EXPORT_SYMBOL_IF_KUNIT(gt_tunings); + +static const struct xe_rtp_table_sr engine_tunings = XE_RTP_TABLE_SR( + { XE_RTP_NAME("Tuning: L3 Hashing Mask"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(CLR(XELP_GARBCNTL, XELP_BUS_HASH_CTL_BIT_EXC)) + }, + { XE_RTP_NAME("Tuning: Set Indirect State Override"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(SAMPLER_MODE, INDIRECT_STATE_BASE_ADDR_OVERRIDE)) + }, + { XE_RTP_NAME("Tuning: Disable NULL query for Anyhit Shader"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(RT_CTRL, DIS_NULL_QUERY)) + }, + { XE_RTP_NAME("Tuning: disable HW reporting of ctx switch to GHWSP"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), + GHWSP_CSB_REPORT_DIS, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("Tuning: TileY 2x2 Walk"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN2, TILEY_LOCALID)) + }, +); + +static const struct xe_rtp_table_sr lrc_tunings = XE_RTP_TABLE_SR( + { XE_RTP_NAME("Tuning: Windower HW Filtering"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3599), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, HW_FILTERING)) + }, + + /* DG2 */ + + { XE_RTP_NAME("Tuning: L3 cache"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(FIELD_SET(XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK, + REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f))) + }, + { XE_RTP_NAME("Tuning: TDS gang timer"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + /* read verification is ignored as in i915 - need to check enabling */ + XE_RTP_ACTIONS(FIELD_SET_NO_READ_MASK(XEHP_FF_MODE2, + FF_MODE2_TDS_TIMER_MASK, + FF_MODE2_TDS_TIMER_128)) + }, + { XE_RTP_NAME("Tuning: TBIMR fast clip"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(CHICKEN_RASTER_2, TBIMR_FAST_CLIP)) + }, + + /* Xe_LPG */ + + { XE_RTP_NAME("Tuning: L3 cache"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(FIELD_SET(XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK, + REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f))) + }, + + /* Xe2_HPG */ + + { XE_RTP_NAME("Tuning: vs hit max value"), + XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(FIELD_SET(FF_MODE, VS_HIT_MAX_VALUE_MASK, + REG_FIELD_PREP(VS_HIT_MAX_VALUE_MASK, 0x3f))) + }, +); + +/** + * xe_tuning_init - initialize gt with tunings bookkeeping + * @gt: GT instance to initialize + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_tuning_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + size_t n_lrc, n_engine, n_gt, total; + unsigned long *p; + + n_gt = BITS_TO_LONGS(gt_tunings.n_entries); + n_engine = BITS_TO_LONGS(engine_tunings.n_entries); + n_lrc = BITS_TO_LONGS(lrc_tunings.n_entries); + total = n_gt + n_engine + n_lrc; + + p = drmm_kzalloc(&xe->drm, sizeof(*p) * total, GFP_KERNEL); + if (!p) + return -ENOMEM; + + gt->tuning_active.gt = p; + p += n_gt; + gt->tuning_active.engine = p; + p += n_engine; + gt->tuning_active.lrc = p; + + return 0; +} +ALLOW_ERROR_INJECTION(xe_tuning_init, ERRNO); /* See xe_pci_probe() */ + +void xe_tuning_process_gt(struct xe_gt *gt) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, + gt->tuning_active.gt, + gt_tunings.n_entries); + xe_rtp_process_to_sr(&ctx, >_tunings, >->reg_sr, false); +} +EXPORT_SYMBOL_IF_KUNIT(xe_tuning_process_gt); + +void xe_tuning_process_engine(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, + hwe->gt->tuning_active.engine, + engine_tunings.n_entries); + xe_rtp_process_to_sr(&ctx, &engine_tunings, &hwe->reg_sr, false); +} +EXPORT_SYMBOL_IF_KUNIT(xe_tuning_process_engine); + +/** + * xe_tuning_process_lrc - process lrc tunings + * @hwe: engine instance to process tunings for + * + * Process LRC table for this platform, saving in @hwe all the tunings that need + * to be applied on context restore. These are tunings touching registers that + * are part of the HW context image. + */ +void xe_tuning_process_lrc(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, + hwe->gt->tuning_active.lrc, + lrc_tunings.n_entries); + xe_rtp_process_to_sr(&ctx, &lrc_tunings, &hwe->reg_lrc, true); +} + +/** + * xe_tuning_dump() - Dump GT tuning info into a drm printer. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Return: always 0. + */ +int xe_tuning_dump(struct xe_gt *gt, struct drm_printer *p) +{ + size_t idx; + + drm_printf(p, "GT Tunings\n"); + for_each_set_bit(idx, gt->tuning_active.gt, gt_tunings.n_entries) + drm_printf_indent(p, 1, "%s\n", gt_tunings.entries[idx].name); + + drm_puts(p, "\n"); + drm_printf(p, "Engine Tunings\n"); + for_each_set_bit(idx, gt->tuning_active.engine, engine_tunings.n_entries) + drm_printf_indent(p, 1, "%s\n", engine_tunings.entries[idx].name); + + drm_puts(p, "\n"); + drm_printf(p, "LRC Tunings\n"); + for_each_set_bit(idx, gt->tuning_active.lrc, lrc_tunings.n_entries) + drm_printf_indent(p, 1, "%s\n", lrc_tunings.entries[idx].name); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_tuning.h b/drivers/gpu/drm/xe/xe_tuning.h new file mode 100644 index 000000000..869564e3e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tuning.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TUNING_H_ +#define _XE_TUNING_H_ + +#include + +struct drm_printer; +struct xe_gt; +struct xe_hw_engine; + +int xe_tuning_init(struct xe_gt *gt); +void xe_tuning_process_gt(struct xe_gt *gt); +void xe_tuning_process_engine(struct xe_hw_engine *hwe); +void xe_tuning_process_lrc(struct xe_hw_engine *hwe); +int xe_tuning_dump(struct xe_gt *gt, struct drm_printer *p); + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +extern const struct xe_rtp_table_sr gt_tunings; +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c new file mode 100644 index 000000000..65f59f061 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_uc.h" + +#include "xe_assert.h" +#include "xe_device.h" +#include "xe_gsc.h" +#include "xe_gt.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_vf.h" +#include "xe_guc.h" +#include "xe_guc_pc.h" +#include "xe_guc_rc.h" +#include "xe_guc_engine_activity.h" +#include "xe_guc_submit.h" +#include "xe_huc.h" +#include "xe_sriov.h" +#include "xe_wopcm.h" + +static struct xe_gt * +uc_to_gt(struct xe_uc *uc) +{ + return container_of(uc, struct xe_gt, uc); +} + +static struct xe_device * +uc_to_xe(struct xe_uc *uc) +{ + return gt_to_xe(uc_to_gt(uc)); +} + +/* Should be called once at driver load only */ +int xe_uc_init_noalloc(struct xe_uc *uc) +{ + int ret; + + ret = xe_guc_init_noalloc(&uc->guc); + if (ret) + goto err; + + /* HuC and GSC have no early dependencies and will be initialized during xe_uc_init(). */ + return 0; + +err: + xe_gt_err(uc_to_gt(uc), "Failed to early initialize uC (%pe)\n", ERR_PTR(ret)); + return ret; +} + +int xe_uc_init(struct xe_uc *uc) +{ + int ret; + + /* + * We call the GuC/HuC/GSC init functions even if GuC submission is off + * to correctly move our tracking of the FW state to "disabled". + */ + ret = xe_guc_init(&uc->guc); + if (ret) + goto err; + + ret = xe_huc_init(&uc->huc); + if (ret) + goto err; + + ret = xe_gsc_init(&uc->gsc); + if (ret) + goto err; + + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return 0; + + if (!IS_SRIOV_VF(uc_to_xe(uc))) { + ret = xe_wopcm_init(&uc->wopcm); + if (ret) + goto err; + } + + ret = xe_guc_min_load_for_hwconfig(&uc->guc); + if (ret) + goto err; + + return 0; +err: + xe_gt_err(uc_to_gt(uc), "Failed to initialize uC (%pe)\n", ERR_PTR(ret)); + return ret; +} + +/** + * xe_uc_init_post_hwconfig - init Uc post hwconfig load + * @uc: The UC object + * + * Return: 0 on success, negative error code on error. + */ +int xe_uc_init_post_hwconfig(struct xe_uc *uc) +{ + int err; + + /* GuC submission not enabled, nothing to do */ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return 0; + + err = xe_uc_sanitize_reset(uc); + if (err) + return err; + + err = xe_guc_init_post_hwconfig(&uc->guc); + if (err) + return err; + + err = xe_huc_init_post_hwconfig(&uc->huc); + if (err) + return err; + + return xe_gsc_init_post_hwconfig(&uc->gsc); +} + +static int uc_reset(struct xe_uc *uc) +{ + struct xe_device *xe = uc_to_xe(uc); + int ret; + + ret = xe_guc_reset(&uc->guc); + if (ret) { + drm_err(&xe->drm, "Failed to reset GuC, ret = %d\n", ret); + return ret; + } + + return 0; +} + +static void xe_uc_sanitize(struct xe_uc *uc) +{ + xe_huc_sanitize(&uc->huc); + xe_guc_sanitize(&uc->guc); +} + +int xe_uc_sanitize_reset(struct xe_uc *uc) +{ + xe_uc_sanitize(uc); + + return uc_reset(uc); +} + +static int vf_uc_load_hw(struct xe_uc *uc) +{ + int err; + + err = xe_uc_sanitize_reset(uc); + if (err) + return err; + + err = xe_guc_enable_communication(&uc->guc); + if (err) + return err; + + err = xe_gt_sriov_vf_connect(uc_to_gt(uc)); + if (err) + return err; + + err = xe_guc_opt_in_features_enable(&uc->guc); + if (err) + return err; + + err = xe_guc_submit_enable(&uc->guc); + if (err) + return err; + + err = xe_gt_record_default_lrcs(uc_to_gt(uc)); + if (err) + return err; + + return 0; +} + +/* + * Should be called during driver load, after every GT reset, and after every + * suspend to reload / auth the firmwares. + */ +int xe_uc_load_hw(struct xe_uc *uc) +{ + int ret; + + /* GuC submission not enabled, nothing to do */ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return 0; + + if (IS_SRIOV_VF(uc_to_xe(uc))) + return vf_uc_load_hw(uc); + + ret = xe_huc_upload(&uc->huc); + if (ret) + return ret; + + ret = xe_guc_upload(&uc->guc); + if (ret) + return ret; + + ret = xe_guc_enable_communication(&uc->guc); + if (ret) + return ret; + + ret = xe_gt_record_default_lrcs(uc_to_gt(uc)); + if (ret) + return ret; + + ret = xe_guc_post_load_init(&uc->guc); + if (ret) + return ret; + + ret = xe_guc_pc_start(&uc->guc.pc); + if (ret) + return ret; + + ret = xe_guc_rc_enable(&uc->guc); + if (ret) + return ret; + + xe_guc_engine_activity_enable_stats(&uc->guc); + + /* We don't fail the driver load if HuC fails to auth */ + ret = xe_huc_auth(&uc->huc, XE_HUC_AUTH_VIA_GUC); + if (ret) + xe_gt_err(uc_to_gt(uc), + "HuC authentication failed (%pe), continuing with no HuC\n", + ERR_PTR(ret)); + + /* GSC load is async */ + xe_gsc_load_start(&uc->gsc); + + return 0; +} + +int xe_uc_reset_prepare(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return 0; + + return xe_guc_reset_prepare(&uc->guc); +} + +void xe_uc_stop_prepare(struct xe_uc *uc) +{ + xe_gsc_stop_prepare(&uc->gsc); + xe_guc_stop_prepare(&uc->guc); +} + +void xe_uc_stop(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return; + + xe_guc_stop(&uc->guc); +} + +int xe_uc_start(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return 0; + + return xe_guc_start(&uc->guc); +} + +static void uc_reset_wait(struct xe_uc *uc) +{ + int ret; + +again: + xe_guc_reset_wait(&uc->guc); + + ret = xe_uc_reset_prepare(uc); + if (ret) + goto again; +} + +void xe_uc_suspend_prepare(struct xe_uc *uc) +{ + xe_gsc_wait_for_worker_completion(&uc->gsc); + xe_guc_stop_prepare(&uc->guc); +} + +int xe_uc_suspend(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return 0; + + uc_reset_wait(uc); + + xe_uc_stop(uc); + + return xe_guc_suspend(&uc->guc); +} + +/** + * xe_uc_runtime_suspend() - UC runtime suspend + * @uc: the UC object + * + * Runtime suspend all UCs. + */ +void xe_uc_runtime_suspend(struct xe_uc *uc) +{ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return; + + xe_guc_runtime_suspend(&uc->guc); +} + +/** + * xe_uc_runtime_resume() - UC runtime resume + * @uc: the UC object + * + * Runtime resume all UCs. + */ +void xe_uc_runtime_resume(struct xe_uc *uc) +{ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return; + + xe_guc_runtime_resume(&uc->guc); +} + +/** + * xe_uc_declare_wedged() - Declare UC wedged + * @uc: the UC object + * + * Wedge the UC which stops all submission, saves desired debug state, and + * cleans up anything which could timeout. + */ +void xe_uc_declare_wedged(struct xe_uc *uc) +{ + xe_gt_assert(uc_to_gt(uc), uc_to_xe(uc)->wedged.mode); + + xe_guc_declare_wedged(&uc->guc); +} diff --git a/drivers/gpu/drm/xe/xe_uc.h b/drivers/gpu/drm/xe/xe_uc.h new file mode 100644 index 000000000..255a54a8f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_H_ +#define _XE_UC_H_ + +struct xe_uc; + +int xe_uc_init_noalloc(struct xe_uc *uc); +int xe_uc_init(struct xe_uc *uc); +int xe_uc_init_post_hwconfig(struct xe_uc *uc); +int xe_uc_load_hw(struct xe_uc *uc); +int xe_uc_reset_prepare(struct xe_uc *uc); +void xe_uc_runtime_resume(struct xe_uc *uc); +void xe_uc_runtime_suspend(struct xe_uc *uc); +void xe_uc_stop_prepare(struct xe_uc *uc); +void xe_uc_stop(struct xe_uc *uc); +int xe_uc_start(struct xe_uc *uc); +void xe_uc_suspend_prepare(struct xe_uc *uc); +int xe_uc_suspend(struct xe_uc *uc); +int xe_uc_sanitize_reset(struct xe_uc *uc); +void xe_uc_declare_wedged(struct xe_uc *uc); + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_debugfs.c b/drivers/gpu/drm/xe/xe_uc_debugfs.c new file mode 100644 index 000000000..45119993f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_debugfs.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include + +#include "xe_gsc_debugfs.h" +#include "xe_guc_debugfs.h" +#include "xe_huc_debugfs.h" +#include "xe_macros.h" +#include "xe_uc_debugfs.h" +#include "xe_uc_types.h" + +void xe_uc_debugfs_register(struct xe_uc *uc, struct dentry *parent) +{ + struct dentry *root; + + root = debugfs_create_dir("uc", parent); + if (IS_ERR(root)) { + XE_WARN_ON("Create UC directory failed"); + return; + } + + xe_gsc_debugfs_register(&uc->gsc, root); + xe_guc_debugfs_register(&uc->guc, root); + xe_huc_debugfs_register(&uc->huc, root); +} diff --git a/drivers/gpu/drm/xe/xe_uc_debugfs.h b/drivers/gpu/drm/xe/xe_uc_debugfs.h new file mode 100644 index 000000000..a13382df2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_DEBUGFS_H_ +#define _XE_UC_DEBUGFS_H_ + +struct dentry; +struct xe_uc; + +void xe_uc_debugfs_register(struct xe_uc *uc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c new file mode 100644 index 000000000..a8e6f18cc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -0,0 +1,973 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include + +#include + +#include "regs/xe_guc_regs.h" +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gsc.h" +#include "xe_gt_printk.h" +#include "xe_gt_sriov_vf.h" +#include "xe_gt_types.h" +#include "xe_guc.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_module.h" +#include "xe_sriov.h" +#include "xe_uc_fw.h" + +/* + * List of required GuC and HuC binaries per-platform. They must be ordered + * based on platform, from newer to older. + * + * Versioning follows the guidelines from + * Documentation/driver-api/firmware/firmware-usage-guidelines.rst. There is a + * distinction for platforms being officially supported by the driver or not. + * Platforms not available publicly or not yet officially supported by the + * driver (under force-probe), use the mmp_ver(): the firmware autoselect logic + * will select the firmware from disk with filename that matches the full + * "mpp version", i.e. major.minor.patch. mmp_ver() should only be used for + * this case. + * + * For platforms officially supported by the driver, the filename always only + * ever contains the major version (GuC) or no version at all (HuC). + * + * After loading the file, the driver parses the versions embedded in the blob. + * The major version needs to match a major version supported by the driver (if + * any). The minor version is also checked and a notice emitted to the log if + * the version found is smaller than the version wanted. This is done only for + * informational purposes so users may have a chance to upgrade, but the driver + * still loads and use the older firmware. + * + * Examples: + * + * 1) Platform officially supported by i915 - using Tigerlake as example. + * Driver loads the following firmware blobs from disk: + * + * - i915/tgl_guc_.bin + * - i915/tgl_huc.bin + * + * number for GuC is checked that it matches the version inside + * the blob. version is checked and if smaller than the expected + * an info message is emitted about that. + * + * 1) XE_, still under require_force_probe. Using + * "wipplat" as a short-name. Driver loads the following firmware blobs + * from disk: + * + * - xe/wipplat_guc_...bin + * - xe/wipplat_huc_...bin + * + * and are checked that they match the version inside + * the blob. Both of them need to match exactly what the driver is + * expecting, otherwise it fails. + * + * 3) Platform officially supported by xe and out of force-probe. Using + * "plat" as a short-name. Except for the different directory, the + * behavior is the same as (1). Driver loads the following firmware + * blobs from disk: + * + * - xe/plat_guc_.bin + * - xe/plat_huc.bin + * + * number for GuC is checked that it matches the version inside + * the blob. version is checked and if smaller than the expected + * an info message is emitted about that. + * + * For the platforms already released with a major version, they should never be + * removed from the table. Instead new entries with newer versions may be added + * before them, so they take precedence. + * + * TODO: Currently there's no fallback on major version. That's because xe + * driver only supports the one major version of each firmware in the table. + * This needs to be fixed when the major version of GuC is updated. + */ + +struct uc_fw_entry { + enum xe_platform platform; + enum xe_gt_type gt_type; + + struct { + const char *path; + u16 major; + u16 minor; + u16 patch; + bool full_ver_required; + }; +}; + +struct fw_blobs_by_type { + const struct uc_fw_entry *entries; + u32 count; +}; + +/* + * Add an "ANY" define just to convey the meaning it's given here. + */ +#define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED + +#define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \ + fw_def(NOVALAKE_S, GT_TYPE_ANY, major_ver(xe, guc, nvl, 70, 71, 0)) \ + fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 54, 0)) \ + fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 54, 0)) \ + fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 53, 0)) \ + fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 53, 0)) \ + fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 53, 0)) \ + fw_def(DG1, GT_TYPE_ANY, major_ver(i915, guc, dg1, 70, 44, 1)) \ + fw_def(ALDERLAKE_N, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \ + fw_def(ALDERLAKE_P, GT_TYPE_ANY, major_ver(i915, guc, adlp, 70, 44, 1)) \ + fw_def(ALDERLAKE_S, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \ + fw_def(ROCKETLAKE, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \ + fw_def(TIGERLAKE, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) + +#define XE_HUC_FIRMWARE_DEFS(fw_def, mmp_ver, no_ver) \ + fw_def(PANTHERLAKE, GT_TYPE_ANY, no_ver(xe, huc, ptl)) \ + fw_def(BATTLEMAGE, GT_TYPE_ANY, no_ver(xe, huc, bmg)) \ + fw_def(LUNARLAKE, GT_TYPE_ANY, no_ver(xe, huc, lnl)) \ + fw_def(METEORLAKE, GT_TYPE_ANY, no_ver(i915, huc_gsc, mtl)) \ + fw_def(DG1, GT_TYPE_ANY, no_ver(i915, huc, dg1)) \ + fw_def(ALDERLAKE_P, GT_TYPE_ANY, no_ver(i915, huc, tgl)) \ + fw_def(ALDERLAKE_S, GT_TYPE_ANY, no_ver(i915, huc, tgl)) \ + fw_def(ROCKETLAKE, GT_TYPE_ANY, no_ver(i915, huc, tgl)) \ + fw_def(TIGERLAKE, GT_TYPE_ANY, no_ver(i915, huc, tgl)) + +/* for the GSC FW we match the compatibility version and not the release one */ +#define XE_GSC_FIRMWARE_DEFS(fw_def, major_ver) \ + fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, gsc, ptl, 105, 1, 0)) \ + fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, gsc, lnl, 104, 1, 0)) \ + fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, gsc, mtl, 102, 1, 0)) + +#define MAKE_FW_PATH(dir__, uc__, shortname__, version__) \ + __stringify(dir__) "/" __stringify(shortname__) "_" __stringify(uc__) version__ ".bin" + +#define fw_filename_mmp_ver(dir_, uc_, shortname_, a, b, c) \ + MAKE_FW_PATH(dir_, uc_, shortname_, "_" __stringify(a ## . ## b ## . ## c)) +#define fw_filename_major_ver(dir_, uc_, shortname_, a, b, c) \ + MAKE_FW_PATH(dir_, uc_, shortname_, "_" __stringify(a)) +#define fw_filename_no_ver(dir_, uc_, shortname_) \ + MAKE_FW_PATH(dir_, uc_, shortname_, "") +#define fw_filename_gsc(dir_, uc_, shortname_, a, b, c) \ + MAKE_FW_PATH(dir_, uc_, shortname_, "_" __stringify(b)) + +#define uc_fw_entry_mmp_ver(dir_, uc_, shortname_, a, b, c) \ + { fw_filename_mmp_ver(dir_, uc_, shortname_, a, b, c), \ + a, b, c, true } +#define uc_fw_entry_major_ver(dir_, uc_, shortname_, a, b, c) \ + { fw_filename_major_ver(dir_, uc_, shortname_, a, b, c), \ + a, b, c } +#define uc_fw_entry_no_ver(dir_, uc_, shortname_) \ + { fw_filename_no_ver(dir_, uc_, shortname_), \ + 0, 0 } +#define uc_fw_entry_gsc(dir_, uc_, shortname_, a, b, c) \ + { fw_filename_gsc(dir_, uc_, shortname_, a, b, c), \ + a, b, c } + +/* All blobs need to be declared via MODULE_FIRMWARE() */ +#define XE_UC_MODULE_FIRMWARE(platform__, gt_type__, fw_filename) \ + MODULE_FIRMWARE(fw_filename); + +#define XE_UC_FW_ENTRY(platform__, gt_type__, entry__) \ + { \ + .platform = XE_ ## platform__, \ + .gt_type = XE_ ## gt_type__, \ + entry__, \ + }, + +XE_GUC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, + fw_filename_mmp_ver, fw_filename_major_ver) +XE_HUC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, + fw_filename_mmp_ver, fw_filename_no_ver) +XE_GSC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, fw_filename_gsc) + +static struct xe_gt * +__uc_fw_to_gt(struct xe_uc_fw *uc_fw, enum xe_uc_fw_type type) +{ + XE_WARN_ON(type >= XE_UC_FW_NUM_TYPES); + + switch (type) { + case XE_UC_FW_TYPE_GUC: + return container_of(uc_fw, struct xe_gt, uc.guc.fw); + case XE_UC_FW_TYPE_HUC: + return container_of(uc_fw, struct xe_gt, uc.huc.fw); + case XE_UC_FW_TYPE_GSC: + return container_of(uc_fw, struct xe_gt, uc.gsc.fw); + default: + return NULL; + } +} + +static struct xe_gt *uc_fw_to_gt(struct xe_uc_fw *uc_fw) +{ + return __uc_fw_to_gt(uc_fw, uc_fw->type); +} + +static struct xe_device *uc_fw_to_xe(struct xe_uc_fw *uc_fw) +{ + return gt_to_xe(uc_fw_to_gt(uc_fw)); +} + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status) +{ + xe_gt_dbg(uc_fw_to_gt(uc_fw), "%s %s->%s\n", + xe_uc_fw_type_repr(uc_fw->type), + xe_uc_fw_status_repr(uc_fw->status), + xe_uc_fw_status_repr(status)); + uc_fw->__status = status; +} +#endif + +static void +uc_fw_auto_select(struct xe_device *xe, struct xe_uc_fw *uc_fw) +{ + static const struct uc_fw_entry entries_guc[] = { + XE_GUC_FIRMWARE_DEFS(XE_UC_FW_ENTRY, + uc_fw_entry_mmp_ver, + uc_fw_entry_major_ver) + }; + static const struct uc_fw_entry entries_huc[] = { + XE_HUC_FIRMWARE_DEFS(XE_UC_FW_ENTRY, + uc_fw_entry_mmp_ver, + uc_fw_entry_no_ver) + }; + static const struct uc_fw_entry entries_gsc[] = { + XE_GSC_FIRMWARE_DEFS(XE_UC_FW_ENTRY, uc_fw_entry_gsc) + }; + static const struct fw_blobs_by_type blobs_all[XE_UC_FW_NUM_TYPES] = { + [XE_UC_FW_TYPE_GUC] = { entries_guc, ARRAY_SIZE(entries_guc) }, + [XE_UC_FW_TYPE_HUC] = { entries_huc, ARRAY_SIZE(entries_huc) }, + [XE_UC_FW_TYPE_GSC] = { entries_gsc, ARRAY_SIZE(entries_gsc) }, + }; + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + enum xe_platform p = xe->info.platform; + const struct uc_fw_entry *entries; + u32 count; + int i; + + xe_gt_assert(gt, uc_fw->type < ARRAY_SIZE(blobs_all)); + xe_gt_assert(gt, gt->info.type != XE_GT_TYPE_UNINITIALIZED); + + entries = blobs_all[uc_fw->type].entries; + count = blobs_all[uc_fw->type].count; + + for (i = 0; i < count && p <= entries[i].platform; i++) { + if (p != entries[i].platform) + continue; + + if (entries[i].gt_type != XE_GT_TYPE_ANY && + entries[i].gt_type != gt->info.type) + continue; + + uc_fw->path = entries[i].path; + uc_fw->versions.wanted.major = entries[i].major; + uc_fw->versions.wanted.minor = entries[i].minor; + uc_fw->versions.wanted.patch = entries[i].patch; + uc_fw->full_ver_required = entries[i].full_ver_required; + + if (uc_fw->type == XE_UC_FW_TYPE_GSC) + uc_fw->versions.wanted_type = XE_UC_FW_VER_COMPATIBILITY; + else + uc_fw->versions.wanted_type = XE_UC_FW_VER_RELEASE; + + break; + } +} + +static void +uc_fw_override(struct xe_uc_fw *uc_fw) +{ + char *path_override = NULL; + + /* empty string disables, but it's not allowed for GuC */ + switch (uc_fw->type) { + case XE_UC_FW_TYPE_GUC: + if (xe_modparam.guc_firmware_path && *xe_modparam.guc_firmware_path) + path_override = xe_modparam.guc_firmware_path; + break; + case XE_UC_FW_TYPE_HUC: + path_override = xe_modparam.huc_firmware_path; + break; + case XE_UC_FW_TYPE_GSC: + path_override = xe_modparam.gsc_firmware_path; + break; + default: + break; + } + + if (path_override) { + uc_fw->path = path_override; + uc_fw->user_overridden = true; + } +} + +/** + * xe_uc_fw_copy_rsa - copy fw RSA to buffer + * + * @uc_fw: uC firmware + * @dst: dst buffer + * @max_len: max number of bytes to copy + * + * Return: number of copied bytes. + */ +size_t xe_uc_fw_copy_rsa(struct xe_uc_fw *uc_fw, void *dst, u32 max_len) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + u32 size = min_t(u32, uc_fw->rsa_size, max_len); + + xe_assert(xe, !(size % 4)); + xe_assert(xe, xe_uc_fw_is_available(uc_fw)); + + xe_map_memcpy_from(xe, dst, &uc_fw->bo->vmap, + xe_uc_fw_rsa_offset(uc_fw), size); + + return size; +} + +static void uc_fw_fini(struct drm_device *drm, void *arg) +{ + struct xe_uc_fw *uc_fw = arg; + + if (!xe_uc_fw_is_available(uc_fw)) + return; + + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_SELECTED); +} + +static int guc_read_css_info(struct xe_uc_fw *uc_fw, struct uc_css_guc_info *guc_info) +{ + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct xe_uc_fw_version *release = &uc_fw->versions.found[XE_UC_FW_VER_RELEASE]; + struct xe_uc_fw_version *compatibility = &uc_fw->versions.found[XE_UC_FW_VER_COMPATIBILITY]; + + xe_gt_assert(gt, uc_fw->type == XE_UC_FW_TYPE_GUC); + + /* We don't support GuC releases older than 70.29.2 */ + if (MAKE_GUC_VER_STRUCT(*release) < MAKE_GUC_VER(70, 29, 2)) { + xe_gt_err(gt, "Unsupported GuC v%u.%u.%u! v70.29.2 or newer is required\n", + release->major, release->minor, release->patch); + return -EINVAL; + } + + compatibility->major = FIELD_GET(CSS_SW_VERSION_UC_MAJOR, guc_info->submission_version); + compatibility->minor = FIELD_GET(CSS_SW_VERSION_UC_MINOR, guc_info->submission_version); + compatibility->patch = FIELD_GET(CSS_SW_VERSION_UC_PATCH, guc_info->submission_version); + + uc_fw->build_type = FIELD_GET(CSS_UKERNEL_INFO_BUILDTYPE, guc_info->ukernel_info); + uc_fw->private_data_size = guc_info->private_data_size; + + return 0; +} + +int xe_uc_fw_check_version_requirements(struct xe_uc_fw *uc_fw) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_uc_fw_version *wanted = &uc_fw->versions.wanted; + struct xe_uc_fw_version *found = &uc_fw->versions.found[uc_fw->versions.wanted_type]; + + /* Driver has no requirement on any version, any is good. */ + if (!wanted->major) + return 0; + + /* + * If full version is required, both major and minor should match. + * Otherwise, at least the major version. + */ + if (wanted->major != found->major || + (uc_fw->full_ver_required && + ((wanted->minor != found->minor) || + (wanted->patch != found->patch)))) { + drm_notice(&xe->drm, "%s firmware %s: unexpected version: %u.%u.%u != %u.%u.%u\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + found->major, found->minor, found->patch, + wanted->major, wanted->minor, wanted->patch); + goto fail; + } + + if (wanted->minor > found->minor || + (wanted->minor == found->minor && wanted->patch > found->patch)) { + drm_notice(&xe->drm, "%s firmware (%u.%u.%u) is recommended, but only (%u.%u.%u) was found in %s\n", + xe_uc_fw_type_repr(uc_fw->type), + wanted->major, wanted->minor, wanted->patch, + found->major, found->minor, found->patch, + uc_fw->path); + drm_info(&xe->drm, "Consider updating your linux-firmware pkg or downloading from %s\n", + XE_UC_FIRMWARE_URL); + } + + return 0; + +fail: + if (xe_uc_fw_is_overridden(uc_fw)) + return 0; + + return -ENOEXEC; +} + +/* Refer to the "CSS-based Firmware Layout" documentation entry for details */ +static int parse_css_header(struct xe_uc_fw *uc_fw, const void *fw_data, size_t fw_size) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_uc_fw_version *release = &uc_fw->versions.found[XE_UC_FW_VER_RELEASE]; + struct uc_css_header *css; + size_t size; + + /* Check the size of the blob before examining buffer contents */ + if (unlikely(fw_size < sizeof(struct uc_css_header))) { + drm_warn(&xe->drm, "%s firmware %s: invalid size: %zu < %zu\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + fw_size, sizeof(struct uc_css_header)); + return -ENODATA; + } + + css = (struct uc_css_header *)fw_data; + + /* Check integrity of size values inside CSS header */ + size = (css->header_size_dw - css->rsa_info.key_size_dw - css->rsa_info.modulus_size_dw - + css->rsa_info.exponent_size_dw) * sizeof(u32); + if (unlikely(size != sizeof(struct uc_css_header))) { + drm_warn(&xe->drm, + "%s firmware %s: unexpected header size: %zu != %zu\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + fw_size, sizeof(struct uc_css_header)); + return -EPROTO; + } + + /* uCode size must calculated from other sizes */ + uc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32); + + /* now RSA */ + uc_fw->rsa_size = css->rsa_info.key_size_dw * sizeof(u32); + + /* At least, it should have header, uCode and RSA. Size of all three. */ + size = sizeof(struct uc_css_header) + uc_fw->ucode_size + + uc_fw->rsa_size; + if (unlikely(fw_size < size)) { + drm_warn(&xe->drm, "%s firmware %s: invalid size: %zu < %zu\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + fw_size, size); + return -ENOEXEC; + } + + /* Get version numbers from the CSS header */ + release->major = FIELD_GET(CSS_SW_VERSION_UC_MAJOR, css->guc_info.sw_version); + release->minor = FIELD_GET(CSS_SW_VERSION_UC_MINOR, css->guc_info.sw_version); + release->patch = FIELD_GET(CSS_SW_VERSION_UC_PATCH, css->guc_info.sw_version); + + if (uc_fw->type == XE_UC_FW_TYPE_GUC) + return guc_read_css_info(uc_fw, &css->guc_info); + + return 0; +} + +static bool is_cpd_header(const void *data) +{ + const u32 *marker = data; + + return *marker == GSC_CPD_HEADER_MARKER; +} + +static u32 entry_offset(const struct gsc_cpd_header_v2 *header, const char *name) +{ + const struct gsc_cpd_entry *entry; + int i; + + entry = (void *)header + header->header_length; + + for (i = 0; i < header->num_of_entries; i++, entry++) + if (strcmp(entry->name, name) == 0) + return entry->offset & GSC_CPD_ENTRY_OFFSET_MASK; + + return 0; +} + +/* Refer to the "GSC-based Firmware Layout" documentation entry for details */ +static int parse_cpd_header(struct xe_uc_fw *uc_fw, const void *data, size_t size, + const char *manifest_entry, const char *css_entry) +{ + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct xe_device *xe = gt_to_xe(gt); + const struct gsc_cpd_header_v2 *header = data; + struct xe_uc_fw_version *release = &uc_fw->versions.found[XE_UC_FW_VER_RELEASE]; + const struct gsc_manifest_header *manifest; + size_t min_size = sizeof(*header); + u32 offset; + + /* manifest_entry is mandatory, css_entry is optional */ + xe_assert(xe, manifest_entry); + + if (size < min_size || !is_cpd_header(header)) + return -ENOENT; + + if (header->header_length < sizeof(struct gsc_cpd_header_v2)) { + xe_gt_err(gt, "invalid CPD header length %u!\n", header->header_length); + return -EINVAL; + } + + min_size = header->header_length + sizeof(struct gsc_cpd_entry) * header->num_of_entries; + if (size < min_size) { + xe_gt_err(gt, "FW too small! %zu < %zu\n", size, min_size); + return -ENODATA; + } + + /* Look for the manifest first */ + offset = entry_offset(header, manifest_entry); + if (!offset) { + xe_gt_err(gt, "Failed to find %s manifest!\n", + xe_uc_fw_type_repr(uc_fw->type)); + return -ENODATA; + } + + min_size = offset + sizeof(struct gsc_manifest_header); + if (size < min_size) { + xe_gt_err(gt, "FW too small! %zu < %zu\n", size, min_size); + return -ENODATA; + } + + manifest = data + offset; + + release->major = manifest->fw_version.major; + release->minor = manifest->fw_version.minor; + release->patch = manifest->fw_version.hotfix; + + if (uc_fw->type == XE_UC_FW_TYPE_GSC) { + struct xe_gsc *gsc = container_of(uc_fw, struct xe_gsc, fw); + + release->build = manifest->fw_version.build; + gsc->security_version = manifest->security_version; + } + + /* then optionally look for the css header */ + if (css_entry) { + int ret; + + /* + * This section does not contain a CSS entry on DG2. We + * don't support DG2 HuC right now, so no need to handle + * it, just add a reminder in case that changes. + */ + xe_assert(xe, xe->info.platform != XE_DG2); + + offset = entry_offset(header, css_entry); + + /* the CSS header parser will check that the CSS header fits */ + if (offset > size) { + xe_gt_err(gt, "FW too small! %zu < %u\n", size, offset); + return -ENODATA; + } + + ret = parse_css_header(uc_fw, data + offset, size - offset); + if (ret) + return ret; + + uc_fw->css_offset = offset; + } + + uc_fw->has_gsc_headers = true; + + return 0; +} + +static int parse_gsc_layout(struct xe_uc_fw *uc_fw, const void *data, size_t size) +{ + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + const struct gsc_layout_pointers *layout = data; + const struct gsc_bpdt_header *bpdt_header = NULL; + const struct gsc_bpdt_entry *bpdt_entry = NULL; + size_t min_size = sizeof(*layout); + int i; + + if (size < min_size) { + xe_gt_err(gt, "GSC FW too small! %zu < %zu\n", size, min_size); + return -ENODATA; + } + + min_size = layout->boot1.offset + layout->boot1.size; + if (size < min_size) { + xe_gt_err(gt, "GSC FW too small for boot section! %zu < %zu\n", + size, min_size); + return -ENODATA; + } + + min_size = sizeof(*bpdt_header); + if (layout->boot1.size < min_size) { + xe_gt_err(gt, "GSC FW boot section too small for BPDT header: %u < %zu\n", + layout->boot1.size, min_size); + return -ENODATA; + } + + bpdt_header = data + layout->boot1.offset; + if (bpdt_header->signature != GSC_BPDT_HEADER_SIGNATURE) { + xe_gt_err(gt, "invalid signature for BPDT header: 0x%08x!\n", + bpdt_header->signature); + return -EINVAL; + } + + min_size += sizeof(*bpdt_entry) * bpdt_header->descriptor_count; + if (layout->boot1.size < min_size) { + xe_gt_err(gt, "GSC FW boot section too small for BPDT entries: %u < %zu\n", + layout->boot1.size, min_size); + return -ENODATA; + } + + bpdt_entry = (void *)bpdt_header + sizeof(*bpdt_header); + for (i = 0; i < bpdt_header->descriptor_count; i++, bpdt_entry++) { + if ((bpdt_entry->type & GSC_BPDT_ENTRY_TYPE_MASK) != + GSC_BPDT_ENTRY_TYPE_GSC_RBE) + continue; + + min_size = bpdt_entry->sub_partition_offset; + + /* the CPD header parser will check that the CPD header fits */ + if (layout->boot1.size < min_size) { + xe_gt_err(gt, "GSC FW boot section too small for CPD offset: %u < %zu\n", + layout->boot1.size, min_size); + return -ENODATA; + } + + return parse_cpd_header(uc_fw, + (void *)bpdt_header + min_size, + layout->boot1.size - min_size, + "RBEP.man", NULL); + } + + xe_gt_err(gt, "couldn't find CPD header in GSC binary!\n"); + return -ENODATA; +} + +static int parse_headers(struct xe_uc_fw *uc_fw, const struct firmware *fw) +{ + int ret; + + /* + * All GuC releases and older HuC ones use CSS headers, while newer HuC + * releases use GSC CPD headers. + */ + switch (uc_fw->type) { + case XE_UC_FW_TYPE_GSC: + return parse_gsc_layout(uc_fw, fw->data, fw->size); + case XE_UC_FW_TYPE_HUC: + ret = parse_cpd_header(uc_fw, fw->data, fw->size, "HUCP.man", "huc_fw"); + if (!ret || ret != -ENOENT) + return ret; + fallthrough; + case XE_UC_FW_TYPE_GUC: + return parse_css_header(uc_fw, fw->data, fw->size); + default: + return -EINVAL; + } + + return 0; +} + +#define print_uc_fw_version(p_, version_, prefix_, ...) \ +do { \ + struct xe_uc_fw_version *ver_ = (version_); \ + if (ver_->build) \ + drm_printf(p_, prefix_ " version %u.%u.%u.%u\n", ##__VA_ARGS__, \ + ver_->major, ver_->minor, \ + ver_->patch, ver_->build); \ + else \ + drm_printf(p_, prefix_ " version %u.%u.%u\n", ##__VA_ARGS__, \ + ver_->major, ver_->minor, ver_->patch); \ +} while (0) + +static void uc_fw_vf_override(struct xe_uc_fw *uc_fw) +{ + struct xe_uc_fw_version *compat = &uc_fw->versions.found[XE_UC_FW_VER_COMPATIBILITY]; + struct xe_uc_fw_version *wanted = &uc_fw->versions.wanted; + + /* Only GuC/HuC are supported */ + if (uc_fw->type != XE_UC_FW_TYPE_GUC && uc_fw->type != XE_UC_FW_TYPE_HUC) + uc_fw->path = NULL; + + /* VF will support only firmwares that driver can autoselect */ + xe_uc_fw_change_status(uc_fw, uc_fw->path ? + XE_UC_FIRMWARE_PRELOADED : + XE_UC_FIRMWARE_NOT_SUPPORTED); + + if (!xe_uc_fw_is_supported(uc_fw)) + return; + + /* PF is doing the loading, so we don't need a path on the VF */ + uc_fw->path = "Loaded by PF"; + + /* The GuC versions are set up during the VF bootstrap */ + if (uc_fw->type == XE_UC_FW_TYPE_GUC) { + uc_fw->versions.wanted_type = XE_UC_FW_VER_COMPATIBILITY; + xe_gt_sriov_vf_guc_versions(uc_fw_to_gt(uc_fw), wanted, compat); + } +} + +static int uc_fw_request(struct xe_uc_fw *uc_fw, const struct firmware **firmware_p) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct drm_printer p = xe_gt_info_printer(gt); + struct device *dev = xe->drm.dev; + const struct firmware *fw = NULL; + int err; + + /* + * we use FIRMWARE_UNINITIALIZED to detect checks against uc_fw->status + * before we're looked at the HW caps to see if we have uc support + */ + BUILD_BUG_ON(XE_UC_FIRMWARE_UNINITIALIZED); + xe_gt_assert(gt, !uc_fw->status); + xe_gt_assert(gt, !uc_fw->path); + + uc_fw_auto_select(xe, uc_fw); + + if (IS_SRIOV_VF(xe)) { + uc_fw_vf_override(uc_fw); + return 0; + } + + uc_fw_override(uc_fw); + + xe_uc_fw_change_status(uc_fw, uc_fw->path ? + XE_UC_FIRMWARE_SELECTED : + XE_UC_FIRMWARE_NOT_SUPPORTED); + + if (!xe_uc_fw_is_supported(uc_fw)) { + if (uc_fw->type == XE_UC_FW_TYPE_GUC) { + xe_gt_err(gt, "No GuC firmware defined for platform\n"); + return -ENOENT; + } + return 0; + } + + /* an empty path means the firmware is disabled */ + if (!xe_device_uc_enabled(xe) || !(*uc_fw->path)) { + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_DISABLED); + xe_gt_dbg(gt, "%s disabled\n", xe_uc_fw_type_repr(uc_fw->type)); + return 0; + } + + err = firmware_request_nowarn(&fw, uc_fw->path, dev); + if (err) + goto fail; + + err = parse_headers(uc_fw, fw); + if (err) + goto fail; + + print_uc_fw_version(&p, + &uc_fw->versions.found[XE_UC_FW_VER_RELEASE], + "Using %s firmware from %s", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path); + + /* for GSC FW we want the compatibility version, which we query after load */ + if (uc_fw->type != XE_UC_FW_TYPE_GSC) { + err = xe_uc_fw_check_version_requirements(uc_fw); + if (err) + goto fail; + } + + *firmware_p = fw; + + return 0; + +fail: + xe_uc_fw_change_status(uc_fw, err == -ENOENT ? + XE_UC_FIRMWARE_MISSING : + XE_UC_FIRMWARE_ERROR); + + if (err == -ENOENT) + xe_gt_info(gt, "%s firmware %s not found\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path); + else + xe_gt_notice(gt, "%s firmware %s: fetch failed with error %pe\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, ERR_PTR(err)); + xe_gt_info(gt, "%s firmware(s) can be downloaded from %s\n", + xe_uc_fw_type_repr(uc_fw->type), XE_UC_FIRMWARE_URL); + + release_firmware(fw); /* OK even if fw is NULL */ + + return err; +} + +static void uc_fw_release(const struct firmware *fw) +{ + release_firmware(fw); +} + +static int uc_fw_copy(struct xe_uc_fw *uc_fw, const void *data, size_t size, u32 flags) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_bo *obj; + int err; + + obj = xe_managed_bo_create_from_data(xe, tile, data, size, flags); + if (IS_ERR(obj)) { + drm_notice(&xe->drm, "%s firmware %s: failed to create / populate bo", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path); + err = PTR_ERR(obj); + goto fail; + } + + uc_fw->bo = obj; + uc_fw->size = size; + + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_AVAILABLE); + + err = drmm_add_action_or_reset(&xe->drm, uc_fw_fini, uc_fw); + if (err) + goto fail; + + return 0; + +fail: + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_ERROR); + drm_notice(&xe->drm, "%s firmware %s: copy failed with error %d\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, err); + + return err; +} + +int xe_uc_fw_init(struct xe_uc_fw *uc_fw) +{ + const struct firmware *fw = NULL; + int err; + + err = uc_fw_request(uc_fw, &fw); + if (err) + return err; + + /* no error and no firmware means nothing to copy */ + if (!fw) + return 0; + + err = uc_fw_copy(uc_fw, fw->data, fw->size, + XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE); + + uc_fw_release(fw); + + return err; +} +ALLOW_ERROR_INJECTION(xe_uc_fw_init, ERRNO); /* See xe_pci_probe() */ + +static u32 uc_fw_ggtt_offset(struct xe_uc_fw *uc_fw) +{ + return xe_bo_ggtt_addr(uc_fw->bo); +} + +static int uc_fw_xfer(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct xe_mmio *mmio = >->mmio; + u64 src_offset; + u32 dma_ctrl; + int ret; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + /* Set the source address for the uCode */ + src_offset = uc_fw_ggtt_offset(uc_fw) + uc_fw->css_offset; + xe_mmio_write32(mmio, DMA_ADDR_0_LOW, lower_32_bits(src_offset)); + xe_mmio_write32(mmio, DMA_ADDR_0_HIGH, + upper_32_bits(src_offset) | DMA_ADDRESS_SPACE_GGTT); + + /* Set the DMA destination */ + xe_mmio_write32(mmio, DMA_ADDR_1_LOW, offset); + xe_mmio_write32(mmio, DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM); + + /* + * Set the transfer size. The header plus uCode will be copied to WOPCM + * via DMA, excluding any other components + */ + xe_mmio_write32(mmio, DMA_COPY_SIZE, + sizeof(struct uc_css_header) + uc_fw->ucode_size); + + /* Start the DMA */ + xe_mmio_write32(mmio, DMA_CTRL, + REG_MASKED_FIELD_ENABLE(dma_flags | START_DMA)); + + /* Wait for DMA to finish */ + ret = xe_mmio_wait32(mmio, DMA_CTRL, START_DMA, 0, 100000, &dma_ctrl, + false); + if (ret) + drm_err(&xe->drm, "DMA for %s fw failed, DMA_CTRL=%u\n", + xe_uc_fw_type_repr(uc_fw->type), dma_ctrl); + + /* Disable the bits once DMA is over */ + xe_mmio_write32(mmio, DMA_CTRL, REG_MASKED_FIELD_DISABLE(dma_flags)); + + return ret; +} + +int xe_uc_fw_upload(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + int err; + + /* make sure the status was cleared the last time we reset the uc */ + xe_assert(xe, !xe_uc_fw_is_loaded(uc_fw)); + + if (!xe_uc_fw_is_loadable(uc_fw)) + return -ENOEXEC; + + /* Call custom loader */ + err = uc_fw_xfer(uc_fw, offset, dma_flags); + if (err) + goto fail; + + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_TRANSFERRED); + return 0; + +fail: + drm_err(&xe->drm, "Failed to load %s firmware %s (%d)\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + err); + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_LOAD_FAIL); + return err; +} + +static const char *version_type_repr(enum xe_uc_fw_version_types type) +{ + switch (type) { + case XE_UC_FW_VER_RELEASE: + return "release"; + case XE_UC_FW_VER_COMPATIBILITY: + return "compatibility"; + default: + return "Unknown version type"; + } +} + +void xe_uc_fw_print(struct xe_uc_fw *uc_fw, struct drm_printer *p) +{ + int i; + + drm_printf(p, "%s firmware: %s\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path); + drm_printf(p, "\tstatus: %s\n", + xe_uc_fw_status_repr(uc_fw->status)); + + print_uc_fw_version(p, &uc_fw->versions.wanted, "\twanted %s", + version_type_repr(uc_fw->versions.wanted_type)); + + for (i = 0; i < XE_UC_FW_VER_TYPE_COUNT; i++) { + struct xe_uc_fw_version *ver = &uc_fw->versions.found[i]; + + if (ver->major) + print_uc_fw_version(p, ver, "\tfound %s", + version_type_repr(i)); + } + + if (uc_fw->ucode_size) + drm_printf(p, "\tuCode: %u bytes\n", uc_fw->ucode_size); + if (uc_fw->rsa_size) + drm_printf(p, "\tRSA: %u bytes\n", uc_fw->rsa_size); +} diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h new file mode 100644 index 000000000..f2d3a3e72 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_FW_H_ +#define _XE_UC_FW_H_ + +#include + +#include "abi/uc_fw_abi.h" +#include "xe_macros.h" +#include "xe_uc_fw_types.h" + +struct drm_printer; + +int xe_uc_fw_init(struct xe_uc_fw *uc_fw); +size_t xe_uc_fw_copy_rsa(struct xe_uc_fw *uc_fw, void *dst, u32 max_len); +int xe_uc_fw_upload(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags); +int xe_uc_fw_check_version_requirements(struct xe_uc_fw *uc_fw); +void xe_uc_fw_print(struct xe_uc_fw *uc_fw, struct drm_printer *p); + +static inline u32 xe_uc_fw_rsa_offset(struct xe_uc_fw *uc_fw) +{ + return sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->css_offset; +} + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status); +#else +static inline void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, + enum xe_uc_fw_status status) +{ + uc_fw->__status = status; +} +#endif + +static inline +const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status) +{ + switch (status) { + case XE_UC_FIRMWARE_NOT_SUPPORTED: + return "N/A"; + case XE_UC_FIRMWARE_UNINITIALIZED: + return "UNINITIALIZED"; + case XE_UC_FIRMWARE_DISABLED: + return "DISABLED"; + case XE_UC_FIRMWARE_SELECTED: + return "SELECTED"; + case XE_UC_FIRMWARE_MISSING: + return "MISSING"; + case XE_UC_FIRMWARE_ERROR: + return "ERROR"; + case XE_UC_FIRMWARE_AVAILABLE: + return "AVAILABLE"; + case XE_UC_FIRMWARE_INIT_FAIL: + return "INIT FAIL"; + case XE_UC_FIRMWARE_LOADABLE: + return "LOADABLE"; + case XE_UC_FIRMWARE_LOAD_FAIL: + return "LOAD FAIL"; + case XE_UC_FIRMWARE_TRANSFERRED: + return "TRANSFERRED"; + case XE_UC_FIRMWARE_RUNNING: + return "RUNNING"; + case XE_UC_FIRMWARE_PRELOADED: + return "PRELOADED"; + } + return ""; +} + +static inline int xe_uc_fw_status_to_error(const enum xe_uc_fw_status status) +{ + switch (status) { + case XE_UC_FIRMWARE_NOT_SUPPORTED: + return -ENODEV; + case XE_UC_FIRMWARE_UNINITIALIZED: + return -EACCES; + case XE_UC_FIRMWARE_DISABLED: + return -EPERM; + case XE_UC_FIRMWARE_MISSING: + return -ENOENT; + case XE_UC_FIRMWARE_ERROR: + return -ENOEXEC; + case XE_UC_FIRMWARE_INIT_FAIL: + case XE_UC_FIRMWARE_LOAD_FAIL: + return -EIO; + case XE_UC_FIRMWARE_SELECTED: + return -ESTALE; + case XE_UC_FIRMWARE_AVAILABLE: + case XE_UC_FIRMWARE_LOADABLE: + case XE_UC_FIRMWARE_TRANSFERRED: + case XE_UC_FIRMWARE_RUNNING: + case XE_UC_FIRMWARE_PRELOADED: + return 0; + } + return -EINVAL; +} + +static inline const char *xe_uc_fw_type_repr(enum xe_uc_fw_type type) +{ + switch (type) { + case XE_UC_FW_TYPE_GUC: + return "GuC"; + case XE_UC_FW_TYPE_HUC: + return "HuC"; + case XE_UC_FW_TYPE_GSC: + return "GSC"; + default: + return "uC"; + } +} + +static inline enum xe_uc_fw_status +__xe_uc_fw_status(const struct xe_uc_fw *uc_fw) +{ + /* shouldn't call this before checking hw/blob availability */ + XE_WARN_ON(uc_fw->status == XE_UC_FIRMWARE_UNINITIALIZED); + return uc_fw->status; +} + +static inline bool xe_uc_fw_is_supported(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) != XE_UC_FIRMWARE_NOT_SUPPORTED; +} + +static inline bool xe_uc_fw_is_enabled(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) > XE_UC_FIRMWARE_DISABLED; +} + +static inline bool xe_uc_fw_is_disabled(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) == XE_UC_FIRMWARE_DISABLED; +} + +static inline bool xe_uc_fw_is_available(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_AVAILABLE; +} + +static inline bool xe_uc_fw_is_loadable(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_LOADABLE && + __xe_uc_fw_status(uc_fw) != XE_UC_FIRMWARE_PRELOADED; +} + +static inline bool xe_uc_fw_is_loaded(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_TRANSFERRED; +} + +static inline bool xe_uc_fw_is_running(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_RUNNING; +} + +static inline bool xe_uc_fw_is_overridden(const struct xe_uc_fw *uc_fw) +{ + return uc_fw->user_overridden; +} + +static inline bool xe_uc_fw_is_in_error_state(const struct xe_uc_fw *uc_fw) +{ + return xe_uc_fw_status_to_error(__xe_uc_fw_status(uc_fw)) < 0; +} + +static inline void xe_uc_fw_sanitize(struct xe_uc_fw *uc_fw) +{ + if (xe_uc_fw_is_loadable(uc_fw)) + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_LOADABLE); +} + +static inline u32 __xe_uc_fw_get_upload_size(struct xe_uc_fw *uc_fw) +{ + return sizeof(struct uc_css_header) + uc_fw->ucode_size; +} + +/** + * xe_uc_fw_get_upload_size() - Get size of firmware needed to be uploaded. + * @uc_fw: uC firmware. + * + * Get the size of the firmware and header that will be uploaded to WOPCM. + * + * Return: Upload firmware size, or zero on firmware fetch failure. + */ +static inline u32 xe_uc_fw_get_upload_size(struct xe_uc_fw *uc_fw) +{ + if (!xe_uc_fw_is_available(uc_fw)) + return 0; + + return __xe_uc_fw_get_upload_size(uc_fw); +} + +#define XE_UC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git" + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_fw_types.h b/drivers/gpu/drm/xe/xe_uc_fw_types.h new file mode 100644 index 000000000..2ebe8c9db --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw_types.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_FW_TYPES_H_ +#define _XE_UC_FW_TYPES_H_ + +#include + +struct xe_bo; + +/* + * +------------+---------------------------------------------------+ + * | PHASE | FIRMWARE STATUS TRANSITIONS | + * +============+===================================================+ + * | | UNINITIALIZED | + * +------------+- / | \ -+ + * | | DISABLED <--/ | \--> NOT_SUPPORTED | + * | init_early | V | + * | | SELECTED | + * +------------+- / | \ -+ + * | | MISSING <--/ | \--> ERROR | + * | fetch | V | + * | | AVAILABLE | + * +------------+- | \ -+ + * | | | \--> INIT FAIL | + * | init | V | + * | | /------> LOADABLE <----<-----------\ | + * +------------+- \ / \ \ \ -+ + * | | LOAD FAIL <--< \--> TRANSFERRED \ | + * | upload | \ / \ / | + * | | \---------/ \--> RUNNING | + * +------------+---------------------------------------------------+ + */ + +/* + * FIXME: Ported from the i915 and this is state machine is way too complicated. + * Circle back and simplify this. + */ +enum xe_uc_fw_status { + XE_UC_FIRMWARE_NOT_SUPPORTED = -1, /* no uc HW */ + XE_UC_FIRMWARE_UNINITIALIZED = 0, /* used to catch checks done too early */ + XE_UC_FIRMWARE_DISABLED, /* disabled */ + XE_UC_FIRMWARE_SELECTED, /* selected the blob we want to load */ + XE_UC_FIRMWARE_MISSING, /* blob not found on the system */ + XE_UC_FIRMWARE_ERROR, /* invalid format or version */ + XE_UC_FIRMWARE_AVAILABLE, /* blob found and copied in mem */ + XE_UC_FIRMWARE_INIT_FAIL, /* failed to prepare fw objects for load */ + XE_UC_FIRMWARE_LOADABLE, /* all fw-required objects are ready */ + XE_UC_FIRMWARE_LOAD_FAIL, /* failed to xfer or init/auth the fw */ + XE_UC_FIRMWARE_TRANSFERRED, /* dma xfer done */ + XE_UC_FIRMWARE_RUNNING, /* init/auth done */ + XE_UC_FIRMWARE_PRELOADED, /* preloaded by the PF driver */ +}; + +enum xe_uc_fw_type { + XE_UC_FW_TYPE_GUC = 0, + XE_UC_FW_TYPE_HUC, + XE_UC_FW_TYPE_GSC, + XE_UC_FW_NUM_TYPES +}; + +/** + * struct xe_uc_fw_version - Version for Xe micro controller firmware + */ +struct xe_uc_fw_version { + /** @branch: branch version of the FW (not always available) */ + u16 branch; + /** @major: major version of the FW */ + u16 major; + /** @minor: minor version of the FW */ + u16 minor; + /** @patch: patch version of the FW */ + u16 patch; + /** @build: build version of the FW (not always available) */ + u16 build; +}; + +enum xe_uc_fw_version_types { + XE_UC_FW_VER_RELEASE, + XE_UC_FW_VER_COMPATIBILITY, + XE_UC_FW_VER_TYPE_COUNT +}; + +/** + * struct xe_uc_fw - Xe micro controller firmware + */ +struct xe_uc_fw { + /** @type: type uC firmware */ + enum xe_uc_fw_type type; + union { + /** @status: firmware load status */ + const enum xe_uc_fw_status status; + /** + * @__status: private firmware load status - only to be used + * by firmware loading code + */ + enum xe_uc_fw_status __status; + }; + /** @path: path to uC firmware */ + const char *path; + /** @user_overridden: user provided path to uC firmware via modparam */ + bool user_overridden; + /** + * @full_ver_required: driver still under development and not ready + * for backward-compatible firmware. To be used only for **new** + * platforms, i.e. still under require_force_probe protection and not + * supported by i915. + */ + bool full_ver_required; + /** @size: size of uC firmware including css header */ + size_t size; + + /** @bo: Xe BO for uC firmware */ + struct xe_bo *bo; + + /** @has_gsc_headers: whether the FW image starts with GSC headers */ + bool has_gsc_headers; + + /* + * The firmware build process will generate a version header file with + * major and minor version defined. The versions are built into CSS + * header of firmware. The xe kernel driver set the minimal firmware + * version required per platform. + */ + + /** @versions: FW versions wanted and found */ + struct { + /** @versions.wanted: firmware version wanted by platform */ + struct xe_uc_fw_version wanted; + /** + * @versions.wanted_type: type of firmware version wanted + * (release vs compatibility) + */ + enum xe_uc_fw_version_types wanted_type; + /** @versions.found: fw versions found in firmware blob */ + struct xe_uc_fw_version found[XE_UC_FW_VER_TYPE_COUNT]; + } versions; + + /** @rsa_size: RSA size */ + u32 rsa_size; + /** @ucode_size: micro kernel size */ + u32 ucode_size; + /** @css_offset: offset within the blob at which the CSS is located */ + u32 css_offset; + + /** @private_data_size: size of private data found in uC css header */ + u32 private_data_size; + + /** @build_type: Firmware build type (see CSS_UKERNEL_INFO_BUILDTYPE for definitions) */ + u32 build_type; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_types.h b/drivers/gpu/drm/xe/xe_uc_types.h new file mode 100644 index 000000000..1708379dc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_types.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_TYPES_H_ +#define _XE_UC_TYPES_H_ + +#include "xe_gsc_types.h" +#include "xe_guc_types.h" +#include "xe_huc_types.h" +#include "xe_wopcm_types.h" + +/** + * struct xe_uc - Xe micro controllers + */ +struct xe_uc { + /** @guc: Graphics micro controller */ + struct xe_guc guc; + /** @huc: HuC */ + struct xe_huc huc; + /** @gsc: Graphics Security Controller */ + struct xe_gsc gsc; + /** @wopcm: WOPCM */ + struct xe_wopcm wopcm; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c new file mode 100644 index 000000000..8b2d461ea --- /dev/null +++ b/drivers/gpu/drm/xe/xe_userptr.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_svm.h" +#include "xe_userptr.h" + +#include + +#include "xe_tlb_inval.h" +#include "xe_trace_bo.h" + +static void xe_userptr_assert_in_notifier(struct xe_vm *vm) +{ + lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || + (lockdep_is_held(&vm->lock) && + lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && + dma_resv_held(xe_vm_resv(vm)))); +} + +/** + * xe_vma_userptr_check_repin() - Advisory check for repin needed + * @uvma: The userptr vma + * + * Check if the userptr vma has been invalidated since last successful + * repin. The check is advisory only and can the function can be called + * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the + * vma userptr will remain valid after a lockless check, so typically + * the call needs to be followed by a proper check under the notifier_lock. + * + * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. + */ +int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) +{ + return mmu_interval_check_retry(&uvma->userptr.notifier, + uvma->userptr.pages.notifier_seq) ? + -EAGAIN : 0; +} + +/** + * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs + * that need repinning. + * @vm: The VM. + * + * This function checks for whether the VM has userptrs that need repinning, + * and provides a release-type barrier on the svm.gpusvm.notifier_lock after + * checking. + * + * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. + */ +int __xe_vm_userptr_needs_repin(struct xe_vm *vm) +{ + lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock); + + return (list_empty(&vm->userptr.repin_list) && + list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; +} + +int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) +{ + struct xe_vma *vma = &uvma->vma; + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_device *xe = vm->xe; + struct drm_gpusvm_ctx ctx = { + .read_only = xe_vma_read_only(vma), + .device_private_page_owner = xe_svm_private_page_owner(vm, false), + .allow_mixed = true, + }; + + lockdep_assert_held(&vm->lock); + xe_assert(xe, xe_vma_is_userptr(vma)); + + if (vma->gpuva.flags & XE_VMA_DESTROYED) + return 0; + + return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, + uvma->userptr.notifier.mm, + &uvma->userptr.notifier, + xe_vma_userptr(vma), + xe_vma_userptr(vma) + xe_vma_size(vma), + &ctx); +} + +static struct mmu_interval_notifier_finish * +xe_vma_userptr_do_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma, bool is_deferred) +{ + struct xe_userptr *userptr = &uvma->userptr; + struct xe_vma *vma = &uvma->vma; + struct drm_gpusvm_ctx ctx = { + .in_notifier = true, + .read_only = xe_vma_read_only(vma), + }; + long err; + + xe_userptr_assert_in_notifier(vm); + if (is_deferred) + xe_assert(vm->xe, userptr->finish_inuse && !userptr->tlb_inval_submitted); + + err = dma_resv_wait_timeout(xe_vm_resv(vm), + DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + XE_WARN_ON(err <= 0); + + if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) { + if (!userptr->finish_inuse) { + /* + * Defer the TLB wait to an extra pass so the caller + * can pipeline TLB flushes across GPUs before waiting + * on any of them. + */ + xe_assert(vm->xe, !userptr->tlb_inval_submitted); + userptr->finish_inuse = true; + userptr->tlb_inval_submitted = true; + err = xe_vm_invalidate_vma_submit(vma, &userptr->inval_batch); + XE_WARN_ON(err); + return &userptr->finish; + } + err = xe_vm_invalidate_vma(vma); + XE_WARN_ON(err); + } + + if (is_deferred) + userptr->finish_inuse = false; + drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages, + xe_vma_size(vma) >> PAGE_SHIFT, &ctx); + return NULL; +} + +static void +xe_vma_userptr_complete_tlb_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma) +{ + struct xe_userptr *userptr = &uvma->userptr; + struct xe_vma *vma = &uvma->vma; + struct drm_gpusvm_ctx ctx = { + .in_notifier = true, + .read_only = xe_vma_read_only(vma), + }; + + xe_userptr_assert_in_notifier(vm); + xe_assert(vm->xe, userptr->finish_inuse); + xe_assert(vm->xe, userptr->tlb_inval_submitted); + + xe_tlb_inval_batch_wait(&userptr->inval_batch); + userptr->tlb_inval_submitted = false; + userptr->finish_inuse = false; + drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages, + xe_vma_size(vma) >> PAGE_SHIFT, &ctx); +} + +static struct mmu_interval_notifier_finish * +xe_vma_userptr_invalidate_pass1(struct xe_vm *vm, struct xe_userptr_vma *uvma) +{ + struct xe_userptr *userptr = &uvma->userptr; + struct xe_vma *vma = &uvma->vma; + struct dma_resv_iter cursor; + struct dma_fence *fence; + bool signaled = true; + + xe_userptr_assert_in_notifier(vm); + + /* + * Tell exec and rebind worker they need to repin and rebind this + * userptr. + */ + if (!xe_vm_in_fault_mode(vm) && + !(vma->gpuva.flags & XE_VMA_DESTROYED)) { + spin_lock(&vm->userptr.invalidated_lock); + list_move_tail(&userptr->invalidate_link, + &vm->userptr.invalidated); + spin_unlock(&vm->userptr.invalidated_lock); + } + + /* + * Preempt fences turn into schedule disables, pipeline these. + * Note that even in fault mode, we need to wait for binds and + * unbinds to complete, and those are attached as BOOKMARK fences + * to the vm. + */ + dma_resv_iter_begin(&cursor, xe_vm_resv(vm), + DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) { + dma_fence_enable_signaling(fence); + if (signaled && !dma_fence_is_signaled(fence)) + signaled = false; + } + dma_resv_iter_end(&cursor); + + /* + * Only one caller at a time can use the multi-pass state. + * If it's already in use, or all fences are already signaled, + * proceed directly to invalidation without deferring. + */ + if (signaled || userptr->finish_inuse) + return xe_vma_userptr_do_inval(vm, uvma, false); + + /* Defer: the notifier core will call invalidate_finish once done. */ + userptr->finish_inuse = true; + + return &userptr->finish; +} + +static bool xe_vma_userptr_invalidate_start(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq, + struct mmu_interval_notifier_finish **p_finish) +{ + struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier); + struct xe_vma *vma = &uvma->vma; + struct xe_vm *vm = xe_vma_vm(vma); + + xe_assert(vm->xe, xe_vma_is_userptr(vma)); + trace_xe_vma_userptr_invalidate(vma); + + if (!mmu_notifier_range_blockable(range)) + return false; + + vm_dbg(&xe_vma_vm(vma)->xe->drm, + "NOTIFIER PASS1: addr=0x%016llx, range=0x%016llx", + xe_vma_start(vma), xe_vma_size(vma)); + + down_write(&vm->svm.gpusvm.notifier_lock); + mmu_interval_set_seq(mni, cur_seq); + + *p_finish = xe_vma_userptr_invalidate_pass1(vm, uvma); + + up_write(&vm->svm.gpusvm.notifier_lock); + if (!*p_finish) + trace_xe_vma_userptr_invalidate_complete(vma); + + return true; +} + +static void xe_vma_userptr_invalidate_finish(struct mmu_interval_notifier_finish *finish) +{ + struct xe_userptr_vma *uvma = container_of(finish, typeof(*uvma), userptr.finish); + struct xe_vma *vma = &uvma->vma; + struct xe_vm *vm = xe_vma_vm(vma); + + vm_dbg(&xe_vma_vm(vma)->xe->drm, + "NOTIFIER PASS2: addr=0x%016llx, range=0x%016llx", + xe_vma_start(vma), xe_vma_size(vma)); + + down_write(&vm->svm.gpusvm.notifier_lock); + /* + * If a TLB invalidation was previously submitted (deferred from the + * synchronous pass1 fallback), wait for it and unmap pages. + * Otherwise, fences have now completed: invalidate the TLB and unmap. + */ + if (uvma->userptr.tlb_inval_submitted) + xe_vma_userptr_complete_tlb_inval(vm, uvma); + else + xe_vma_userptr_do_inval(vm, uvma, true); + up_write(&vm->svm.gpusvm.notifier_lock); + trace_xe_vma_userptr_invalidate_complete(vma); +} + +static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { + .invalidate_start = xe_vma_userptr_invalidate_start, + .invalidate_finish = xe_vma_userptr_invalidate_finish, +}; + +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) +/** + * xe_vma_userptr_force_invalidate() - force invalidate a userptr + * @uvma: The userptr vma to invalidate + * + * Perform a forced userptr invalidation for testing purposes. + */ +void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) +{ + struct mmu_interval_notifier_finish *finish; + struct xe_vm *vm = xe_vma_vm(&uvma->vma); + + /* Protect against concurrent userptr pinning */ + lockdep_assert_held(&vm->lock); + /* Protect against concurrent notifiers */ + lockdep_assert_held(&vm->svm.gpusvm.notifier_lock); + /* + * Protect against concurrent instances of this function and + * the critical exec sections + */ + xe_vm_assert_held(vm); + + if (!mmu_interval_read_retry(&uvma->userptr.notifier, + uvma->userptr.pages.notifier_seq)) + uvma->userptr.pages.notifier_seq -= 2; + + finish = xe_vma_userptr_invalidate_pass1(vm, uvma); + if (finish) + finish = xe_vma_userptr_do_inval(vm, uvma, true); + if (finish) + xe_vma_userptr_complete_tlb_inval(vm, uvma); +} +#endif + +int xe_vm_userptr_pin(struct xe_vm *vm) +{ + struct xe_userptr_vma *uvma, *next; + int err = 0; + + xe_assert(vm->xe, !xe_vm_in_fault_mode(vm)); + lockdep_assert_held_write(&vm->lock); + + /* Collect invalidated userptrs */ + spin_lock(&vm->userptr.invalidated_lock); + xe_assert(vm->xe, list_empty(&vm->userptr.repin_list)); + list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, + userptr.invalidate_link) { + list_del_init(&uvma->userptr.invalidate_link); + list_add_tail(&uvma->userptr.repin_link, + &vm->userptr.repin_list); + } + spin_unlock(&vm->userptr.invalidated_lock); + + /* Pin and move to bind list */ + list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, + userptr.repin_link) { + err = xe_vma_userptr_pin_pages(uvma); + if (err == -EFAULT) { + list_del_init(&uvma->userptr.repin_link); + /* + * We might have already done the pin once already, but + * then had to retry before the re-bind happened, due + * some other condition in the caller, but in the + * meantime the userptr got dinged by the notifier such + * that we need to revalidate here, but this time we hit + * the EFAULT. In such a case make sure we remove + * ourselves from the rebind list to avoid going down in + * flames. + */ + if (!list_empty(&uvma->vma.combined_links.rebind)) + list_del_init(&uvma->vma.combined_links.rebind); + + /* Wait for pending binds */ + xe_vm_lock(vm, false); + dma_resv_wait_timeout(xe_vm_resv(vm), + DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + + down_read(&vm->svm.gpusvm.notifier_lock); + err = xe_vm_invalidate_vma(&uvma->vma); + up_read(&vm->svm.gpusvm.notifier_lock); + xe_vm_unlock(vm); + if (err) + break; + } else { + if (err) + break; + + list_del_init(&uvma->userptr.repin_link); + list_move_tail(&uvma->vma.combined_links.rebind, + &vm->rebind_list); + } + } + + if (err) { + down_write(&vm->svm.gpusvm.notifier_lock); + spin_lock(&vm->userptr.invalidated_lock); + list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, + userptr.repin_link) { + list_del_init(&uvma->userptr.repin_link); + list_move_tail(&uvma->userptr.invalidate_link, + &vm->userptr.invalidated); + } + spin_unlock(&vm->userptr.invalidated_lock); + up_write(&vm->svm.gpusvm.notifier_lock); + } + return err; +} + +/** + * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs + * that need repinning. + * @vm: The VM. + * + * This function does an advisory check for whether the VM has userptrs that + * need repinning. + * + * Return: 0 if there are no indications of userptrs needing repinning, + * -EAGAIN if there are. + */ +int xe_vm_userptr_check_repin(struct xe_vm *vm) +{ + return (list_empty_careful(&vm->userptr.repin_list) && + list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; +} + +int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start, + unsigned long range) +{ + struct xe_userptr *userptr = &uvma->userptr; + struct xe_vm *vm = xe_vma_vm(&uvma->vma); + int err; + + INIT_LIST_HEAD(&userptr->invalidate_link); + INIT_LIST_HEAD(&userptr->repin_link); + + drm_gpusvm_init_pages(&userptr->pages, &vm->xe->drm); + + err = mmu_interval_notifier_insert(&userptr->notifier, current->mm, + start, range, + &vma_userptr_notifier_ops); + if (err) + return err; + + return 0; +} + +void xe_userptr_remove(struct xe_userptr_vma *uvma) +{ + struct xe_vm *vm = xe_vma_vm(&uvma->vma); + struct xe_userptr *userptr = &uvma->userptr; + + drm_gpusvm_free_pages(&vm->svm.gpusvm, &uvma->userptr.pages, + xe_vma_size(&uvma->vma) >> PAGE_SHIFT); + + /* + * Since userptr pages are not pinned, we can't remove + * the notifier until we're sure the GPU is not accessing + * them anymore + */ + mmu_interval_notifier_remove(&userptr->notifier); +} + +void xe_userptr_destroy(struct xe_userptr_vma *uvma) +{ + struct xe_vm *vm = xe_vma_vm(&uvma->vma); + + spin_lock(&vm->userptr.invalidated_lock); + xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link)); + list_del(&uvma->userptr.invalidate_link); + spin_unlock(&vm->userptr.invalidated_lock); +} diff --git a/drivers/gpu/drm/xe/xe_userptr.h b/drivers/gpu/drm/xe/xe_userptr.h new file mode 100644 index 000000000..2a3cd1b5e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_userptr.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_USERPTR_H_ +#define _XE_USERPTR_H_ + +#include +#include +#include +#include +#include + +#include + +#include "xe_tlb_inval_types.h" + +struct xe_vm; +struct xe_vma; +struct xe_userptr_vma; + +/** struct xe_userptr_vm - User pointer VM level state */ +struct xe_userptr_vm { + /** + * @userptr.repin_list: list of VMAs which are user pointers, + * and needs repinning. Protected by @lock. + */ + struct list_head repin_list; + /** + * @userptr.invalidated_lock: Protects the + * @userptr.invalidated list. + */ + spinlock_t invalidated_lock; + /** + * @userptr.invalidated: List of invalidated userptrs, not yet + * picked + * up for revalidation. Protected from access with the + * @invalidated_lock. Removing items from the list + * additionally requires @lock in write mode, and adding + * items to the list requires either the @svm.gpusvm.notifier_lock in + * write mode, OR @lock in write mode. + */ + struct list_head invalidated; +}; + +/** struct xe_userptr - User pointer */ +struct xe_userptr { + /** @invalidate_link: Link for the vm::userptr.invalidated list */ + struct list_head invalidate_link; + /** @userptr: link into VM repin list if userptr. */ + struct list_head repin_link; + /** + * @pages: gpusvm pages for this user pointer. + */ + struct drm_gpusvm_pages pages; + /** + * @notifier: MMU notifier for user pointer (invalidation call back) + */ + struct mmu_interval_notifier notifier; + /** + * @finish: MMU notifier finish structure for two-pass invalidation. + * Embedded here to avoid allocation in the notifier callback. + * Protected by struct xe_vm::svm.gpusvm.notifier_lock in write mode + * alternatively by the same lock in read mode *and* the vm resv held. + */ + struct mmu_interval_notifier_finish finish; + /** + * @inval_batch: TLB invalidation batch for deferred completion. + * Stores an in-flight TLB invalidation submitted during a two-pass + * notifier so the wait can be deferred to a subsequent pass, allowing + * multiple GPUs to be signalled before any of them are waited on. + * Protected using the same locking as @finish. + */ + struct xe_tlb_inval_batch inval_batch; + /** + * @finish_inuse: Whether @finish is currently in use by an in-progress + * two-pass invalidation. + * Protected using the same locking as @finish. + */ + bool finish_inuse; + /** + * @tlb_inval_submitted: Whether a TLB invalidation has been submitted + * via @inval_batch and is pending completion. When set, the next pass + * must call xe_tlb_inval_batch_wait() before reusing @inval_batch. + * Protected using the same locking as @finish. + */ + bool tlb_inval_submitted; + /** + * @initial_bind: user pointer has been bound at least once. + * write: vm->svm.gpusvm.notifier_lock in read mode and vm->resv held. + * read: vm->svm.gpusvm.notifier_lock in write mode or vm->resv held. + */ + bool initial_bind; +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + u32 divisor; +#endif +}; + +#if IS_ENABLED(CONFIG_DRM_GPUSVM) +void xe_userptr_remove(struct xe_userptr_vma *uvma); +int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start, + unsigned long range); +void xe_userptr_destroy(struct xe_userptr_vma *uvma); + +int xe_vm_userptr_pin(struct xe_vm *vm); +int __xe_vm_userptr_needs_repin(struct xe_vm *vm); +int xe_vm_userptr_check_repin(struct xe_vm *vm); +int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma); +int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma); +#else +static inline void xe_userptr_remove(struct xe_userptr_vma *uvma) {} + +static inline int xe_userptr_setup(struct xe_userptr_vma *uvma, + unsigned long start, unsigned long range) +{ + return -ENODEV; +} + +static inline void xe_userptr_destroy(struct xe_userptr_vma *uvma) {} + +static inline int xe_vm_userptr_pin(struct xe_vm *vm) { return 0; } +static inline int __xe_vm_userptr_needs_repin(struct xe_vm *vm) { return 0; } +static inline int xe_vm_userptr_check_repin(struct xe_vm *vm) { return 0; } +static inline int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) { return -ENODEV; } +static inline int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) { return -ENODEV; }; +#endif + +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) +void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma); +#else +static inline void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) +{ +} +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_validation.c b/drivers/gpu/drm/xe/xe_validation.c new file mode 100644 index 000000000..8dff4d0ec --- /dev/null +++ b/drivers/gpu/drm/xe/xe_validation.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ +#include +#include +#include + +#include "xe_assert.h" +#include "xe_validation.h" + +#ifdef CONFIG_DRM_XE_DEBUG +/** + * xe_validation_assert_exec() - Assert that the drm_exec pointer is suitable + * for validation. + * @xe: Pointer to the xe device. + * @exec: The drm_exec pointer to check. + * @obj: Pointer to the object subject to validation. + * + * NULL exec pointers are not allowed. + * For XE_VALIDATION_UNIMPLEMENTED, no checking. + * For XE_VLIDATION_OPT_OUT, check that the caller is a kunit test + * For XE_VALIDATION_UNSUPPORTED, check that the object subject to + * validation is a dma-buf, for which support for ww locking is + * not in place in the dma-buf layer. + */ +void xe_validation_assert_exec(const struct xe_device *xe, + const struct drm_exec *exec, + const struct drm_gem_object *obj) +{ + xe_assert(xe, exec); + if (IS_ERR(exec)) { + switch (PTR_ERR(exec)) { + case __XE_VAL_UNIMPLEMENTED: + break; + case __XE_VAL_UNSUPPORTED: + xe_assert(xe, !!obj->dma_buf); + break; +#if IS_ENABLED(CONFIG_KUNIT) + case __XE_VAL_OPT_OUT: + xe_assert(xe, current->kunit_test); + break; +#endif + default: + xe_assert(xe, false); + } + } +} +#endif + +static int xe_validation_lock(struct xe_validation_ctx *ctx) +{ + struct xe_validation_device *val = ctx->val; + int ret = 0; + + if (ctx->val_flags.interruptible) { + if (ctx->request_exclusive) + ret = down_write_killable(&val->lock); + else + ret = down_read_interruptible(&val->lock); + } else { + if (ctx->request_exclusive) + down_write(&val->lock); + else + down_read(&val->lock); + } + + if (!ret) { + ctx->lock_held = true; + ctx->lock_held_exclusive = ctx->request_exclusive; + } + + return ret; +} + +static int xe_validation_trylock(struct xe_validation_ctx *ctx) +{ + struct xe_validation_device *val = ctx->val; + bool locked; + + if (ctx->request_exclusive) + locked = down_write_trylock(&val->lock); + else + locked = down_read_trylock(&val->lock); + + if (locked) { + ctx->lock_held = true; + ctx->lock_held_exclusive = ctx->request_exclusive; + } + + return locked ? 0 : -EWOULDBLOCK; +} + +static void xe_validation_unlock(struct xe_validation_ctx *ctx) +{ + if (!ctx->lock_held) + return; + + if (ctx->lock_held_exclusive) + up_write(&ctx->val->lock); + else + up_read(&ctx->val->lock); + + ctx->lock_held = false; +} + +/** + * xe_validation_ctx_init() - Initialize an xe_validation_ctx + * @ctx: The xe_validation_ctx to initialize. + * @val: The xe_validation_device representing the validation domain. + * @exec: The struct drm_exec to use for the transaction. May be NULL. + * @flags: The flags to use for initialization. + * + * Initialize and lock a an xe_validation transaction using the validation domain + * represented by @val. Also initialize the drm_exec object forwarding parts of + * @flags to the drm_exec initialization. The @flags.exclusive flag should + * typically be set to false to avoid locking out other validators from the + * domain until an OOM is hit. For testing- or final attempt purposes it can, + * however, be set to true. + * + * Return: %0 on success, %-EINTR if interruptible initial locking failed with a + * signal pending. If @flags.no_block is set to true, a failed trylock + * returns %-EWOULDBLOCK. + */ +int xe_validation_ctx_init(struct xe_validation_ctx *ctx, struct xe_validation_device *val, + struct drm_exec *exec, const struct xe_val_flags flags) +{ + int ret; + + ctx->exec = exec; + ctx->val = val; + ctx->lock_held = false; + ctx->lock_held_exclusive = false; + ctx->request_exclusive = flags.exclusive; + ctx->val_flags = flags; + ctx->exec_flags = 0; + ctx->nr = 0; + + if (flags.no_block) + ret = xe_validation_trylock(ctx); + else + ret = xe_validation_lock(ctx); + if (ret) + return ret; + + if (exec) { + if (flags.interruptible) + ctx->exec_flags |= DRM_EXEC_INTERRUPTIBLE_WAIT; + if (flags.exec_ignore_duplicates) + ctx->exec_flags |= DRM_EXEC_IGNORE_DUPLICATES; + drm_exec_init(exec, ctx->exec_flags, ctx->nr); + } + + return 0; +} + +#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH +/* + * This abuses ww_mutex internals and should be + * replaced by checking for -EDEADLK when we can make TTM + * stop converting -EDEADLK to -ENOMEM. + * An alternative is to not have exhaustive eviction with + * CONFIG_DEBUG_WW_MUTEX_SLOWPATH until that happens. + */ +static bool xe_validation_contention_injected(struct drm_exec *exec) +{ + return !!drm_exec_ticket(exec)->contending_lock; +} + +#else + +static bool xe_validation_contention_injected(struct drm_exec *exec) +{ + return false; +} + +#endif + +static bool __xe_validation_should_retry(struct xe_validation_ctx *ctx, int ret) +{ + if (ret == -ENOMEM && + ((ctx->request_exclusive && + xe_validation_contention_injected(ctx->exec)) || + !ctx->request_exclusive)) { + ctx->request_exclusive = true; + return true; + } + + return false; +} + +/** + * xe_validation_exec_lock() - Perform drm_gpuvm_exec_lock within a validation + * transaction. + * @ctx: An uninitialized xe_validation_ctx. + * @vm_exec: An initialized struct vm_exec. + * @val: The validation domain. + * + * The drm_gpuvm_exec_lock() function internally initializes its drm_exec + * transaction and therefore doesn't lend itself very well to be using + * xe_validation_ctx_init(). Provide a helper that takes an uninitialized + * xe_validation_ctx and calls drm_gpuvm_exec_lock() with OOM retry. + * + * Return: %0 on success, negative error code on failure. + */ +int xe_validation_exec_lock(struct xe_validation_ctx *ctx, + struct drm_gpuvm_exec *vm_exec, + struct xe_validation_device *val) +{ + int ret; + + memset(ctx, 0, sizeof(*ctx)); + ctx->exec = &vm_exec->exec; + ctx->exec_flags = vm_exec->flags; + ctx->val = val; + if (ctx->exec_flags & DRM_EXEC_INTERRUPTIBLE_WAIT) + ctx->val_flags.interruptible = 1; + if (ctx->exec_flags & DRM_EXEC_IGNORE_DUPLICATES) + ctx->val_flags.exec_ignore_duplicates = 1; +retry: + ret = xe_validation_lock(ctx); + if (ret) + return ret; + + ret = drm_gpuvm_exec_lock(vm_exec); + if (ret) { + xe_validation_unlock(ctx); + if (__xe_validation_should_retry(ctx, ret)) + goto retry; + } + + return ret; +} + +/** + * xe_validation_ctx_fini() - Finalize a validation transaction + * @ctx: The Validation transaction to finalize. + * + * Finalize a validation transaction and its related drm_exec transaction. + */ +void xe_validation_ctx_fini(struct xe_validation_ctx *ctx) +{ + if (ctx->exec) + drm_exec_fini(ctx->exec); + xe_validation_unlock(ctx); +} + +/** + * xe_validation_should_retry() - Determine if a validation transaction should retry + * @ctx: The validation transaction. + * @ret: Pointer to a return value variable. + * + * Determines whether a validation transaction should retry based on the + * internal transaction state and the return value pointed to by @ret. + * If a validation should be retried, the transaction is prepared for that, + * and the validation locked might be re-locked in exclusive mode, and *@ret + * is set to %0. If the re-locking errors, typically due to interruptible + * locking with signal pending, *@ret is instead set to -EINTR and the + * function returns %false. + * + * Return: %true if validation should be retried, %false otherwise. + */ +bool xe_validation_should_retry(struct xe_validation_ctx *ctx, int *ret) +{ + if (__xe_validation_should_retry(ctx, *ret)) { + drm_exec_fini(ctx->exec); + *ret = 0; + if (ctx->request_exclusive != ctx->lock_held_exclusive) { + xe_validation_unlock(ctx); + *ret = xe_validation_lock(ctx); + } + drm_exec_init(ctx->exec, ctx->exec_flags, ctx->nr); + return !*ret; + } + + return false; +} diff --git a/drivers/gpu/drm/xe/xe_validation.h b/drivers/gpu/drm/xe/xe_validation.h new file mode 100644 index 000000000..4cd955ce6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_validation.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ +#ifndef _XE_VALIDATION_H_ +#define _XE_VALIDATION_H_ + +#include +#include +#include + +struct drm_exec; +struct drm_gem_object; +struct drm_gpuvm_exec; +struct xe_device; + +#ifdef CONFIG_PROVE_LOCKING +/** + * xe_validation_lockdep() - Assert that a drm_exec locking transaction can + * be initialized at this point. + */ +static inline void xe_validation_lockdep(void) +{ + struct ww_acquire_ctx ticket; + + ww_acquire_init(&ticket, &reservation_ww_class); + ww_acquire_fini(&ticket); +} +#else +static inline void xe_validation_lockdep(void) +{ +} +#endif + +/* + * Various values of the drm_exec pointer where we've not (yet) + * implemented full ww locking. + * + * XE_VALIDATION_UNIMPLEMENTED means implementation is pending. + * A lockdep check is made to assure that a drm_exec locking + * transaction can actually take place where the macro is + * used. If this asserts, the exec pointer needs to be assigned + * higher up in the callchain and passed down. + * + * XE_VALIDATION_UNSUPPORTED is for dma-buf code only where + * the dma-buf layer doesn't support WW locking. + * + * XE_VALIDATION_OPT_OUT is for simplification of kunit tests where + * exhaustive eviction isn't necessary. + */ +#define __XE_VAL_UNIMPLEMENTED -EINVAL +#define XE_VALIDATION_UNIMPLEMENTED (xe_validation_lockdep(), \ + (struct drm_exec *)ERR_PTR(__XE_VAL_UNIMPLEMENTED)) + +#define __XE_VAL_UNSUPPORTED -EOPNOTSUPP +#define XE_VALIDATION_UNSUPPORTED ((struct drm_exec *)ERR_PTR(__XE_VAL_UNSUPPORTED)) + +#define __XE_VAL_OPT_OUT -ENOMEM +#define XE_VALIDATION_OPT_OUT (xe_validation_lockdep(), \ + (struct drm_exec *)ERR_PTR(__XE_VAL_OPT_OUT)) +#ifdef CONFIG_DRM_XE_DEBUG +void xe_validation_assert_exec(const struct xe_device *xe, const struct drm_exec *exec, + const struct drm_gem_object *obj); +#else +#define xe_validation_assert_exec(_xe, _exec, _obj) \ + do { \ + (void)_xe; (void)_exec; (void)_obj; \ + } while (0) +#endif + +/** + * struct xe_validation_device - The domain for exhaustive eviction + * @lock: The lock used to exclude other processes from allocating graphics memory + * + * The struct xe_validation_device represents the domain for which we want to use + * exhaustive eviction. The @lock is typically grabbed in read mode for allocations + * but when graphics memory allocation fails, it is retried with the write mode held. + */ +struct xe_validation_device { + struct rw_semaphore lock; +}; + +/** + * struct xe_val_flags - Flags for xe_validation_ctx_init(). + * @exclusive: Start the validation transaction by locking out all other validators. + * @no_block: Don't block on initialization. + * @interruptible: Block interruptible if blocking. Implies initializing the drm_exec + * context with the DRM_EXEC_INTERRUPTIBLE_WAIT flag. + * @exec_ignore_duplicates: Initialize the drm_exec context with the + * DRM_EXEC_IGNORE_DUPLICATES flag. + */ +struct xe_val_flags { + u32 exclusive :1; + u32 no_block :1; + u32 interruptible :1; + u32 exec_ignore_duplicates :1; +}; + +/** + * struct xe_validation_ctx - A struct drm_exec subclass with support for + * exhaustive eviction + * @exec: The drm_exec object base class. Note that we use a pointer instead of + * embedding to avoid diamond inheritance. + * @val: The exhaustive eviction domain. + * @val_flags: Copy of the struct xe_val_flags passed to xe_validation_ctx_init. + * @lock_held: Whether The domain lock is currently held. + * @lock_held_exclusive: Whether the domain lock is held in exclusive mode. + * @request_exclusive: Whether to lock exclusively (write mode) the next time + * the domain lock is locked. + * @exec_flags: The drm_exec flags used for drm_exec (re-)initialization. + * @nr: The drm_exec nr parameter used for drm_exec (re-)initialization. + */ +struct xe_validation_ctx { + struct drm_exec *exec; + struct xe_validation_device *val; + struct xe_val_flags val_flags; + bool lock_held; + bool lock_held_exclusive; + bool request_exclusive; + u32 exec_flags; + unsigned int nr; +}; + +int xe_validation_ctx_init(struct xe_validation_ctx *ctx, struct xe_validation_device *val, + struct drm_exec *exec, const struct xe_val_flags flags); + +int xe_validation_exec_lock(struct xe_validation_ctx *ctx, struct drm_gpuvm_exec *vm_exec, + struct xe_validation_device *val); + +void xe_validation_ctx_fini(struct xe_validation_ctx *ctx); + +bool xe_validation_should_retry(struct xe_validation_ctx *ctx, int *ret); + +/** + * xe_validation_retry_on_oom() - Retry on oom in an xe_validaton transaction + * @_ctx: Pointer to the xe_validation_ctx + * @_ret: The current error value possibly holding -ENOMEM + * + * Use this in way similar to drm_exec_retry_on_contention(). + * If @_ret contains -ENOMEM the transaction is restarted once in a way that + * blocks other transactions and allows exhastive eviction. If the transaction + * was already restarted once, Just return the -ENOMEM. May also set + * _ret to -EINTR if not retrying and waits are interruptible. + * May only be used within a drm_exec_until_all_locked() loop. + */ +#define xe_validation_retry_on_oom(_ctx, _ret) \ + do { \ + if (xe_validation_should_retry(_ctx, _ret)) \ + drm_exec_retry((_ctx)->exec); \ + } while (0) + +/** + * xe_validation_device_init - Initialize a struct xe_validation_device + * @val: The xe_validation_device to init. + */ +static inline void +xe_validation_device_init(struct xe_validation_device *val) +{ + init_rwsem(&val->lock); +} + +/* + * Make guard() and scoped_guard() work with xe_validation_ctx + * so that we can exit transactions without caring about the + * cleanup. + */ +DEFINE_CLASS(xe_validation, struct xe_validation_ctx *, + if (_T) xe_validation_ctx_fini(_T);, + ({*_ret = xe_validation_ctx_init(_ctx, _val, _exec, _flags); + *_ret ? NULL : _ctx; }), + struct xe_validation_ctx *_ctx, struct xe_validation_device *_val, + struct drm_exec *_exec, const struct xe_val_flags _flags, int *_ret); +static inline void *class_xe_validation_lock_ptr(class_xe_validation_t *_T) +{return *_T; } +#define class_xe_validation_is_conditional true + +/** + * xe_validation_guard() - An auto-cleanup xe_validation_ctx transaction + * @_ctx: The xe_validation_ctx. + * @_val: The xe_validation_device. + * @_exec: The struct drm_exec object + * @_flags: Flags for the xe_validation_ctx initialization. + * @_ret: Return in / out parameter. May be set by this macro. Typically 0 when called. + * + * This macro is will initiate a drm_exec transaction with additional support for + * exhaustive eviction. + */ +#define xe_validation_guard(_ctx, _val, _exec, _flags, _ret) \ + scoped_guard(xe_validation, _ctx, _val, _exec, _flags, &_ret) \ + drm_exec_until_all_locked(_exec) + +#endif diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c new file mode 100644 index 000000000..9e0176861 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -0,0 +1,4992 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_vm.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "regs/xe_gtt_defs.h" +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_drm_client.h" +#include "xe_exec_queue.h" +#include "xe_gt.h" +#include "xe_migrate.h" +#include "xe_pat.h" +#include "xe_pm.h" +#include "xe_preempt_fence.h" +#include "xe_pt.h" +#include "xe_pxp.h" +#include "xe_sriov_vf.h" +#include "xe_svm.h" +#include "xe_sync.h" +#include "xe_tile.h" +#include "xe_tlb_inval.h" +#include "xe_trace_bo.h" +#include "xe_vm_madvise.h" +#include "xe_wa.h" + +static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm) +{ + return vm->gpuvm.r_obj; +} + +/** + * xe_vm_drm_exec_lock() - Lock the vm's resv with a drm_exec transaction + * @vm: The vm whose resv is to be locked. + * @exec: The drm_exec transaction. + * + * Helper to lock the vm's resv as part of a drm_exec transaction. + * + * Return: %0 on success. See drm_exec_lock_obj() for error codes. + */ +int xe_vm_drm_exec_lock(struct xe_vm *vm, struct drm_exec *exec) +{ + return drm_exec_lock_obj(exec, xe_vm_obj(vm)); +} + +static bool preempt_fences_waiting(struct xe_vm *vm) +{ + struct xe_exec_queue *q; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { + if (!q->lr.pfence || + test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &q->lr.pfence->flags)) { + return true; + } + } + + return false; +} + +static void free_preempt_fences(struct list_head *list) +{ + struct list_head *link, *next; + + list_for_each_safe(link, next, list) + xe_preempt_fence_free(to_preempt_fence_from_link(link)); +} + +static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, + unsigned int *count) +{ + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + if (*count >= vm->preempt.num_exec_queues) + return 0; + + for (; *count < vm->preempt.num_exec_queues; ++(*count)) { + struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); + + if (IS_ERR(pfence)) + return PTR_ERR(pfence); + + list_move_tail(xe_preempt_fence_link(pfence), list); + } + + return 0; +} + +static int wait_for_existing_preempt_fences(struct xe_vm *vm) +{ + struct xe_exec_queue *q; + bool vf_migration = IS_SRIOV_VF(vm->xe) && + xe_sriov_vf_migration_supported(vm->xe); + signed long wait_time = vf_migration ? HZ / 5 : MAX_SCHEDULE_TIMEOUT; + + xe_vm_assert_held(vm); + + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { + if (q->lr.pfence) { + long timeout; + + timeout = dma_fence_wait_timeout(q->lr.pfence, false, + wait_time); + if (!timeout) { + xe_assert(vm->xe, vf_migration); + return -EAGAIN; + } + + /* Only -ETIME on fence indicates VM needs to be killed */ + if (timeout < 0 || q->lr.pfence->error == -ETIME) + return -ETIME; + + dma_fence_put(q->lr.pfence); + q->lr.pfence = NULL; + } + } + + return 0; +} + +static bool xe_vm_is_idle(struct xe_vm *vm) +{ + struct xe_exec_queue *q; + + xe_vm_assert_held(vm); + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { + if (!xe_exec_queue_is_idle(q)) + return false; + } + + return true; +} + +static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) +{ + struct list_head *link; + struct xe_exec_queue *q; + + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { + struct dma_fence *fence; + + link = list->next; + xe_assert(vm->xe, link != list); + + fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), + q, q->lr.context, + ++q->lr.seqno); + dma_fence_put(q->lr.pfence); + q->lr.pfence = fence; + } +} + +static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) +{ + struct xe_exec_queue *q; + int err; + + xe_bo_assert_held(bo); + + if (!vm->preempt.num_exec_queues) + return 0; + + err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues); + if (err) + return err; + + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) + if (q->lr.pfence) { + dma_resv_add_fence(bo->ttm.base.resv, + q->lr.pfence, + DMA_RESV_USAGE_BOOKKEEP); + } + + return 0; +} + +static void resume_and_reinstall_preempt_fences(struct xe_vm *vm, + struct drm_exec *exec) +{ + struct xe_exec_queue *q; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { + /* + * Only resume queues whose suspend() actually succeeded. A + * failed suspend() (e.g. killed/banned/wedged) leaves the queue + * un-suspended, so it must not be resumed. + * + * Also skip queues that have since been reset/killed/banned/ + * wedged: their suspend may not have completed (suspend_pending + * can still be set, e.g. a preempt fence signalled with -ENOENT + * without waiting), so resuming would trip the !suspend_pending + * assert in the backend. Such queues are being torn down anyway, + * so leave them marked suspended and let teardown resolve their + * state. + */ + if (READ_ONCE(q->lr.suspended) && !q->ops->reset_status(q)) { + WRITE_ONCE(q->lr.suspended, false); + q->ops->resume(q); + } + + drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence, + DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); + } +} + +int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) +{ + struct drm_gpuvm_exec vm_exec = { + .vm = &vm->gpuvm, + .flags = DRM_EXEC_INTERRUPTIBLE_WAIT, + .num_fences = 1, + }; + struct drm_exec *exec = &vm_exec.exec; + struct xe_validation_ctx ctx; + struct dma_fence *pfence; + int err; + bool wait; + + xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); + + down_write(&vm->lock); + err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val); + if (err) + goto out_up_write; + + pfence = xe_preempt_fence_create(q, q->lr.context, + ++q->lr.seqno); + if (IS_ERR(pfence)) { + err = PTR_ERR(pfence); + goto out_fini; + } + + list_add(&q->lr.link, &vm->preempt.exec_queues); + ++vm->preempt.num_exec_queues; + q->lr.pfence = pfence; + + xe_svm_notifier_lock(vm); + + drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence, + DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); + + /* + * Check to see if a preemption on VM is in flight or userptr + * invalidation, if so trigger this preempt fence to sync state with + * other preempt fences on the VM. + */ + wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); + if (wait) + dma_fence_enable_signaling(pfence); + + xe_svm_notifier_unlock(vm); + +out_fini: + xe_validation_ctx_fini(&ctx); +out_up_write: + up_write(&vm->lock); + + return err; +} +ALLOW_ERROR_INJECTION(xe_vm_add_compute_exec_queue, ERRNO); + +/** + * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM + * @vm: The VM. + * @q: The exec_queue + * + * Note that this function might be called multiple times on the same queue. + */ +void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) +{ + if (!xe_vm_in_preempt_fence_mode(vm)) + return; + + down_write(&vm->lock); + if (!list_empty(&q->lr.link)) { + list_del_init(&q->lr.link); + --vm->preempt.num_exec_queues; + } + if (q->lr.pfence) { + dma_fence_enable_signaling(q->lr.pfence); + dma_fence_put(q->lr.pfence); + q->lr.pfence = NULL; + } + up_write(&vm->lock); +} + +#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000 + +/** + * xe_vm_kill() - VM Kill + * @vm: The VM. + * @unlocked: Flag indicates the VM's dma-resv is not held + * + * Kill the VM by setting banned flag indicated VM is no longer available for + * use. If in preempt fence mode, also kill all exec queue attached to the VM. + */ +void xe_vm_kill(struct xe_vm *vm, bool unlocked) +{ + struct xe_exec_queue *q; + + lockdep_assert_held(&vm->lock); + + if (unlocked) + xe_vm_lock(vm, false); + + vm->flags |= XE_VM_FLAG_BANNED; + trace_xe_vm_kill(vm); + + list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) + q->ops->kill(q); + + if (unlocked) + xe_vm_unlock(vm); + + /* TODO: Inform user the VM is banned */ +} + +static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) +{ + struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); + struct xe_bo *bo = gem_to_xe_bo(vm_bo->obj); + struct drm_gpuva *gpuva; + int ret; + + lockdep_assert_held(&vm->lock); + drm_gpuvm_bo_for_each_va(gpuva, vm_bo) + list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind, + &vm->rebind_list); + + /* Skip re-populating purged BOs, rebind maps scratch pages. */ + if (xe_bo_is_purged(bo)) { + vm_bo->evicted = false; + return 0; + } + + if (!try_wait_for_completion(&vm->xe->pm_block)) + return -EAGAIN; + + ret = xe_bo_validate(bo, vm, false, exec); + if (ret) + return ret; + + vm_bo->evicted = false; + return 0; +} + +/** + * xe_vm_validate_rebind() - Validate buffer objects and rebind vmas + * @vm: The vm for which we are rebinding. + * @exec: The struct drm_exec with the locked GEM objects. + * @num_fences: The number of fences to reserve for the operation, not + * including rebinds and validations. + * + * Validates all evicted gem objects and rebinds their vmas. Note that + * rebindings may cause evictions and hence the validation-rebind + * sequence is rerun until there are no more objects to validate. + * + * Return: 0 on success, negative error code on error. In particular, + * may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if + * the drm_exec transaction needs to be restarted. + */ +int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, + unsigned int num_fences) +{ + struct drm_gem_object *obj; + int ret; + + do { + ret = drm_gpuvm_validate(&vm->gpuvm, exec); + if (ret) + return ret; + + ret = xe_vm_rebind(vm, false); + if (ret) + return ret; + } while (!list_empty(&vm->gpuvm.evict.list)); + + drm_exec_for_each_locked_object(exec, obj) { + ret = dma_resv_reserve_fences(obj->resv, num_fences); + if (ret) + return ret; + } + + return 0; +} + +static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm, + bool *done) +{ + int err; + + err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0); + if (err) + return err; + + if (xe_vm_is_idle(vm)) { + vm->preempt.rebind_deactivated = true; + *done = true; + return 0; + } + + if (!preempt_fences_waiting(vm)) { + *done = true; + return 0; + } + + err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0); + if (err) + return err; + + err = wait_for_existing_preempt_fences(vm); + if (err) + return err; + + /* + * Add validation and rebinding to the locking loop since both can + * cause evictions which may require blocing dma_resv locks. + * The fence reservation here is intended for the new preempt fences + * we attach at the end of the rebind work. + */ + return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues); +} + +static bool vm_suspend_rebind_worker(struct xe_vm *vm) +{ + struct xe_device *xe = vm->xe; + bool ret = false; + + mutex_lock(&xe->rebind_resume_lock); + if (!try_wait_for_completion(&vm->xe->pm_block)) { + ret = true; + list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list); + } + mutex_unlock(&xe->rebind_resume_lock); + + return ret; +} + +/** + * xe_vm_resume_rebind_worker() - Resume the rebind worker. + * @vm: The vm whose preempt worker to resume. + * + * Resume a preempt worker that was previously suspended by + * vm_suspend_rebind_worker(). + */ +void xe_vm_resume_rebind_worker(struct xe_vm *vm) +{ + queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work); +} + +static void preempt_rebind_work_func(struct work_struct *w) +{ + struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); + struct xe_validation_ctx ctx; + struct drm_exec exec; + unsigned int fence_count = 0; + LIST_HEAD(preempt_fences); + int err = 0; + long wait; + int __maybe_unused tries = 0; + + xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); + trace_xe_vm_rebind_worker_enter(vm); + + down_write(&vm->lock); + + if (xe_vm_is_closed_or_banned(vm)) { + up_write(&vm->lock); + trace_xe_vm_rebind_worker_exit(vm); + return; + } + +retry: + if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) { + up_write(&vm->lock); + /* We don't actually block but don't make progress. */ + xe_pm_might_block_on_suspend(); + return; + } + + if (xe_vm_userptr_check_repin(vm)) { + err = xe_vm_userptr_pin(vm); + if (err) + goto out_unlock_outer; + } + + err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, + (struct xe_val_flags) {.interruptible = true}); + if (err) + goto out_unlock_outer; + + drm_exec_until_all_locked(&exec) { + bool done = false; + + err = xe_preempt_work_begin(&exec, vm, &done); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + if (err || done) { + xe_validation_ctx_fini(&ctx); + goto out_unlock_outer; + } + } + + err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); + if (err) + goto out_unlock; + + xe_vm_set_validation_exec(vm, &exec); + err = xe_vm_rebind(vm, true); + xe_vm_set_validation_exec(vm, NULL); + if (err) + goto out_unlock; + + /* Wait on rebinds and munmap style VM unbinds */ + wait = dma_resv_wait_timeout(xe_vm_resv(vm), + DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + if (wait <= 0) { + err = -ETIME; + goto out_unlock; + } + +#define retry_required(__tries, __vm) \ + (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \ + (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \ + __xe_vm_userptr_needs_repin(__vm)) + + xe_svm_notifier_lock(vm); + if (retry_required(tries, vm)) { + xe_svm_notifier_unlock(vm); + err = -EAGAIN; + goto out_unlock; + } + +#undef retry_required + + spin_lock(&vm->xe->ttm.lru_lock); + ttm_lru_bulk_move_tail(&vm->lru_bulk_move); + spin_unlock(&vm->xe->ttm.lru_lock); + + /* Point of no return. */ + arm_preempt_fences(vm, &preempt_fences); + resume_and_reinstall_preempt_fences(vm, &exec); + xe_svm_notifier_unlock(vm); + +out_unlock: + xe_validation_ctx_fini(&ctx); +out_unlock_outer: + if (err == -EAGAIN) { + trace_xe_vm_rebind_worker_retry(vm); + + /* + * We can't block in workers on a VF which supports migration + * given this can block the VF post-migration workers from + * getting scheduled. + */ + if (IS_SRIOV_VF(vm->xe) && + xe_sriov_vf_migration_supported(vm->xe)) { + up_write(&vm->lock); + xe_vm_queue_rebind_worker(vm); + return; + } + + goto retry; + } + + if (err) { + drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); + xe_vm_kill(vm, true); + } + up_write(&vm->lock); + + free_preempt_fences(&preempt_fences); + + trace_xe_vm_rebind_worker_exit(vm); +} + +/** + * xe_vm_add_fault_entry_pf() - Add pagefault to vm fault list + * @vm: The VM. + * @pf: The pagefault. + * + * This function takes the data from the pagefault @pf and saves it to @vm->faults.list. + * + * The function exits silently if the list is full, and reports a warning if the pagefault + * could not be saved to the list. + */ +void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf) +{ + struct xe_vm_fault_entry *e; + struct xe_hw_engine *hwe; + + /* Do not report faults on reserved engines */ + hwe = xe_gt_hw_engine(pf->gt, pf->consumer.engine_class, + pf->consumer.engine_instance, false); + if (!hwe || xe_hw_engine_is_reserved(hwe)) + return; + + e = kzalloc_obj(*e); + if (!e) { + drm_warn(&vm->xe->drm, + "Could not allocate memory for fault!\n"); + return; + } + + guard(spinlock)(&vm->faults.lock); + + /* + * Limit the number of faults in the fault list to prevent + * memory overuse. + */ + if (vm->faults.len >= MAX_FAULTS_SAVED_PER_VM) { + kfree(e); + return; + } + + e->address = pf->consumer.page_addr; + /* + * TODO: + * Address precision is currently always SZ_4K, but this may change + * in the future. + */ + e->address_precision = SZ_4K; + e->access_type = pf->consumer.access_type; + e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK, + pf->consumer.fault_type_level); + e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, + pf->consumer.fault_type_level); + + list_add_tail(&e->list, &vm->faults.list); + vm->faults.len++; +} + +static void xe_vm_clear_fault_entries(struct xe_vm *vm) +{ + struct xe_vm_fault_entry *e, *tmp; + + guard(spinlock)(&vm->faults.lock); + list_for_each_entry_safe(e, tmp, &vm->faults.list, list) { + list_del(&e->list); + kfree(e); + } + vm->faults.len = 0; +} + +static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds) +{ + int i; + + for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) { + if (!vops->pt_update_ops[i].num_ops) + continue; + + vops->pt_update_ops[i].ops = + kmalloc_objs(*vops->pt_update_ops[i].ops, + vops->pt_update_ops[i].num_ops, + GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + if (!vops->pt_update_ops[i].ops) + return array_of_binds ? -ENOBUFS : -ENOMEM; + } + + return 0; +} +ALLOW_ERROR_INJECTION(xe_vma_ops_alloc, ERRNO); + +static void xe_vma_svm_prefetch_op_fini(struct xe_vma_op *op) +{ + struct xe_vma *vma; + + vma = gpuva_to_vma(op->base.prefetch.va); + + if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma)) + xa_destroy(&op->prefetch_range.range); +} + +static void xe_vma_svm_prefetch_ops_fini(struct xe_vma_ops *vops) +{ + struct xe_vma_op *op; + + if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) + return; + + list_for_each_entry(op, &vops->list, link) + xe_vma_svm_prefetch_op_fini(op); +} + +static void xe_vma_ops_fini(struct xe_vma_ops *vops) +{ + int i; + + xe_vma_svm_prefetch_ops_fini(vops); + + for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) + kfree(vops->pt_update_ops[i].ops); +} + +static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val) +{ + int i; + + if (!inc_val) + return; + + for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) + if (BIT(i) & tile_mask) + vops->pt_update_ops[i].num_ops += inc_val; +} + +#define XE_VMA_CREATE_MASK ( \ + XE_VMA_READ_ONLY | \ + XE_VMA_DUMPABLE | \ + XE_VMA_SYSTEM_ALLOCATOR | \ + DRM_GPUVA_SPARSE | \ + XE_VMA_MADV_AUTORESET) + +static void xe_vm_populate_rebind(struct xe_vma_op *op, struct xe_vma *vma, + u8 tile_mask) +{ + INIT_LIST_HEAD(&op->link); + op->tile_mask = tile_mask; + op->base.op = DRM_GPUVA_OP_MAP; + op->base.map.va.addr = vma->gpuva.va.addr; + op->base.map.va.range = vma->gpuva.va.range; + op->base.map.gem.obj = vma->gpuva.gem.obj; + op->base.map.gem.offset = vma->gpuva.gem.offset; + op->map.vma = vma; + op->map.immediate = true; + op->map.vma_flags = vma->gpuva.flags & XE_VMA_CREATE_MASK; +} + +static int xe_vm_ops_add_rebind(struct xe_vma_ops *vops, struct xe_vma *vma, + u8 tile_mask) +{ + struct xe_vma_op *op; + + op = kzalloc_obj(*op); + if (!op) + return -ENOMEM; + + xe_vm_populate_rebind(op, vma, tile_mask); + list_add_tail(&op->link, &vops->list); + xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1); + + return 0; +} + +static struct dma_fence *ops_execute(struct xe_vm *vm, + struct xe_vma_ops *vops); +static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, + struct xe_exec_queue *q, + struct xe_sync_entry *syncs, u32 num_syncs); + +int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) +{ + struct dma_fence *fence; + struct xe_vma *vma, *next; + struct xe_vma_ops vops; + struct xe_vma_op *op, *next_op; + int err, i; + + lockdep_assert_held(&vm->lock); + if ((xe_vm_in_lr_mode(vm) && !rebind_worker) || + list_empty(&vm->rebind_list)) + return 0; + + xe_vma_ops_init(&vops, vm, NULL, NULL, 0); + for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) + vops.pt_update_ops[i].wait_vm_bookkeep = true; + + xe_vm_assert_held(vm); + list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) { + xe_assert(vm->xe, vma->tile_present); + + if (rebind_worker) + trace_xe_vma_rebind_worker(vma); + else + trace_xe_vma_rebind_exec(vma); + + err = xe_vm_ops_add_rebind(&vops, vma, + vma->tile_present); + if (err) + goto free_ops; + } + + err = xe_vma_ops_alloc(&vops, false); + if (err) + goto free_ops; + + fence = ops_execute(vm, &vops); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + } else { + dma_fence_put(fence); + list_for_each_entry_safe(vma, next, &vm->rebind_list, + combined_links.rebind) + list_del_init(&vma->combined_links.rebind); + } +free_ops: + list_for_each_entry_safe(op, next_op, &vops.list, link) { + list_del(&op->link); + kfree(op); + } + xe_vma_ops_fini(&vops); + + return err; +} + +struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask) +{ + struct dma_fence *fence = NULL; + struct xe_vma_ops vops; + struct xe_vma_op *op, *next_op; + struct xe_tile *tile; + u8 id; + int err; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); + + xe_vma_ops_init(&vops, vm, NULL, NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT; + for_each_tile(tile, vm->xe, id) { + vops.pt_update_ops[id].wait_vm_bookkeep = true; + vops.pt_update_ops[tile->id].q = + xe_migrate_exec_queue(tile->migrate); + } + + err = xe_vm_ops_add_rebind(&vops, vma, tile_mask); + if (err) + return ERR_PTR(err); + + err = xe_vma_ops_alloc(&vops, false); + if (err) { + fence = ERR_PTR(err); + goto free_ops; + } + + fence = ops_execute(vm, &vops); + +free_ops: + list_for_each_entry_safe(op, next_op, &vops.list, link) { + list_del(&op->link); + kfree(op); + } + xe_vma_ops_fini(&vops); + + return fence; +} + +static void xe_vm_populate_range_rebind(struct xe_vma_op *op, + struct xe_vma *vma, + struct xe_svm_range *range, + u8 tile_mask) +{ + INIT_LIST_HEAD(&op->link); + op->tile_mask = tile_mask; + op->base.op = DRM_GPUVA_OP_DRIVER; + op->subop = XE_VMA_SUBOP_MAP_RANGE; + op->map_range.vma = vma; + op->map_range.range = range; +} + +static int +xe_vm_ops_add_range_rebind(struct xe_vma_ops *vops, + struct xe_vma *vma, + struct xe_svm_range *range, + u8 tile_mask) +{ + struct xe_vma_op *op; + + op = kzalloc_obj(*op); + if (!op) + return -ENOMEM; + + xe_vm_populate_range_rebind(op, vma, range, tile_mask); + list_add_tail(&op->link, &vops->list); + xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1); + + return 0; +} + +/** + * xe_vm_range_rebind() - VM range (re)bind + * @vm: The VM which the range belongs to. + * @vma: The VMA which the range belongs to. + * @range: SVM range to rebind. + * @tile_mask: Tile mask to bind the range to. + * + * (re)bind SVM range setting up GPU page tables for the range. + * + * Return: dma fence for rebind to signal completion on success, ERR_PTR on + * failure + */ +struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, + struct xe_vma *vma, + struct xe_svm_range *range, + u8 tile_mask) +{ + struct dma_fence *fence = NULL; + struct xe_vma_ops vops; + struct xe_vma_op *op, *next_op; + struct xe_tile *tile; + u8 id; + int err; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); + xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); + + xe_vma_ops_init(&vops, vm, NULL, NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT; + for_each_tile(tile, vm->xe, id) { + vops.pt_update_ops[id].wait_vm_bookkeep = true; + vops.pt_update_ops[tile->id].q = + xe_migrate_exec_queue(tile->migrate); + } + + err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask); + if (err) + return ERR_PTR(err); + + err = xe_vma_ops_alloc(&vops, false); + if (err) { + fence = ERR_PTR(err); + goto free_ops; + } + + fence = ops_execute(vm, &vops); + +free_ops: + list_for_each_entry_safe(op, next_op, &vops.list, link) { + list_del(&op->link); + kfree(op); + } + xe_vma_ops_fini(&vops); + + return fence; +} + +static void xe_vm_populate_range_unbind(struct xe_vma_op *op, + struct xe_svm_range *range) +{ + INIT_LIST_HEAD(&op->link); + op->tile_mask = range->tile_present; + op->base.op = DRM_GPUVA_OP_DRIVER; + op->subop = XE_VMA_SUBOP_UNMAP_RANGE; + op->unmap_range.range = range; +} + +static int +xe_vm_ops_add_range_unbind(struct xe_vma_ops *vops, + struct xe_svm_range *range) +{ + struct xe_vma_op *op; + + op = kzalloc_obj(*op); + if (!op) + return -ENOMEM; + + xe_vm_populate_range_unbind(op, range); + list_add_tail(&op->link, &vops->list); + xe_vma_ops_incr_pt_update_ops(vops, range->tile_present, 1); + + return 0; +} + +/** + * xe_vm_range_unbind() - VM range unbind + * @vm: The VM which the range belongs to. + * @range: SVM range to rebind. + * + * Unbind SVM range removing the GPU page tables for the range. + * + * Return: dma fence for unbind to signal completion on success, ERR_PTR on + * failure + */ +struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, + struct xe_svm_range *range) +{ + struct dma_fence *fence = NULL; + struct xe_vma_ops vops; + struct xe_vma_op *op, *next_op; + struct xe_tile *tile; + u8 id; + int err; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); + + if (!range->tile_present) + return dma_fence_get_stub(); + + xe_vma_ops_init(&vops, vm, NULL, NULL, 0); + for_each_tile(tile, vm->xe, id) { + vops.pt_update_ops[id].wait_vm_bookkeep = true; + vops.pt_update_ops[tile->id].q = + xe_migrate_exec_queue(tile->migrate); + } + + err = xe_vm_ops_add_range_unbind(&vops, range); + if (err) + return ERR_PTR(err); + + err = xe_vma_ops_alloc(&vops, false); + if (err) { + fence = ERR_PTR(err); + goto free_ops; + } + + fence = ops_execute(vm, &vops); + +free_ops: + list_for_each_entry_safe(op, next_op, &vops.list, link) { + list_del(&op->link); + kfree(op); + } + xe_vma_ops_fini(&vops); + + return fence; +} + +static void xe_vma_mem_attr_fini(struct xe_vma_mem_attr *attr) +{ + drm_pagemap_put(attr->preferred_loc.dpagemap); +} + +static void xe_vma_free(struct xe_vma *vma) +{ + xe_vma_mem_attr_fini(&vma->attr); + + if (xe_vma_is_userptr(vma)) + kfree(to_userptr_vma(vma)); + else + kfree(vma); +} + +/** + * xe_vma_mem_attr_copy() - copy an xe_vma_mem_attr structure. + * @to: Destination. + * @from: Source. + * + * Copies an xe_vma_mem_attr structure taking care to get reference + * counting of individual members right. + */ +void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from) +{ + xe_vma_mem_attr_fini(to); + *to = *from; + if (to->preferred_loc.dpagemap) + drm_pagemap_get(to->preferred_loc.dpagemap); +} + +static struct xe_vma *xe_vma_create(struct xe_vm *vm, + struct xe_bo *bo, + u64 bo_offset_or_userptr, + u64 start, u64 end, + struct xe_vma_mem_attr *attr, + unsigned int flags) +{ + struct xe_vma *vma; + struct xe_tile *tile; + u8 id; + bool is_null = (flags & DRM_GPUVA_SPARSE); + bool is_cpu_addr_mirror = (flags & XE_VMA_SYSTEM_ALLOCATOR); + + xe_assert(vm->xe, start < end); + xe_assert(vm->xe, end < vm->size); + + /* + * Allocate and ensure that the xe_vma_is_userptr() return + * matches what was allocated. + */ + if (!bo && !is_null && !is_cpu_addr_mirror) { + struct xe_userptr_vma *uvma = kzalloc_obj(*uvma); + + if (!uvma) + return ERR_PTR(-ENOMEM); + + vma = &uvma->vma; + } else { + vma = kzalloc_obj(*vma); + if (!vma) + return ERR_PTR(-ENOMEM); + + if (bo) + vma->gpuva.gem.obj = &bo->ttm.base; + } + + INIT_LIST_HEAD(&vma->combined_links.rebind); + + INIT_LIST_HEAD(&vma->gpuva.gem.entry); + vma->gpuva.vm = &vm->gpuvm; + vma->gpuva.va.addr = start; + vma->gpuva.va.range = end - start + 1; + vma->gpuva.flags = flags; + + for_each_tile(tile, vm->xe, id) + vma->tile_mask |= 0x1 << id; + + if (vm->xe->info.has_atomic_enable_pte_bit) + vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; + + xe_vma_mem_attr_copy(&vma->attr, attr); + if (bo) { + struct drm_gpuvm_bo *vm_bo; + + xe_bo_assert_held(bo); + + /* + * Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This + * gates new vm_bind ioctls (user supplies WILLNEED) while + * still allowing partial-unbind / remap splits whose new VMAs + * inherit the parent's DONTNEED attr. It must also run before + * xe_bo_willneed_get_locked() below so a 0->1 holder bump + * cannot silently promote DONTNEED back to WILLNEED. + */ + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { + if (xe_bo_madv_is_dontneed(bo)) { + xe_vma_free(vma); + return ERR_PTR(-EBUSY); + } + if (xe_bo_is_purged(bo)) { + xe_vma_free(vma); + return ERR_PTR(-EINVAL); + } + } + + vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base); + if (IS_ERR(vm_bo)) { + xe_vma_free(vma); + return ERR_CAST(vm_bo); + } + + drm_gpuvm_bo_extobj_add(vm_bo); + drm_gem_object_get(&bo->ttm.base); + vma->gpuva.gem.offset = bo_offset_or_userptr; + drm_gpuva_link(&vma->gpuva, vm_bo); + drm_gpuvm_bo_put(vm_bo); + + xe_bo_vma_count_inc_locked(bo); + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) + xe_bo_willneed_get_locked(bo); + } else /* userptr or null */ { + if (!is_null && !is_cpu_addr_mirror) { + struct xe_userptr_vma *uvma = to_userptr_vma(vma); + u64 size = end - start + 1; + int err; + + vma->gpuva.gem.offset = bo_offset_or_userptr; + + err = xe_userptr_setup(uvma, xe_vma_userptr(vma), size); + if (err) { + xe_vma_free(vma); + return ERR_PTR(err); + } + } + + xe_vm_get(vm); + } + + return vma; +} + +static void xe_vma_destroy_late(struct xe_vma *vma) +{ + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_bo *bo = xe_vma_bo(vma); + + if (vma->ufence) { + xe_sync_ufence_put(vma->ufence); + vma->ufence = NULL; + } + + if (xe_vma_is_userptr(vma)) { + struct xe_userptr_vma *uvma = to_userptr_vma(vma); + + xe_userptr_remove(uvma); + xe_vm_put(vm); + } else if (xe_vma_is_null(vma) || xe_vma_is_cpu_addr_mirror(vma)) { + xe_vm_put(vm); + } else { + xe_bo_put(bo); + } + + xe_vma_free(vma); +} + +static void vma_destroy_work_func(struct work_struct *w) +{ + struct xe_vma *vma = + container_of(w, struct xe_vma, destroy_work); + + xe_vma_destroy_late(vma); +} + +static void vma_destroy_cb(struct dma_fence *fence, + struct dma_fence_cb *cb) +{ + struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb); + + INIT_WORK(&vma->destroy_work, vma_destroy_work_func); + queue_work(system_dfl_wq, &vma->destroy_work); +} + +static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) +{ + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_bo *bo = xe_vma_bo(vma); + + lockdep_assert_held_write(&vm->lock); + xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); + + if (xe_vma_is_userptr(vma)) { + xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); + xe_userptr_destroy(to_userptr_vma(vma)); + } else if (!xe_vma_is_null(vma) && !xe_vma_is_cpu_addr_mirror(vma)) { + xe_bo_assert_held(bo); + + drm_gpuva_unlink(&vma->gpuva); + + xe_bo_vma_count_dec_locked(bo); + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) + xe_bo_willneed_put_locked(bo); + } + + xe_vm_assert_held(vm); + if (fence) { + int ret = dma_fence_add_callback(fence, &vma->destroy_cb, + vma_destroy_cb); + + if (ret) { + XE_WARN_ON(ret != -ENOENT); + xe_vma_destroy_late(vma); + } + } else { + xe_vma_destroy_late(vma); + } +} + +/** + * xe_vm_lock_vma() - drm_exec utility to lock a vma + * @exec: The drm_exec object we're currently locking for. + * @vma: The vma for witch we want to lock the vm resv and any attached + * object's resv. + * + * Return: 0 on success, negative error code on error. In particular + * may return -EDEADLK on WW transaction contention and -EINTR if + * an interruptible wait is terminated by a signal. + */ +int xe_vm_lock_vma(struct drm_exec *exec, struct xe_vma *vma) +{ + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_bo *bo = xe_vma_bo(vma); + int err; + + XE_WARN_ON(!vm); + + err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); + if (!err && bo && !bo->vm) + err = drm_exec_lock_obj(exec, &bo->ttm.base); + + return err; +} + +static void xe_vma_destroy_unlocked(struct xe_vma *vma) +{ + struct xe_device *xe = xe_vma_vm(vma)->xe; + struct xe_validation_ctx ctx; + struct drm_exec exec; + int err = 0; + + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { + err = xe_vm_lock_vma(&exec, vma); + drm_exec_retry_on_contention(&exec); + if (XE_WARN_ON(err)) + break; + xe_vma_destroy(vma, NULL); + } + xe_assert(xe, !err); +} + +struct xe_vma * +xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range) +{ + struct drm_gpuva *gpuva; + + lockdep_assert_held(&vm->lock); + + if (xe_vm_is_closed_or_banned(vm)) + return NULL; + + xe_assert(vm->xe, start + range <= vm->size); + + gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range); + + return gpuva ? gpuva_to_vma(gpuva) : NULL; +} + +static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) +{ + int err; + + xe_assert(vm->xe, xe_vma_vm(vma) == vm); + lockdep_assert_held(&vm->lock); + + mutex_lock(&vm->snap_mutex); + err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva); + mutex_unlock(&vm->snap_mutex); + XE_WARN_ON(err); /* Shouldn't be possible */ + + return err; +} + +static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) +{ + xe_assert(vm->xe, xe_vma_vm(vma) == vm); + lockdep_assert_held(&vm->lock); + + mutex_lock(&vm->snap_mutex); + drm_gpuva_remove(&vma->gpuva); + mutex_unlock(&vm->snap_mutex); + if (vm->usm.last_fault_vma == vma) + vm->usm.last_fault_vma = NULL; +} + +static struct drm_gpuva_op *xe_vm_op_alloc(void) +{ + struct xe_vma_op *op; + + op = kzalloc_obj(*op); + + if (unlikely(!op)) + return NULL; + + return &op->base; +} + +static void xe_vm_free(struct drm_gpuvm *gpuvm); + +static const struct drm_gpuvm_ops gpuvm_ops = { + .op_alloc = xe_vm_op_alloc, + .vm_bo_validate = xe_gpuvm_validate, + .vm_free = xe_vm_free, +}; + +static u64 pde_encode_pat_index(u16 pat_index) +{ + u64 pte = 0; + + if (pat_index & BIT(0)) + pte |= XE_PPGTT_PTE_PAT0; + + if (pat_index & BIT(1)) + pte |= XE_PPGTT_PTE_PAT1; + + return pte; +} + +static u64 pte_encode_pat_index(u16 pat_index, u32 pt_level) +{ + u64 pte = 0; + + if (pat_index & BIT(0)) + pte |= XE_PPGTT_PTE_PAT0; + + if (pat_index & BIT(1)) + pte |= XE_PPGTT_PTE_PAT1; + + if (pat_index & BIT(2)) { + if (pt_level) + pte |= XE_PPGTT_PDE_PDPE_PAT2; + else + pte |= XE_PPGTT_PTE_PAT2; + } + + if (pat_index & BIT(3)) + pte |= XELPG_PPGTT_PTE_PAT3; + + if (pat_index & (BIT(4))) + pte |= XE2_PPGTT_PTE_PAT4; + + return pte; +} + +static u64 pte_encode_ps(u32 pt_level) +{ + XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL); + + if (pt_level == 1) + return XE_PDE_PS_2M; + else if (pt_level == 2) + return XE_PDPE_PS_1G; + + return 0; +} + +static u16 pde_pat_index(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + u16 pat_index; + + /* + * We only have two bits to encode the PAT index in non-leaf nodes, but + * these only point to other paging structures so we only need a minimal + * selection of options. The user PAT index is only for encoding leaf + * nodes, where we have use of more bits to do the encoding. The + * non-leaf nodes are instead under driver control so the chosen index + * here should be distinct from the user PAT index. Also the + * corresponding coherency of the PAT index should be tied to the + * allocation type of the page table (or at least we should pick + * something which is always safe). + */ + if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) + pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); + else + pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE); + + xe_assert(xe, pat_index <= 3); + + return pat_index; +} + +static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset) +{ + u64 pde; + + pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); + pde |= XE_PAGE_PRESENT | XE_PAGE_RW; + pde |= pde_encode_pat_index(pde_pat_index(bo)); + + return pde; +} + +static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, + u16 pat_index, u32 pt_level) +{ + u64 pte; + + pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); + pte |= XE_PAGE_PRESENT | XE_PAGE_RW; + pte |= pte_encode_pat_index(pat_index, pt_level); + pte |= pte_encode_ps(pt_level); + + if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)) + pte |= XE_PPGTT_PTE_DM; + + return pte; +} + +static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma, + u16 pat_index, u32 pt_level) +{ + struct xe_bo *bo = xe_vma_bo(vma); + struct xe_vm *vm = xe_vma_vm(vma); + + pte |= XE_PAGE_PRESENT; + + if (likely(!xe_vma_read_only(vma))) + pte |= XE_PAGE_RW; + + pte |= pte_encode_pat_index(pat_index, pt_level); + pte |= pte_encode_ps(pt_level); + + /* + * NULL PTEs redirect to scratch page (return zeros on read). + * Set for: 1) explicit null VMAs, 2) purged BOs on scratch VMs. + * Never set NULL flag without scratch page - causes undefined behavior. + */ + if (unlikely(xe_vma_is_null(vma) || + (bo && xe_bo_is_purged(bo) && xe_vm_has_scratch(vm)))) + pte |= XE_PTE_NULL; + + return pte; +} + +static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr, + u16 pat_index, + u32 pt_level, bool devmem, u64 flags) +{ + u64 pte; + + /* Avoid passing random bits directly as flags */ + xe_assert(xe, !(flags & ~XE_PTE_PS64)); + + pte = addr; + pte |= XE_PAGE_PRESENT | XE_PAGE_RW; + pte |= pte_encode_pat_index(pat_index, pt_level); + pte |= pte_encode_ps(pt_level); + + if (devmem) + pte |= XE_PPGTT_PTE_DM; + + pte |= flags; + + return pte; +} + +static const struct xe_pt_ops xelp_pt_ops = { + .pte_encode_bo = xelp_pte_encode_bo, + .pte_encode_vma = xelp_pte_encode_vma, + .pte_encode_addr = xelp_pte_encode_addr, + .pde_encode_bo = xelp_pde_encode_bo, +}; + +static void vm_destroy_work_func(struct work_struct *w); + +/** + * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the + * given tile and vm. + * @xe: xe device. + * @tile: tile to set up for. + * @vm: vm to set up for. + * @exec: The struct drm_exec object used to lock the vm resv. + * + * Sets up a pagetable tree with one page-table per level and a single + * leaf PTE. All pagetable entries point to the single page-table or, + * for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and + * writes become NOPs. + * + * Return: 0 on success, negative error code on error. + */ +static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, struct drm_exec *exec) +{ + u8 id = tile->id; + int i; + + for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { + vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i, exec); + if (IS_ERR(vm->scratch_pt[id][i])) { + int err = PTR_ERR(vm->scratch_pt[id][i]); + + vm->scratch_pt[id][i] = NULL; + return err; + } + xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); + } + + return 0; +} +ALLOW_ERROR_INJECTION(xe_vm_create_scratch, ERRNO); + +static void xe_vm_free_scratch(struct xe_vm *vm) +{ + struct xe_tile *tile; + u8 id; + + if (!xe_vm_has_scratch(vm)) + return; + + for_each_tile(tile, vm->xe, id) { + u32 i; + + if (!vm->pt_root[id]) + continue; + + for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i) + if (vm->scratch_pt[id][i]) + xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL); + } +} + +static void xe_vm_pt_destroy(struct xe_vm *vm) +{ + struct xe_tile *tile; + u8 id; + + xe_vm_assert_held(vm); + + for_each_tile(tile, vm->xe, id) { + if (vm->pt_root[id]) { + xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); + vm->pt_root[id] = NULL; + } + } +} + +static void xe_vm_init_prove_locking(struct xe_device *xe, struct xe_vm *vm) +{ + if (!IS_ENABLED(CONFIG_PROVE_LOCKING)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&vm->exec_queues.lock); + fs_reclaim_release(GFP_KERNEL); + + down_read(&vm->exec_queues.lock); + might_lock(&xe_root_mmio_gt(xe)->uc.guc.ct.lock); + up_read(&vm->exec_queues.lock); +} + +struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) +{ + struct drm_gem_object *vm_resv_obj; + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_vm *vm; + int err; + struct xe_tile *tile; + u8 id; + + /* + * Since the GSCCS is not user-accessible, we don't expect a GSC VM to + * ever be in faulting mode. + */ + xe_assert(xe, !((flags & XE_VM_FLAG_GSC) && (flags & XE_VM_FLAG_FAULT_MODE))); + + vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return ERR_PTR(-ENOMEM); + + vm->xe = xe; + + vm->size = 1ull << xe->info.va_bits; + vm->flags = flags; + + if (xef) + vm->xef = xe_file_get(xef); + /* + * GSC VMs are kernel-owned, only used for PXP ops and can sometimes be + * manipulated under the PXP mutex. However, the PXP mutex can be taken + * under a user-VM lock when the PXP session is started at exec_queue + * creation time. Those are different VMs and therefore there is no risk + * of deadlock, but we need to tell lockdep that this is the case or it + * will print a warning. + */ + if (flags & XE_VM_FLAG_GSC) { + static struct lock_class_key gsc_vm_key; + + __init_rwsem(&vm->lock, "gsc_vm", &gsc_vm_key); + } else { + init_rwsem(&vm->lock); + } + mutex_init(&vm->snap_mutex); + + INIT_LIST_HEAD(&vm->rebind_list); + + INIT_LIST_HEAD(&vm->userptr.repin_list); + INIT_LIST_HEAD(&vm->userptr.invalidated); + spin_lock_init(&vm->userptr.invalidated_lock); + + INIT_LIST_HEAD(&vm->faults.list); + spin_lock_init(&vm->faults.lock); + + ttm_lru_bulk_move_init(&vm->lru_bulk_move); + + INIT_WORK(&vm->destroy_work, vm_destroy_work_func); + + INIT_LIST_HEAD(&vm->preempt.exec_queues); + for (id = 0; id < XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE; ++id) + INIT_LIST_HEAD(&vm->exec_queues.list[id]); + if (flags & XE_VM_FLAG_FAULT_MODE) + vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms; + else + vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms; + + init_rwsem(&vm->exec_queues.lock); + xe_vm_init_prove_locking(xe, vm); + + for_each_tile(tile, xe, id) + xe_range_fence_tree_init(&vm->rftree[id]); + + vm->pt_ops = &xelp_pt_ops; + + /* + * Long-running workloads are not protected by the scheduler references. + * By design, run_job for long-running workloads returns NULL and the + * scheduler drops all the references of it, hence protecting the VM + * for this case is necessary. + */ + if (flags & XE_VM_FLAG_LR_MODE) { + INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); + xe_pm_runtime_get_noresume(xe); + INIT_LIST_HEAD(&vm->preempt.pm_activate_link); + } + + err = xe_svm_init(vm); + if (err) + goto err_no_resv; + + vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); + if (!vm_resv_obj) { + err = -ENOMEM; + goto err_svm_fini; + } + + drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, + vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops); + + drm_gem_object_put(vm_resv_obj); + + err = 0; + xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, + err) { + err = xe_vm_drm_exec_lock(vm, &exec); + drm_exec_retry_on_contention(&exec); + + if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + vm->flags |= XE_VM_FLAG_64K; + + for_each_tile(tile, xe, id) { + if (flags & XE_VM_FLAG_MIGRATION && + tile->id != XE_VM_FLAG_TILE_ID(flags)) + continue; + + vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level, + &exec); + if (IS_ERR(vm->pt_root[id])) { + err = PTR_ERR(vm->pt_root[id]); + vm->pt_root[id] = NULL; + xe_vm_pt_destroy(vm); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + } + if (err) + break; + + if (xe_vm_has_scratch(vm)) { + for_each_tile(tile, xe, id) { + if (!vm->pt_root[id]) + continue; + + err = xe_vm_create_scratch(xe, tile, vm, &exec); + if (err) { + xe_vm_free_scratch(vm); + xe_vm_pt_destroy(vm); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + break; + } + } + if (err) + break; + vm->batch_invalidate_tlb = true; + } + + if (vm->flags & XE_VM_FLAG_LR_MODE) + vm->batch_invalidate_tlb = false; + + /* Fill pt_root after allocating scratch tables */ + for_each_tile(tile, xe, id) { + if (!vm->pt_root[id]) + continue; + + xe_pt_populate_empty(tile, vm, vm->pt_root[id]); + } + } + if (err) + goto err_close; + + /* Kernel migration VM shouldn't have a circular loop.. */ + if (!(flags & XE_VM_FLAG_MIGRATION)) { + for_each_tile(tile, xe, id) { + struct xe_exec_queue *q; + u32 create_flags = EXEC_QUEUE_FLAG_VM; + + if (!vm->pt_root[id]) + continue; + + if (!xef) /* Not from userspace */ + create_flags |= EXEC_QUEUE_FLAG_KERNEL; + + q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0); + if (IS_ERR(q)) { + err = PTR_ERR(q); + goto err_close; + } + vm->q[id] = q; + } + } + + if (xef && xe->info.has_asid) { + u32 asid; + + down_write(&xe->usm.lock); + err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, + XA_LIMIT(1, XE_MAX_ASID - 1), + &xe->usm.next_asid, GFP_NOWAIT); + up_write(&xe->usm.lock); + if (err < 0) + goto err_close; + + vm->usm.asid = asid; + } + + trace_xe_vm_create(vm); + + return vm; + +err_close: + xe_vm_close_and_put(vm); + return ERR_PTR(err); + +err_svm_fini: + vm->size = 0; /* close the vm */ + if (flags & XE_VM_FLAG_FAULT_MODE) + xe_svm_close(vm); + xe_svm_fini(vm); +err_no_resv: + mutex_destroy(&vm->snap_mutex); + for_each_tile(tile, xe, id) + xe_range_fence_tree_fini(&vm->rftree[id]); + ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); + if (vm->xef) + xe_file_put(vm->xef); + kfree(vm); + if (flags & XE_VM_FLAG_LR_MODE) + xe_pm_runtime_put(xe); + return ERR_PTR(err); +} + +static void xe_vm_close(struct xe_vm *vm) +{ + struct xe_device *xe = vm->xe; + bool bound; + int idx; + + bound = drm_dev_enter(&xe->drm, &idx); + + down_write(&vm->lock); + if (xe_vm_in_fault_mode(vm)) + xe_svm_notifier_lock(vm); + + vm->size = 0; + + if (!((vm->flags & XE_VM_FLAG_MIGRATION))) { + struct xe_tile *tile; + struct xe_gt *gt; + u8 id; + + /* Wait for pending binds */ + dma_resv_wait_timeout(xe_vm_resv(vm), + DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + + if (bound) { + for_each_tile(tile, xe, id) + if (vm->pt_root[id]) + xe_pt_clear(xe, vm->pt_root[id]); + + for_each_gt(gt, xe, id) + xe_tlb_inval_vm(>->tlb_inval, vm); + } + } + + if (xe_vm_in_fault_mode(vm)) + xe_svm_notifier_unlock(vm); + up_write(&vm->lock); + + if (bound) + drm_dev_exit(idx); +} + +void xe_vm_close_and_put(struct xe_vm *vm) +{ + LIST_HEAD(contested); + struct xe_device *xe = vm->xe; + struct xe_tile *tile; + struct xe_vma *vma, *next_vma; + struct drm_gpuva *gpuva, *next; + u8 id; + + xe_assert(xe, !vm->preempt.num_exec_queues); + + xe_vm_close(vm); + if (xe_vm_in_preempt_fence_mode(vm)) { + mutex_lock(&xe->rebind_resume_lock); + list_del_init(&vm->preempt.pm_activate_link); + mutex_unlock(&xe->rebind_resume_lock); + flush_work(&vm->preempt.rebind_work); + } + if (xe_vm_in_fault_mode(vm)) + xe_svm_close(vm); + + down_write(&vm->lock); + for_each_tile(tile, xe, id) { + if (vm->q[id]) { + int i; + + xe_exec_queue_last_fence_put(vm->q[id], vm); + for_each_tlb_inval(i) + xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i); + } + } + up_write(&vm->lock); + + for_each_tile(tile, xe, id) { + if (vm->q[id]) { + xe_exec_queue_kill(vm->q[id]); + xe_exec_queue_put(vm->q[id]); + vm->q[id] = NULL; + } + } + + down_write(&vm->lock); + xe_vm_lock(vm, false); + drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) { + vma = gpuva_to_vma(gpuva); + + if (xe_vma_has_no_bo(vma)) { + xe_svm_notifier_lock(vm); + vma->gpuva.flags |= XE_VMA_DESTROYED; + xe_svm_notifier_unlock(vm); + } + + xe_vm_remove_vma(vm, vma); + + /* easy case, remove from VMA? */ + if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) { + list_del_init(&vma->combined_links.rebind); + xe_vma_destroy(vma, NULL); + continue; + } + + list_move_tail(&vma->combined_links.destroy, &contested); + vma->gpuva.flags |= XE_VMA_DESTROYED; + } + + /* + * All vm operations will add shared fences to resv. + * The only exception is eviction for a shared object, + * but even so, the unbind when evicted would still + * install a fence to resv. Hence it's safe to + * destroy the pagetables immediately. + */ + xe_vm_free_scratch(vm); + xe_vm_pt_destroy(vm); + xe_vm_unlock(vm); + + /* + * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL + * Since we hold a refcount to the bo, we can remove and free + * the members safely without locking. + */ + list_for_each_entry_safe(vma, next_vma, &contested, + combined_links.destroy) { + list_del_init(&vma->combined_links.destroy); + xe_vma_destroy_unlocked(vma); + } + + xe_svm_fini(vm); + + up_write(&vm->lock); + + down_write(&xe->usm.lock); + if (vm->usm.asid) { + void *lookup; + + xe_assert(xe, xe->info.has_asid); + xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION)); + + lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); + xe_assert(xe, lookup == vm); + } + up_write(&xe->usm.lock); + + xe_vm_clear_fault_entries(vm); + + for_each_tile(tile, xe, id) + xe_range_fence_tree_fini(&vm->rftree[id]); + + xe_vm_put(vm); +} + +static void vm_destroy_work_func(struct work_struct *w) +{ + struct xe_vm *vm = + container_of(w, struct xe_vm, destroy_work); + struct xe_device *xe = vm->xe; + struct xe_tile *tile; + u8 id; + + /* xe_vm_close_and_put was not called? */ + xe_assert(xe, !vm->size); + + if (xe_vm_in_preempt_fence_mode(vm)) + flush_work(&vm->preempt.rebind_work); + + mutex_destroy(&vm->snap_mutex); + + if (vm->flags & XE_VM_FLAG_LR_MODE) + xe_pm_runtime_put(xe); + + for_each_tile(tile, xe, id) + XE_WARN_ON(vm->pt_root[id]); + + trace_xe_vm_free(vm); + + ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); + + if (vm->xef) + xe_file_put(vm->xef); + + kfree(vm); +} + +static void xe_vm_free(struct drm_gpuvm *gpuvm) +{ + struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm); + + /* To destroy the VM we need to be able to sleep */ + queue_work(system_dfl_wq, &vm->destroy_work); +} + +struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) +{ + struct xe_vm *vm; + + mutex_lock(&xef->vm.lock); + vm = xa_load(&xef->vm.xa, id); + if (vm) + xe_vm_get(vm); + mutex_unlock(&xef->vm.lock); + + return vm; +} + +u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) +{ + return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0); +} + +static struct xe_exec_queue * +to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) +{ + return q ? q : vm->q[0]; +} + +static struct xe_user_fence * +find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs) +{ + unsigned int i; + + for (i = 0; i < num_syncs; i++) { + struct xe_sync_entry *e = &syncs[i]; + + if (xe_sync_is_ufence(e)) + return xe_sync_ufence_get(e); + } + + return NULL; +} + +#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ + DRM_XE_VM_CREATE_FLAG_LR_MODE | \ + DRM_XE_VM_CREATE_FLAG_FAULT_MODE | \ + DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT) + +int xe_vm_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_create *args = data; + struct xe_gt *wa_gt = xe_root_mmio_gt(xe); + struct xe_vm *vm; + u32 id; + int err; + u32 flags = 0; + + if (XE_IOCTL_DBG(xe, args->extensions)) + return -EINVAL; + + if (wa_gt && XE_GT_WA(wa_gt, 22014953428)) + args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; + + if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && + !xe->info.has_usm)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && + args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && + !xe->info.needs_scratch)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && + args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) && + args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT)) + return -EINVAL; + + if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) + flags |= XE_VM_FLAG_SCRATCH_PAGE; + if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) + flags |= XE_VM_FLAG_LR_MODE; + if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) + flags |= XE_VM_FLAG_FAULT_MODE; + if (args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT) + flags |= XE_VM_FLAG_NO_VM_OVERCOMMIT; + + vm = xe_vm_create(xe, flags, xef); + if (IS_ERR(vm)) + return PTR_ERR(vm); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) + /* Warning: Security issue - never enable by default */ + args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); +#endif + + /* user id alloc must always be last in ioctl to prevent UAF */ + err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); + if (err) + goto err_close_and_put; + + args->vm_id = id; + + return 0; + +err_close_and_put: + xe_vm_close_and_put(vm); + + return err; +} + +int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_destroy *args = data; + struct xe_vm *vm; + int err = 0; + + if (XE_IOCTL_DBG(xe, args->pad) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + mutex_lock(&xef->vm.lock); + vm = xa_load(&xef->vm.xa, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + err = -ENOENT; + else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) + err = -EBUSY; + else + xa_erase(&xef->vm.xa, args->vm_id); + mutex_unlock(&xef->vm.lock); + + if (!err) + xe_vm_close_and_put(vm); + + return err; +} + +static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpuva *gpuva; + u32 num_vmas = 0; + + lockdep_assert_held(&vm->lock); + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) + num_vmas++; + + return num_vmas; +} + +static int get_mem_attrs(struct xe_vm *vm, u32 *num_vmas, u64 start, + u64 end, struct drm_xe_mem_range_attr *attrs) +{ + struct drm_gpuva *gpuva; + int i = 0; + + lockdep_assert_held(&vm->lock); + + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + if (i == *num_vmas) + return -ENOSPC; + + attrs[i].start = xe_vma_start(vma); + attrs[i].end = xe_vma_end(vma); + attrs[i].atomic.val = vma->attr.atomic_access; + attrs[i].pat_index.val = vma->attr.pat_index; + attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd; + attrs[i].preferred_mem_loc.migration_policy = + vma->attr.preferred_loc.migration_policy; + + i++; + } + + *num_vmas = i; + return 0; +} + +int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_mem_range_attr *mem_attrs; + struct drm_xe_vm_query_mem_range_attr *args = data; + u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); + struct xe_vm *vm; + int err = 0; + + if (XE_IOCTL_DBG(xe, + ((args->num_mem_ranges == 0 && + (attrs_user || args->sizeof_mem_range_attr != 0)) || + (args->num_mem_ranges > 0 && + (!attrs_user || + args->sizeof_mem_range_attr != + sizeof(struct drm_xe_mem_range_attr)))))) + return -EINVAL; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -EINVAL; + + err = down_read_interruptible(&vm->lock); + if (err) + goto put_vm; + + attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); + + if (args->num_mem_ranges == 0 && !attrs_user) { + args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range); + args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr); + goto unlock_vm; + } + + mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr, + GFP_KERNEL | __GFP_ACCOUNT | + __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + if (!mem_attrs) { + err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM; + goto unlock_vm; + } + + memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr); + err = get_mem_attrs(vm, &args->num_mem_ranges, args->start, + args->start + args->range, mem_attrs); + if (err) + goto free_mem_attrs; + + err = copy_to_user(attrs_user, mem_attrs, + args->sizeof_mem_range_attr * args->num_mem_ranges); + if (err) + err = -EFAULT; + +free_mem_attrs: + kvfree(mem_attrs); +unlock_vm: + up_read(&vm->lock); +put_vm: + xe_vm_put(vm); + return err; +} + +static bool vma_matches(struct xe_vma *vma, u64 page_addr) +{ + if (page_addr > xe_vma_end(vma) - 1 || + page_addr + SZ_4K - 1 < xe_vma_start(vma)) + return false; + + return true; +} + +/** + * xe_vm_find_vma_by_addr() - Find a VMA by its address + * + * @vm: the xe_vm the vma belongs to + * @page_addr: address to look up + */ +struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr) +{ + struct xe_vma *vma = NULL; + + if (vm->usm.last_fault_vma) { /* Fast lookup */ + if (vma_matches(vm->usm.last_fault_vma, page_addr)) + vma = vm->usm.last_fault_vma; + } + if (!vma) + vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K); + + return vma; +} + +static const u32 region_to_mem_type[] = { + XE_PL_TT, + XE_PL_VRAM0, + XE_PL_VRAM1, +}; + +static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma, + bool post_commit) +{ + xe_svm_notifier_lock(vm); + vma->gpuva.flags |= XE_VMA_DESTROYED; + xe_svm_notifier_unlock(vm); + if (post_commit) + xe_vm_remove_vma(vm, vma); +} + +#undef ULL +#define ULL unsigned long long + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) +static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) +{ + struct xe_vma *vma; + + switch (op->op) { + case DRM_GPUVA_OP_MAP: + vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", + (ULL)op->map.va.addr, (ULL)op->map.va.range); + break; + case DRM_GPUVA_OP_REMAP: + vma = gpuva_to_vma(op->remap.unmap->va); + vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", + (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), + op->remap.unmap->keep ? 1 : 0); + if (op->remap.prev) + vm_dbg(&xe->drm, + "REMAP:PREV: addr=0x%016llx, range=0x%016llx", + (ULL)op->remap.prev->va.addr, + (ULL)op->remap.prev->va.range); + if (op->remap.next) + vm_dbg(&xe->drm, + "REMAP:NEXT: addr=0x%016llx, range=0x%016llx", + (ULL)op->remap.next->va.addr, + (ULL)op->remap.next->va.range); + break; + case DRM_GPUVA_OP_UNMAP: + vma = gpuva_to_vma(op->unmap.va); + vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", + (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma), + op->unmap.keep ? 1 : 0); + break; + case DRM_GPUVA_OP_PREFETCH: + vma = gpuva_to_vma(op->prefetch.va); + vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", + (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma)); + break; + default: + drm_warn(&xe->drm, "NOT POSSIBLE\n"); + } +} +#else +static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) +{ +} +#endif + +static bool __xe_vm_needs_clear_scratch_pages(struct xe_vm *vm, u32 bind_flags) +{ + if (!xe_vm_in_fault_mode(vm)) + return false; + + if (!xe_vm_has_scratch(vm)) + return false; + + if (bind_flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE) + return false; + + return true; +} + +static void xe_svm_prefetch_gpuva_ops_fini(struct drm_gpuva_ops *ops) +{ + struct drm_gpuva_op *__op; + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + + xe_vma_svm_prefetch_op_fini(op); + } +} + +/* + * Create operations list from IOCTL arguments, setup operations fields so parse + * and commit steps are decoupled from IOCTL arguments. This step can fail. + */ +static struct drm_gpuva_ops * +vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, + struct xe_bo *bo, u64 bo_offset_or_userptr, + u64 addr, u64 range, + u32 operation, u32 flags, + u32 prefetch_region, u16 pat_index) +{ + struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; + struct drm_gpuva_ops *ops; + struct drm_gpuva_op *__op; + struct drm_gpuvm_bo *vm_bo; + u64 range_start = addr; + u64 range_end = addr + range; + int err; + + lockdep_assert_held_write(&vm->lock); + + vm_dbg(&vm->xe->drm, + "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx", + operation, (ULL)addr, (ULL)range, + (ULL)bo_offset_or_userptr); + + switch (operation) { + case DRM_XE_VM_BIND_OP_MAP: + if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) { + xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end); + vops->flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP; + } + + fallthrough; + case DRM_XE_VM_BIND_OP_MAP_USERPTR: { + struct drm_gpuvm_map_req map_req = { + .map.va.addr = range_start, + .map.va.range = range_end - range_start, + .map.gem.obj = obj, + .map.gem.offset = bo_offset_or_userptr, + }; + + ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req); + break; + } + case DRM_XE_VM_BIND_OP_UNMAP: + ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); + break; + case DRM_XE_VM_BIND_OP_PREFETCH: + ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); + break; + case DRM_XE_VM_BIND_OP_UNMAP_ALL: + xe_assert(vm->xe, bo); + + err = xe_bo_lock(bo, true); + if (err) + return ERR_PTR(err); + + vm_bo = drm_gpuvm_bo_obtain_locked(&vm->gpuvm, obj); + if (IS_ERR(vm_bo)) { + xe_bo_unlock(bo); + return ERR_CAST(vm_bo); + } + + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + drm_gpuvm_bo_put(vm_bo); + xe_bo_unlock(bo); + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); + ops = ERR_PTR(-EINVAL); + } + if (IS_ERR(ops)) + return ops; + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + + if (__op->op == DRM_GPUVA_OP_MAP) { + op->map.immediate = + flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE; + if (flags & DRM_XE_VM_BIND_FLAG_READONLY) + op->map.vma_flags |= XE_VMA_READ_ONLY; + if (flags & DRM_XE_VM_BIND_FLAG_NULL) + op->map.vma_flags |= DRM_GPUVA_SPARSE; + if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) + op->map.vma_flags |= XE_VMA_SYSTEM_ALLOCATOR; + if (flags & DRM_XE_VM_BIND_FLAG_DUMPABLE) + op->map.vma_flags |= XE_VMA_DUMPABLE; + if (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) + op->map.vma_flags |= XE_VMA_MADV_AUTORESET; + op->map.request_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS; + op->map.pat_index = pat_index; + op->map.invalidate_on_bind = + __xe_vm_needs_clear_scratch_pages(vm, flags); + } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + struct xe_tile *tile; + struct xe_svm_range *svm_range; + struct drm_gpusvm_ctx ctx = {}; + struct drm_pagemap *dpagemap = NULL; + u8 id, tile_mask = 0; + u32 i; + + if (!xe_vma_is_cpu_addr_mirror(vma)) { + op->prefetch.region = prefetch_region; + break; + } + + ctx.read_only = xe_vma_read_only(vma); + ctx.devmem_possible = IS_DGFX(vm->xe) && + IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); + + for_each_tile(tile, vm->xe, id) + tile_mask |= 0x1 << id; + + xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC); + op->prefetch_range.ranges_count = 0; + + if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { + dpagemap = xe_vma_resolve_pagemap(vma, + xe_device_get_root_tile(vm->xe)); + } else if (prefetch_region) { + tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - + XE_PL_VRAM0]; + dpagemap = xe_tile_local_pagemap(tile); + } + + op->prefetch_range.dpagemap = dpagemap; +alloc_next_range: + svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx); + + if (PTR_ERR(svm_range) == -ENOENT) { + u64 ret = xe_svm_find_vma_start(vm, addr, range_end, vma); + + addr = ret == ULONG_MAX ? 0 : ret; + if (addr) + goto alloc_next_range; + else + goto print_op_label; + } + + if (IS_ERR(svm_range)) { + err = PTR_ERR(svm_range); + goto unwind_prefetch_ops; + } + + if (xe_svm_range_validate(vm, svm_range, tile_mask, dpagemap)) { + xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); + goto check_next_range; + } + + err = xa_alloc(&op->prefetch_range.range, + &i, svm_range, xa_limit_32b, + GFP_KERNEL); + + if (err) + goto unwind_prefetch_ops; + + op->prefetch_range.ranges_count++; + vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH; + xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED"); +check_next_range: + if (range_end > xe_svm_range_end(svm_range) && + xe_svm_range_end(svm_range) < xe_vma_end(vma)) { + addr = xe_svm_range_end(svm_range); + goto alloc_next_range; + } + } +print_op_label: + print_op(vm->xe, __op); + } + + return ops; + +unwind_prefetch_ops: + xe_svm_prefetch_gpuva_ops_fini(ops); + drm_gpuva_ops_free(&vm->gpuvm, ops); + return ERR_PTR(err); +} + +ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_create, ERRNO); + +static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, + struct xe_vma_mem_attr *attr, unsigned int flags) +{ + struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct xe_vma *vma; + int err = 0; + + lockdep_assert_held_write(&vm->lock); + + if (bo) { + err = 0; + xe_validation_guard(&ctx, &vm->xe->val, &exec, + (struct xe_val_flags) {.interruptible = true}, err) { + if (!bo->vm) { + err = drm_exec_lock_obj(&exec, xe_vm_obj(vm)); + drm_exec_retry_on_contention(&exec); + } + if (!err) { + err = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + } + if (err) + return ERR_PTR(err); + + vma = xe_vma_create(vm, bo, op->gem.offset, + op->va.addr, op->va.addr + + op->va.range - 1, attr, flags); + if (IS_ERR(vma)) + return vma; + + if (!bo->vm) { + err = add_preempt_fences(vm, bo); + if (err) { + prep_vma_destroy(vm, vma, false); + xe_vma_destroy(vma, NULL); + } + } + } + if (err) + return ERR_PTR(err); + } else { + vma = xe_vma_create(vm, NULL, op->gem.offset, + op->va.addr, op->va.addr + + op->va.range - 1, attr, flags); + if (IS_ERR(vma)) + return vma; + + if (xe_vma_is_userptr(vma)) { + err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); + /* + * -EBUSY has dedicated meaning that a user fence + * attached to the VMA is busy, in practice + * xe_vma_userptr_pin_pages can only fail with -EBUSY if + * we are low on memory so convert this to -ENOMEM. + */ + if (err == -EBUSY) + err = -ENOMEM; + } + } + if (err) { + prep_vma_destroy(vm, vma, false); + xe_vma_destroy_unlocked(vma); + vma = ERR_PTR(err); + } + + return vma; +} + +static u64 xe_vma_max_pte_size(struct xe_vma *vma) +{ + if (vma->gpuva.flags & XE_VMA_PTE_1G) + return SZ_1G; + else if (vma->gpuva.flags & (XE_VMA_PTE_2M | XE_VMA_PTE_COMPACT)) + return SZ_2M; + else if (vma->gpuva.flags & XE_VMA_PTE_64K) + return SZ_64K; + else if (vma->gpuva.flags & XE_VMA_PTE_4K) + return SZ_4K; + + return SZ_1G; /* Uninitialized, used max size */ +} + +static void xe_vma_set_pte_size(struct xe_vma *vma, u64 size) +{ + switch (size) { + case SZ_1G: + vma->gpuva.flags |= XE_VMA_PTE_1G; + break; + case SZ_2M: + vma->gpuva.flags |= XE_VMA_PTE_2M; + break; + case SZ_64K: + vma->gpuva.flags |= XE_VMA_PTE_64K; + break; + case SZ_4K: + vma->gpuva.flags |= XE_VMA_PTE_4K; + break; + } +} + +static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) +{ + int err = 0; + + lockdep_assert_held_write(&vm->lock); + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + err |= xe_vm_insert_vma(vm, op->map.vma); + if (!err) + op->flags |= XE_VMA_OP_COMMITTED; + break; + case DRM_GPUVA_OP_REMAP: + { + u8 tile_present = + gpuva_to_vma(op->base.remap.unmap->va)->tile_present; + + prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va), + true); + op->flags |= XE_VMA_OP_COMMITTED; + + if (op->remap.prev) { + err |= xe_vm_insert_vma(vm, op->remap.prev); + if (!err) + op->flags |= XE_VMA_OP_PREV_COMMITTED; + if (!err && op->remap.skip_prev) { + op->remap.prev->tile_present = + tile_present; + } + } + if (op->remap.next) { + err |= xe_vm_insert_vma(vm, op->remap.next); + if (!err) + op->flags |= XE_VMA_OP_NEXT_COMMITTED; + if (!err && op->remap.skip_next) { + op->remap.next->tile_present = + tile_present; + } + } + + /* + * Adjust for partial unbind after removing VMA from VM. In case + * of unwind we might need to undo this later. + */ + if (!err) { + op->base.remap.unmap->va->va.addr = op->remap.start; + op->base.remap.unmap->va->va.range = op->remap.range; + } + break; + } + case DRM_GPUVA_OP_UNMAP: + prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true); + op->flags |= XE_VMA_OP_COMMITTED; + break; + case DRM_GPUVA_OP_PREFETCH: + op->flags |= XE_VMA_OP_COMMITTED; + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); + } + + return err; +} + +/** + * xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes + * @vma: Pointer to the xe_vma structure to check + * + * This function determines whether the given VMA (Virtual Memory Area) + * has its memory attributes set to their default values. Specifically, + * it checks the following conditions: + * + * - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED` + * - `pat_index` is equal to `default_pat_index` + * - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE` + * - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES` + * + * Return: true if all attributes are at their default values, false otherwise. + */ +bool xe_vma_has_default_mem_attrs(struct xe_vma *vma) +{ + return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED && + vma->attr.pat_index == vma->attr.default_pat_index && + vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && + vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES); +} + +static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, + struct xe_vma_ops *vops) +{ + struct xe_device *xe = vm->xe; + struct drm_gpuva_op *__op; + struct xe_tile *tile; + u8 id, tile_mask = 0; + int err = 0; + + lockdep_assert_held_write(&vm->lock); + + for_each_tile(tile, vm->xe, id) + tile_mask |= 0x1 << id; + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + struct xe_vma *vma; + unsigned int flags = 0; + + INIT_LIST_HEAD(&op->link); + list_add_tail(&op->link, &vops->list); + op->tile_mask = tile_mask; + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + { + struct xe_vma_mem_attr default_attr = { + .preferred_loc = { + .devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, + .migration_policy = DRM_XE_MIGRATE_ALL_PAGES, + }, + .atomic_access = DRM_XE_ATOMIC_UNDEFINED, + .default_pat_index = op->map.pat_index, + .pat_index = op->map.pat_index, + .purgeable_state = XE_MADV_PURGEABLE_WILLNEED, + }; + + flags |= op->map.vma_flags & XE_VMA_CREATE_MASK; + + vma = new_vma(vm, &op->base.map, &default_attr, + flags); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + op->map.vma = vma; + if (((op->map.immediate || !xe_vm_in_fault_mode(vm)) && + !(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) || + op->map.invalidate_on_bind) + xe_vma_ops_incr_pt_update_ops(vops, + op->tile_mask, 1); + break; + } + case DRM_GPUVA_OP_REMAP: + { + struct xe_vma *old = + gpuva_to_vma(op->base.remap.unmap->va); + bool skip = xe_vma_is_cpu_addr_mirror(old); + u64 start = xe_vma_start(old), end = xe_vma_end(old); + int num_remap_ops = 0; + + if (op->base.remap.prev) + start = op->base.remap.prev->va.addr + + op->base.remap.prev->va.range; + if (op->base.remap.next) + end = op->base.remap.next->va.addr; + + if (xe_vma_is_cpu_addr_mirror(old) && + xe_svm_has_mapping(vm, start, end)) { + if (vops->flags & XE_VMA_OPS_FLAG_MADVISE) + xe_svm_unmap_address_range(vm, start, end); + else + return -EBUSY; + } + + op->remap.start = xe_vma_start(old); + op->remap.range = xe_vma_size(old); + op->remap.old_start = op->remap.start; + op->remap.old_range = op->remap.range; + + flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; + if (op->base.remap.prev) { + vma = new_vma(vm, op->base.remap.prev, + &old->attr, flags); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + op->remap.prev = vma; + + /* + * Userptr creates a new SG mapping so + * we must also rebind. + */ + op->remap.skip_prev = skip || + (!xe_vma_is_userptr(old) && + IS_ALIGNED(xe_vma_end(vma), + xe_vma_max_pte_size(old))); + if (op->remap.skip_prev) { + xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); + op->remap.range -= + xe_vma_end(vma) - + xe_vma_start(old); + op->remap.start = xe_vma_end(vma); + vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx", + (ULL)op->remap.start, + (ULL)op->remap.range); + } else { + num_remap_ops++; + } + } + + if (op->base.remap.next) { + vma = new_vma(vm, op->base.remap.next, + &old->attr, flags); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + op->remap.next = vma; + + /* + * Userptr creates a new SG mapping so + * we must also rebind. + */ + op->remap.skip_next = skip || + (!xe_vma_is_userptr(old) && + IS_ALIGNED(xe_vma_start(vma), + xe_vma_max_pte_size(old))); + if (op->remap.skip_next) { + xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); + op->remap.range -= + xe_vma_end(old) - + xe_vma_start(vma); + vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx", + (ULL)op->remap.start, + (ULL)op->remap.range); + } else { + num_remap_ops++; + } + } + if (!skip) + num_remap_ops++; + + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, num_remap_ops); + break; + } + case DRM_GPUVA_OP_UNMAP: + vma = gpuva_to_vma(op->base.unmap.va); + + if (xe_vma_is_cpu_addr_mirror(vma) && + xe_svm_has_mapping(vm, xe_vma_start(vma), + xe_vma_end(vma)) && + !(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP)) + return -EBUSY; + + if (!xe_vma_is_cpu_addr_mirror(vma)) + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); + break; + case DRM_GPUVA_OP_PREFETCH: + vma = gpuva_to_vma(op->base.prefetch.va); + + if (xe_vma_is_userptr(vma)) { + err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); + if (err) + return err; + } + + if (xe_vma_is_cpu_addr_mirror(vma)) + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, + op->prefetch_range.ranges_count); + else + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); + + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); + } + + err = xe_vma_op_commit(vm, op); + if (err) + return err; + } + + return 0; +} + +static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, + bool post_commit, bool prev_post_commit, + bool next_post_commit) +{ + lockdep_assert_held_write(&vm->lock); + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if (op->map.vma) { + prep_vma_destroy(vm, op->map.vma, post_commit); + xe_vma_destroy_unlocked(op->map.vma); + } + break; + case DRM_GPUVA_OP_UNMAP: + { + struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); + + if (vma) { + xe_svm_notifier_lock(vm); + vma->gpuva.flags &= ~XE_VMA_DESTROYED; + xe_svm_notifier_unlock(vm); + if (post_commit) + xe_vm_insert_vma(vm, vma); + } + break; + } + case DRM_GPUVA_OP_REMAP: + { + struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va); + + if (op->remap.prev) { + prep_vma_destroy(vm, op->remap.prev, prev_post_commit); + xe_vma_destroy_unlocked(op->remap.prev); + } + if (op->remap.next) { + prep_vma_destroy(vm, op->remap.next, next_post_commit); + xe_vma_destroy_unlocked(op->remap.next); + } + if (vma) { + xe_svm_notifier_lock(vm); + vma->gpuva.flags &= ~XE_VMA_DESTROYED; + xe_svm_notifier_unlock(vm); + if (post_commit) { + /* + * Restore the old va range, in case of the + * prev/next skip optimisation. Otherwise what + * we re-insert here could be smaller than the + * original range. + */ + op->base.remap.unmap->va->va.addr = + op->remap.old_start; + op->base.remap.unmap->va->va.range = + op->remap.old_range; + xe_vm_insert_vma(vm, vma); + } + } + break; + } + case DRM_GPUVA_OP_PREFETCH: + /* Nothing to do */ + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); + } +} + +static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, + struct drm_gpuva_ops **ops, + int num_ops_list) +{ + int i; + + for (i = num_ops_list - 1; i >= 0; --i) { + struct drm_gpuva_ops *__ops = ops[i]; + struct drm_gpuva_op *__op; + + if (!__ops) + continue; + + drm_gpuva_for_each_op_reverse(__op, __ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + + xe_vma_op_unwind(vm, op, + op->flags & XE_VMA_OP_COMMITTED, + op->flags & XE_VMA_OP_PREV_COMMITTED, + op->flags & XE_VMA_OP_NEXT_COMMITTED); + } + } +} + +/** + * struct xe_vma_lock_and_validate_flags - Flags for vma_lock_and_validate() + * @res_evict: Allow evicting resources during validation + * @validate: Perform BO validation + * @request_decompress: Request BO decompression + * @check_purged: Reject operation if BO is DONTNEED or PURGED + */ +struct xe_vma_lock_and_validate_flags { + u32 res_evict : 1; + u32 validate : 1; + u32 request_decompress : 1; + u32 check_purged : 1; +}; + +static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, + struct xe_vma_lock_and_validate_flags flags) +{ + struct xe_bo *bo = xe_vma_bo(vma); + struct xe_vm *vm = xe_vma_vm(vma); + bool validate_bo = flags.validate; + int err = 0; + + if (bo) { + if (!bo->vm) + err = drm_exec_lock_obj(exec, &bo->ttm.base); + + /* Reject new mappings to DONTNEED/purged BOs; allow cleanup operations */ + if (!err && flags.check_purged) { + if (xe_bo_madv_is_dontneed(bo)) + err = -EBUSY; /* BO marked purgeable */ + else if (xe_bo_is_purged(bo)) + err = -EINVAL; /* BO already purged */ + } + + /* Don't validate the BO for DONTNEED/PURGED remap remnants. */ + if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED) + validate_bo = false; + + if (!err && validate_bo) + err = xe_bo_validate(bo, vm, + xe_vm_allow_vm_eviction(vm) && + flags.res_evict, exec); + + if (err) + return err; + + if (flags.request_decompress) + err = xe_bo_decompress(bo); + } + + return err; +} + +static int check_ufence(struct xe_vma *vma) +{ + if (vma->ufence) { + struct xe_user_fence * const f = vma->ufence; + + if (!xe_sync_ufence_get_status(f)) + return -EBUSY; + + vma->ufence = NULL; + xe_sync_ufence_put(f); + } + + return 0; +} + +static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op) +{ + bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + struct drm_pagemap *dpagemap = op->prefetch_range.dpagemap; + int err = 0; + + struct xe_svm_range *svm_range; + struct drm_gpusvm_ctx ctx = {}; + unsigned long i; + + if (!xe_vma_is_cpu_addr_mirror(vma)) + return 0; + + ctx.read_only = xe_vma_read_only(vma); + ctx.devmem_possible = devmem_possible; + ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0; + ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); + + /* TODO: Threading the migration */ + xa_for_each(&op->prefetch_range.range, i, svm_range) { + if (!dpagemap) + xe_svm_range_migrate_to_smem(vm, svm_range); + + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) { + drm_dbg(&vm->xe->drm, + "Prefetch pagemap is %s start 0x%016lx end 0x%016lx\n", + dpagemap ? dpagemap->drm->unique : "system", + xe_svm_range_start(svm_range), xe_svm_range_end(svm_range)); + } + + if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, dpagemap)) { + err = xe_svm_alloc_vram(svm_range, &ctx, dpagemap); + if (err) { + drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n", + vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); + return -ENODATA; + } + xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM"); + } + + err = xe_svm_range_get_pages(vm, svm_range, &ctx); + if (err) { + drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n", + vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); + if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) + err = -ENODATA; + return err; + } + xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE"); + } + + return err; +} + +static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, + struct xe_vma_ops *vops, struct xe_vma_op *op) +{ + int err = 0; + bool res_evict; + + /* + * We only allow evicting a BO within the VM if it is not part of an + * array of binds, as an array of binds can evict another BO within the + * bind. + */ + res_evict = !(vops->flags & XE_VMA_OPS_ARRAY_OF_BINDS); + + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if (!op->map.invalidate_on_bind) + err = vma_lock_and_validate(exec, op->map.vma, + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = !xe_vm_in_fault_mode(vm) || + op->map.immediate, + .request_decompress = + op->map.request_decompress, + .check_purged = false, + }); + break; + case DRM_GPUVA_OP_REMAP: + err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va)); + if (err) + break; + + err = vma_lock_and_validate(exec, + gpuva_to_vma(op->base.remap.unmap->va), + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = false, + .request_decompress = false, + .check_purged = false, + }); + if (!err && op->remap.prev) + err = vma_lock_and_validate(exec, op->remap.prev, + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = true, + .request_decompress = false, + .check_purged = false, + }); + if (!err && op->remap.next) + err = vma_lock_and_validate(exec, op->remap.next, + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = true, + .request_decompress = false, + .check_purged = false, + }); + break; + case DRM_GPUVA_OP_UNMAP: + err = check_ufence(gpuva_to_vma(op->base.unmap.va)); + if (err) + break; + + err = vma_lock_and_validate(exec, + gpuva_to_vma(op->base.unmap.va), + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = false, + .request_decompress = false, + .check_purged = false, + }); + break; + case DRM_GPUVA_OP_PREFETCH: + { + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + u32 region; + + if (!xe_vma_is_cpu_addr_mirror(vma)) { + region = op->prefetch.region; + xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC || + region <= ARRAY_SIZE(region_to_mem_type)); + } + + /* + * PREFETCH is the only op that still gates on BO purge state. + * MAP/REMAP handle this inside xe_vma_create() so partial + * unbind on a DONTNEED BO still works. PREFETCH skips + * xe_vma_create() and would migrate a BO with no backing + * store, so reject DONTNEED/PURGED here. + */ + err = vma_lock_and_validate(exec, + gpuva_to_vma(op->base.prefetch.va), + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = false, + .request_decompress = false, + .check_purged = true, + }); + if (!err && !xe_vma_has_no_bo(vma)) { + struct xe_bo *bo = xe_vma_bo(vma); + u32 mem_type; + + if (region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { + unsigned int i; + + mem_type = XE_PL_TT; + for (i = 0; i < bo->placement.num_placement; i++) { + if (mem_type_is_vram(bo->placements[i].mem_type)) { + mem_type = bo->placements[i].mem_type; + break; + } + } + } else { + mem_type = region_to_mem_type[region]; + } + + err = xe_bo_migrate(bo, mem_type, NULL, exec); + } + break; + } + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); + } + + return err; +} + +static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops) +{ + struct xe_vma_op *op; + int err; + + if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) + return 0; + + list_for_each_entry(op, &vops->list, link) { + if (op->base.op == DRM_GPUVA_OP_PREFETCH) { + err = prefetch_ranges(vm, op); + if (err) + return err; + } + } + + return 0; +} + +static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec, + struct xe_vm *vm, + struct xe_vma_ops *vops) +{ + struct xe_vma_op *op; + int err; + + err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); + if (err) + return err; + + list_for_each_entry(op, &vops->list, link) { + err = op_lock_and_prep(exec, vm, vops, op); + if (err) + return err; + } + +#ifdef TEST_VM_OPS_ERROR + if (vops->inject_error && + vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK) + return -ENOSPC; +#endif + + return 0; +} + +static void op_trace(struct xe_vma_op *op) +{ + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + trace_xe_vma_bind(op->map.vma); + break; + case DRM_GPUVA_OP_REMAP: + trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va)); + if (op->remap.prev) + trace_xe_vma_bind(op->remap.prev); + if (op->remap.next) + trace_xe_vma_bind(op->remap.next); + break; + case DRM_GPUVA_OP_UNMAP: + trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va)); + break; + case DRM_GPUVA_OP_PREFETCH: + trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va)); + break; + case DRM_GPUVA_OP_DRIVER: + break; + default: + XE_WARN_ON("NOT POSSIBLE"); + } +} + +static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops) +{ + struct xe_vma_op *op; + + list_for_each_entry(op, &vops->list, link) + op_trace(op); +} + +static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops) +{ + struct xe_exec_queue *q = vops->q; + struct xe_tile *tile; + int number_tiles = 0; + u8 id; + + for_each_tile(tile, vm->xe, id) { + if (vops->pt_update_ops[id].num_ops) + ++number_tiles; + + if (vops->pt_update_ops[id].q) + continue; + + if (q) { + vops->pt_update_ops[id].q = q; + if (vm->pt_root[id] && !list_empty(&q->multi_gt_list)) + q = list_next_entry(q, multi_gt_list); + } else { + vops->pt_update_ops[id].q = vm->q[id]; + } + } + + return number_tiles; +} + +static struct dma_fence *ops_execute(struct xe_vm *vm, + struct xe_vma_ops *vops) +{ + struct xe_tile *tile; + struct dma_fence *fence = NULL; + struct dma_fence **fences = NULL; + struct dma_fence_array *cf = NULL; + int number_tiles = 0, current_fence = 0, n_fence = 0, err, i; + u8 id; + + number_tiles = vm_ops_setup_tile_args(vm, vops); + if (number_tiles == 0) + return ERR_PTR(-ENODATA); + + for_each_tile(tile, vm->xe, id) { + ++n_fence; + + if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) + for_each_tlb_inval(i) + ++n_fence; + } + + fences = kmalloc_objs(*fences, n_fence); + if (!fences) { + fence = ERR_PTR(-ENOMEM); + goto err_trace; + } + + cf = dma_fence_array_alloc(n_fence); + if (!cf) { + fence = ERR_PTR(-ENOMEM); + goto err_out; + } + + for_each_tile(tile, vm->xe, id) { + if (!vops->pt_update_ops[id].num_ops) + continue; + + err = xe_pt_update_ops_prepare(tile, vops); + if (err) { + fence = ERR_PTR(err); + goto err_out; + } + } + + trace_xe_vm_ops_execute(vops); + + for_each_tile(tile, vm->xe, id) { + struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q; + + fence = NULL; + if (!vops->pt_update_ops[id].num_ops) + goto collect_fences; + + fence = xe_pt_update_ops_run(tile, vops); + if (IS_ERR(fence)) + goto err_out; + +collect_fences: + fences[current_fence++] = fence ?: dma_fence_get_stub(); + if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) + continue; + + xe_migrate_job_lock(tile->migrate, q); + for_each_tlb_inval(i) + fences[current_fence++] = + xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); + xe_migrate_job_unlock(tile->migrate, q); + } + + xe_assert(vm->xe, current_fence == n_fence); + dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1), + 1); + fence = &cf->base; + + for_each_tile(tile, vm->xe, id) { + if (!vops->pt_update_ops[id].num_ops) + continue; + + xe_pt_update_ops_fini(tile, vops); + } + + return fence; + +err_out: + for_each_tile(tile, vm->xe, id) { + if (!vops->pt_update_ops[id].num_ops) + continue; + + xe_pt_update_ops_abort(tile, vops); + } + while (current_fence) + dma_fence_put(fences[--current_fence]); + kfree(fences); + kfree(cf); + +err_trace: + trace_xe_vm_ops_fail(vm); + return fence; +} + +static void vma_add_ufence(struct xe_vma *vma, struct xe_user_fence *ufence) +{ + if (vma->ufence) + xe_sync_ufence_put(vma->ufence); + vma->ufence = __xe_sync_ufence_get(ufence); +} + +static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, + struct xe_user_fence *ufence) +{ + switch (op->base.op) { + case DRM_GPUVA_OP_MAP: + if (!xe_vma_is_cpu_addr_mirror(op->map.vma)) + vma_add_ufence(op->map.vma, ufence); + break; + case DRM_GPUVA_OP_REMAP: + if (op->remap.prev) + vma_add_ufence(op->remap.prev, ufence); + if (op->remap.next) + vma_add_ufence(op->remap.next, ufence); + break; + case DRM_GPUVA_OP_UNMAP: + break; + case DRM_GPUVA_OP_PREFETCH: + vma_add_ufence(gpuva_to_vma(op->base.prefetch.va), ufence); + break; + default: + drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); + } +} + +static void vm_bind_ioctl_ops_fini(struct xe_vm *vm, struct xe_vma_ops *vops, + struct dma_fence *fence) +{ + struct xe_user_fence *ufence; + struct xe_vma_op *op; + int i; + + ufence = find_ufence_get(vops->syncs, vops->num_syncs); + list_for_each_entry(op, &vops->list, link) { + if (ufence) + op_add_ufence(vm, op, ufence); + + if (op->base.op == DRM_GPUVA_OP_UNMAP) + xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence); + else if (op->base.op == DRM_GPUVA_OP_REMAP) + xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), + fence); + } + if (ufence) + xe_sync_ufence_put(ufence); + if (fence) { + for (i = 0; i < vops->num_syncs; i++) + xe_sync_entry_signal(vops->syncs + i, fence); + } +} + +static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm, + struct xe_vma_ops *vops) +{ + struct xe_validation_ctx ctx; + struct drm_exec exec; + struct dma_fence *fence; + int err = 0; + + lockdep_assert_held_write(&vm->lock); + + xe_validation_guard(&ctx, &vm->xe->val, &exec, + ((struct xe_val_flags) { + .interruptible = true, + .exec_ignore_duplicates = true, + }), err) { + err = vm_bind_ioctl_ops_lock_and_prep(&exec, vm, vops); + drm_exec_retry_on_contention(&exec); + xe_validation_retry_on_oom(&ctx, &err); + if (err) + return ERR_PTR(err); + + xe_vm_set_validation_exec(vm, &exec); + fence = ops_execute(vm, vops); + xe_vm_set_validation_exec(vm, NULL); + if (IS_ERR(fence)) { + if (PTR_ERR(fence) == -ENODATA) + vm_bind_ioctl_ops_fini(vm, vops, NULL); + return fence; + } + + vm_bind_ioctl_ops_fini(vm, vops, fence); + } + + return err ? ERR_PTR(err) : fence; +} +ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO); + +#define SUPPORTED_FLAGS_STUB \ + (DRM_XE_VM_BIND_FLAG_READONLY | \ + DRM_XE_VM_BIND_FLAG_IMMEDIATE | \ + DRM_XE_VM_BIND_FLAG_NULL | \ + DRM_XE_VM_BIND_FLAG_DUMPABLE | \ + DRM_XE_VM_BIND_FLAG_CHECK_PXP | \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR | \ + DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET | \ + DRM_XE_VM_BIND_FLAG_DECOMPRESS) + +#ifdef TEST_VM_OPS_ERROR +#define SUPPORTED_FLAGS (SUPPORTED_FLAGS_STUB | FORCE_OP_ERROR) +#else +#define SUPPORTED_FLAGS SUPPORTED_FLAGS_STUB +#endif + +#define XE_64K_PAGE_MASK 0xffffull +#define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP) + +static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, + struct drm_xe_vm_bind *args, + struct drm_xe_vm_bind_op **bind_ops) +{ + int err; + int i; + + if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->extensions)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) + return -EINVAL; + + if (args->num_binds > 1) { + u64 __user *bind_user = + u64_to_user_ptr(args->vector_of_binds); + + *bind_ops = kvmalloc_objs(struct drm_xe_vm_bind_op, + args->num_binds, + GFP_KERNEL | __GFP_ACCOUNT | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + if (!*bind_ops) + return args->num_binds > 1 ? -ENOBUFS : -ENOMEM; + + err = copy_from_user(*bind_ops, bind_user, + sizeof(struct drm_xe_vm_bind_op) * + args->num_binds); + if (XE_IOCTL_DBG(xe, err)) { + err = -EFAULT; + goto free_bind_ops; + } + } else { + *bind_ops = &args->bind; + } + + for (i = 0; i < args->num_binds; ++i) { + u64 range = (*bind_ops)[i].range; + u64 addr = (*bind_ops)[i].addr; + u32 op = (*bind_ops)[i].op; + u32 flags = (*bind_ops)[i].flags; + u32 obj = (*bind_ops)[i].obj; + u64 obj_offset = (*bind_ops)[i].obj_offset; + u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; + bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; + bool is_cpu_addr_mirror = flags & + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR; + bool is_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS; + u16 pat_index = (*bind_ops)[i].pat_index; + u16 coh_mode; + bool comp_en; + + if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror && + (!xe_vm_in_fault_mode(vm) || + !IS_ENABLED(CONFIG_DRM_XE_GPUSVM)))) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { + err = -EINVAL; + goto free_bind_ops; + } + + pat_index = array_index_nospec(pat_index, xe->pat.n_entries); + (*bind_ops)[i].pat_index = pat_index; + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + comp_en = xe_pat_index_get_comp_en(xe, pat_index); + if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */ + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_WARN_ON(coh_mode > XE_COH_2WAY)) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || + XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || + XE_IOCTL_DBG(xe, obj && (is_null || is_cpu_addr_mirror)) || + XE_IOCTL_DBG(xe, obj_offset && (is_null || + is_cpu_addr_mirror)) || + XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && + (is_decompress || is_null || is_cpu_addr_mirror)) || + XE_IOCTL_DBG(xe, is_decompress && + xe_pat_index_get_comp_en(xe, pat_index)) || + XE_IOCTL_DBG(xe, !obj && + op == DRM_XE_VM_BIND_OP_MAP && + !is_null && !is_cpu_addr_mirror) || + XE_IOCTL_DBG(xe, !obj && + op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || + XE_IOCTL_DBG(xe, addr && + op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || + XE_IOCTL_DBG(xe, range && + op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || + XE_IOCTL_DBG(xe, obj && + op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && + op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE && + is_cpu_addr_mirror) || + XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) && + (op == DRM_XE_VM_BIND_OP_MAP_USERPTR || + is_cpu_addr_mirror) && + (pat_index != 19 && coh_mode != XE_COH_2WAY)) || + XE_IOCTL_DBG(xe, comp_en && + op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR && + !IS_ENABLED(CONFIG_DRM_GPUSVM)) || + XE_IOCTL_DBG(xe, obj && + op == DRM_XE_VM_BIND_OP_PREFETCH) || + XE_IOCTL_DBG(xe, prefetch_region && + op != DRM_XE_VM_BIND_OP_PREFETCH) || + XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC && + /* Guard against undefined shift in BIT(prefetch_region) */ + (prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) || + !(BIT(prefetch_region) & xe->info.mem_region_mask)))) || + XE_IOCTL_DBG(xe, obj && + op == DRM_XE_VM_BIND_OP_UNMAP) || + XE_IOCTL_DBG(xe, (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) && + (!is_cpu_addr_mirror || op != DRM_XE_VM_BIND_OP_MAP))) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) || + XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || + XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || + XE_IOCTL_DBG(xe, !range && + op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) { + err = -EINVAL; + goto free_bind_ops; + } + + if (is_decompress && (XE_IOCTL_DBG(xe, !xe_device_has_flat_ccs(xe)) || + XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20) || + XE_IOCTL_DBG(xe, !IS_DGFX(xe)))) { + err = -EOPNOTSUPP; + goto free_bind_ops; + } + } + + return 0; + +free_bind_ops: + if (args->num_binds > 1) + kvfree(*bind_ops); + *bind_ops = NULL; + return err; +} + +static int vm_bind_ioctl_signal_fences(struct xe_vm *vm, + struct xe_exec_queue *q, + struct xe_sync_entry *syncs, + int num_syncs) +{ + struct dma_fence *fence = NULL; + int i, err = 0; + + if (num_syncs) { + fence = xe_sync_in_fence_get(syncs, num_syncs, + to_wait_exec_queue(vm, q), vm); + if (IS_ERR(fence)) + return PTR_ERR(fence); + + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], fence); + } + + dma_fence_put(fence); + + return err; +} + +static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, + struct xe_exec_queue *q, + struct xe_sync_entry *syncs, u32 num_syncs) +{ + memset(vops, 0, sizeof(*vops)); + INIT_LIST_HEAD(&vops->list); + vops->vm = vm; + vops->q = q; + vops->syncs = syncs; + vops->num_syncs = num_syncs; + vops->flags = 0; +} + +static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, + u64 addr, u64 range, u64 obj_offset, + u16 pat_index, u32 op, u32 bind_flags) +{ + u16 coh_mode; + bool comp_en; + + if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) && + xe_pat_index_get_comp_en(xe, pat_index))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) || + XE_IOCTL_DBG(xe, obj_offset > + xe_bo_size(bo) - range)) { + return -EINVAL; + } + + /* + * Some platforms require 64k VM_BIND alignment, + * specifically those with XE_VRAM_FLAGS_NEED64K. + * + * Other platforms may have BO's set to 64k physical placement, + * but can be mapped at 4k offsets anyway. This check is only + * there for the former case. + */ + if ((bo->flags & XE_BO_FLAG_INTERNAL_64K) && + (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) { + if (XE_IOCTL_DBG(xe, obj_offset & + XE_64K_PAGE_MASK) || + XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) || + XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) { + return -EINVAL; + } + } + + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + if (bo->cpu_caching) { + if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && + bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { + return -EINVAL; + } + } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) { + /* + * Imported dma-buf from a different device should + * require 1way or 2way coherency since we don't know + * how it was mapped on the CPU. Just assume is it + * potentially cached on CPU side. + */ + return -EINVAL; + } + + /* + * Ensures that imported buffer objects (dma-bufs) are not mapped + * with a PAT index that enables compression. + */ + comp_en = xe_pat_index_get_comp_en(xe, pat_index); + if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && comp_en)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && xe_device_is_l2_flush_optimized(xe) && + (pat_index != 19 && coh_mode != XE_COH_2WAY))) + return -EINVAL; + + /* If a BO is protected it can only be mapped if the key is still valid */ + if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) && + op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL) + if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0)) + return -ENOEXEC; + + return 0; +} + +int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_bind *args = data; + struct drm_xe_sync __user *syncs_user; + struct xe_bo **bos = NULL; + struct drm_gpuva_ops **ops = NULL; + struct xe_vm *vm; + struct xe_exec_queue *q = NULL; + u32 num_syncs, num_ufence = 0; + struct xe_sync_entry *syncs = NULL; + struct drm_xe_vm_bind_op *bind_ops = NULL; + struct xe_vma_ops vops; + struct dma_fence *fence; + int err; + int i; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -EINVAL; + + err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops); + if (err) + goto put_vm; + + if (args->exec_queue_id) { + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) { + err = -ENOENT; + goto free_bind_ops; + } + + if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { + err = -EINVAL; + goto put_exec_queue; + } + } + + if (XE_IOCTL_DBG(xe, q && vm != q->user_vm)) { + err = -EINVAL; + goto put_exec_queue; + } + + /* Ensure all UNMAPs visible */ + xe_svm_flush(vm); + + err = down_write_killable(&vm->lock); + if (err) + goto put_exec_queue; + + if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { + err = -ENOENT; + goto release_vm_lock; + } + + for (i = 0; i < args->num_binds; ++i) { + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + + if (XE_IOCTL_DBG(xe, range > vm->size) || + XE_IOCTL_DBG(xe, addr > vm->size - range)) { + err = -EINVAL; + goto release_vm_lock; + } + } + + if (args->num_binds) { + bos = kvzalloc_objs(*bos, args->num_binds, + GFP_KERNEL | __GFP_ACCOUNT | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + if (!bos) { + err = -ENOMEM; + goto release_vm_lock; + } + + ops = kvzalloc_objs(*ops, args->num_binds, + GFP_KERNEL | __GFP_ACCOUNT | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + if (!ops) { + err = -ENOMEM; + goto free_bos; + } + } + + for (i = 0; i < args->num_binds; ++i) { + struct drm_gem_object *gem_obj; + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + u32 obj = bind_ops[i].obj; + u64 obj_offset = bind_ops[i].obj_offset; + u16 pat_index = bind_ops[i].pat_index; + u32 op = bind_ops[i].op; + u32 bind_flags = bind_ops[i].flags; + + if (!obj) + continue; + + gem_obj = drm_gem_object_lookup(file, obj); + if (XE_IOCTL_DBG(xe, !gem_obj)) { + err = -ENOENT; + goto put_obj; + } + bos[i] = gem_to_xe_bo(gem_obj); + + err = xe_vm_bind_ioctl_validate_bo(xe, bos[i], addr, range, + obj_offset, pat_index, op, + bind_flags); + if (err) + goto put_obj; + } + + if (args->num_syncs) { + syncs = kzalloc_objs(*syncs, args->num_syncs); + if (!syncs) { + err = -ENOMEM; + goto put_obj; + } + } + + syncs_user = u64_to_user_ptr(args->syncs); + for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { + struct xe_exec_queue *__q = q ?: vm->q[0]; + + err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], + &syncs_user[num_syncs], + __q->ufence_syncobj, + ++__q->ufence_timeline_value, + (xe_vm_in_lr_mode(vm) ? + SYNC_PARSE_FLAG_LR_MODE : 0) | + (!args->num_binds ? + SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0)); + if (err) + goto free_syncs; + + if (xe_sync_is_ufence(&syncs[num_syncs])) + num_ufence++; + } + + if (XE_IOCTL_DBG(xe, num_ufence > 1)) { + err = -EINVAL; + goto free_syncs; + } + + if (!args->num_binds) { + err = -ENODATA; + goto free_syncs; + } + + xe_vma_ops_init(&vops, vm, q, syncs, num_syncs); + if (args->num_binds > 1) + vops.flags |= XE_VMA_OPS_ARRAY_OF_BINDS; + for (i = 0; i < args->num_binds; ++i) { + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + u32 op = bind_ops[i].op; + u32 flags = bind_ops[i].flags; + u64 obj_offset = bind_ops[i].obj_offset; + u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; + u16 pat_index = bind_ops[i].pat_index; + + ops[i] = vm_bind_ioctl_ops_create(vm, &vops, bos[i], obj_offset, + addr, range, op, flags, + prefetch_region, pat_index); + if (IS_ERR(ops[i])) { + err = PTR_ERR(ops[i]); + ops[i] = NULL; + goto unwind_ops; + } + + err = vm_bind_ioctl_ops_parse(vm, ops[i], &vops); + if (err) + goto unwind_ops; + +#ifdef TEST_VM_OPS_ERROR + if (flags & FORCE_OP_ERROR) { + vops.inject_error = true; + vm->xe->vm_inject_error_position = + (vm->xe->vm_inject_error_position + 1) % + FORCE_OP_ERROR_COUNT; + } +#endif + } + + /* Nothing to do */ + if (list_empty(&vops.list)) { + err = -ENODATA; + goto unwind_ops; + } + + err = xe_vma_ops_alloc(&vops, args->num_binds > 1); + if (err) + goto unwind_ops; + + err = vm_bind_ioctl_ops_prefetch_ranges(vm, &vops); + if (err) + goto unwind_ops; + + fence = vm_bind_ioctl_ops_execute(vm, &vops); + if (IS_ERR(fence)) + err = PTR_ERR(fence); + else + dma_fence_put(fence); + +unwind_ops: + if (err && err != -ENODATA) + vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds); + xe_vma_ops_fini(&vops); + for (i = args->num_binds - 1; i >= 0; --i) + if (ops[i]) + drm_gpuva_ops_free(&vm->gpuvm, ops[i]); +free_syncs: + if (err == -ENODATA) + err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs); + while (num_syncs--) + xe_sync_entry_cleanup(&syncs[num_syncs]); + + kfree(syncs); +put_obj: + for (i = 0; i < args->num_binds; ++i) + xe_bo_put(bos[i]); + + kvfree(ops); +free_bos: + kvfree(bos); +release_vm_lock: + up_write(&vm->lock); +put_exec_queue: + if (q) + xe_exec_queue_put(q); +free_bind_ops: + if (args->num_binds > 1) + kvfree(bind_ops); +put_vm: + xe_vm_put(vm); + return err; +} + +/* + * Map access type, fault type, and fault level from current bspec + * specification to user spec abstraction. The current mapping is + * approximately 1-to-1, with access type being the only notable + * exception as it carries additional data with respect to prefetch + * status that needs to be masked out. + */ +static u8 xe_to_user_access_type(u8 access_type) +{ + return access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK; +} + +static u8 xe_to_user_fault_type(u8 fault_type) +{ + return fault_type; +} + +static u8 xe_to_user_fault_level(u8 fault_level) +{ + return fault_level; +} + +static int fill_faults(struct xe_vm *vm, + struct drm_xe_vm_get_property *args) +{ + struct xe_vm_fault __user *usr_ptr = u64_to_user_ptr(args->data); + struct xe_vm_fault *fault_list, fault_entry = { 0 }; + struct xe_vm_fault_entry *entry; + int ret = 0, i = 0, count, entry_size; + + entry_size = sizeof(struct xe_vm_fault); + count = args->size / entry_size; + + fault_list = kcalloc(count, sizeof(struct xe_vm_fault), GFP_KERNEL); + if (!fault_list) + return -ENOMEM; + + spin_lock(&vm->faults.lock); + list_for_each_entry(entry, &vm->faults.list, list) { + if (i == count) + break; + + fault_entry.address = xe_device_canonicalize_addr(vm->xe, entry->address); + fault_entry.address_precision = entry->address_precision; + + fault_entry.access_type = xe_to_user_access_type(entry->access_type); + fault_entry.fault_type = xe_to_user_fault_type(entry->fault_type); + fault_entry.fault_level = xe_to_user_fault_level(entry->fault_level); + + memcpy(&fault_list[i], &fault_entry, entry_size); + + i++; + } + spin_unlock(&vm->faults.lock); + + ret = copy_to_user(usr_ptr, fault_list, args->size); + + kfree(fault_list); + return ret ? -EFAULT : 0; +} + +static int xe_vm_get_property_helper(struct xe_vm *vm, + struct drm_xe_vm_get_property *args) +{ + size_t size; + + switch (args->property) { + case DRM_XE_VM_GET_PROPERTY_FAULTS: + spin_lock(&vm->faults.lock); + size = size_mul(sizeof(struct xe_vm_fault), vm->faults.len); + spin_unlock(&vm->faults.lock); + + if (!args->size) { + args->size = size; + return 0; + } + + /* + * Number of faults may increase between calls to + * xe_vm_get_property_ioctl, so just report the number of + * faults the user requests if it's less than or equal to + * the number of faults in the VM fault array. + * + * We should also at least assert that the args->size value + * is a multiple of the xe_vm_fault struct size. + */ + if (args->size > size || args->size % sizeof(struct xe_vm_fault)) + return -EINVAL; + + return fill_faults(vm, args); + } + return -EINVAL; +} + +int xe_vm_get_property_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_get_property *args = data; + struct xe_vm *vm; + int ret = 0; + + if (XE_IOCTL_DBG(xe, (args->reserved[0] || args->reserved[1] || + args->reserved[2] || args->extensions || + args->pad))) + return -EINVAL; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -ENOENT; + + ret = xe_vm_get_property_helper(vm, args); + + xe_vm_put(vm); + return ret; +} + +/** + * xe_vm_bind_kernel_bo - bind a kernel BO to a VM + * @vm: VM to bind the BO to + * @bo: BO to bind + * @q: exec queue to use for the bind (optional) + * @addr: address at which to bind the BO + * @cache_lvl: PAT cache level to use + * + * Execute a VM bind map operation on a kernel-owned BO to bind it into a + * kernel-owned VM. + * + * Returns a dma_fence to track the binding completion if the job to do so was + * successfully submitted, an error pointer otherwise. + */ +struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo, + struct xe_exec_queue *q, u64 addr, + enum xe_cache_level cache_lvl) +{ + struct xe_vma_ops vops; + struct drm_gpuva_ops *ops = NULL; + struct dma_fence *fence; + int err; + + xe_bo_get(bo); + xe_vm_get(vm); + if (q) + xe_exec_queue_get(q); + + down_write(&vm->lock); + + xe_vma_ops_init(&vops, vm, q, NULL, 0); + + ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo), + DRM_XE_VM_BIND_OP_MAP, 0, 0, + xe_cache_pat_idx(vm->xe, cache_lvl)); + if (IS_ERR(ops)) { + err = PTR_ERR(ops); + goto release_vm_lock; + } + + err = vm_bind_ioctl_ops_parse(vm, ops, &vops); + if (err) + goto release_vm_lock; + + xe_assert(vm->xe, !list_empty(&vops.list)); + + err = xe_vma_ops_alloc(&vops, false); + if (err) + goto unwind_ops; + + fence = vm_bind_ioctl_ops_execute(vm, &vops); + if (IS_ERR(fence)) + err = PTR_ERR(fence); + +unwind_ops: + if (err && err != -ENODATA) + vm_bind_ioctl_ops_unwind(vm, &ops, 1); + + xe_vma_ops_fini(&vops); + drm_gpuva_ops_free(&vm->gpuvm, ops); + +release_vm_lock: + up_write(&vm->lock); + + if (q) + xe_exec_queue_put(q); + xe_vm_put(vm); + xe_bo_put(bo); + + if (err) + fence = ERR_PTR(err); + + return fence; +} + +/** + * xe_vm_lock() - Lock the vm's dma_resv object + * @vm: The struct xe_vm whose lock is to be locked + * @intr: Whether to perform any wait interruptible + * + * Return: 0 on success, -EINTR if @intr is true and the wait for a + * contended lock was interrupted. If @intr is false, the function + * always returns 0. + */ +int xe_vm_lock(struct xe_vm *vm, bool intr) +{ + int ret; + + if (intr) + ret = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); + else + ret = dma_resv_lock(xe_vm_resv(vm), NULL); + + return ret; +} + +/** + * xe_vm_unlock() - Unlock the vm's dma_resv object + * @vm: The struct xe_vm whose lock is to be released. + * + * Unlock a buffer object lock that was locked by xe_vm_lock(). + */ +void xe_vm_unlock(struct xe_vm *vm) +{ + dma_resv_unlock(xe_vm_resv(vm)); +} + +/** + * xe_vm_invalidate_vma_submit - Submit a job to invalidate GPU mappings for + * VMA. + * @vma: VMA to invalidate + * @batch: TLB invalidation batch to populate; caller must later call + * xe_tlb_inval_batch_wait() on it to wait for completion + * + * Walks a list of page tables leaves which it memset the entries owned by this + * VMA to zero, invalidates the TLBs, but doesn't block waiting for TLB flush + * to complete, but instead populates @batch which can be waited on using + * xe_tlb_inval_batch_wait(). + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct xe_tlb_inval_batch *batch) +{ + struct xe_device *xe = xe_vma_vm(vma)->xe; + struct xe_vm *vm = xe_vma_vm(vma); + struct xe_tile *tile; + u8 tile_mask = 0; + int ret = 0; + u8 id; + + xe_assert(xe, !xe_vma_is_null(vma)); + xe_assert(xe, !xe_vma_is_cpu_addr_mirror(vma)); + trace_xe_vma_invalidate(vma); + + vm_dbg(&vm->xe->drm, + "INVALIDATE: addr=0x%016llx, range=0x%016llx", + xe_vma_start(vma), xe_vma_size(vma)); + + /* + * Check that we don't race with page-table updates, tile_invalidated + * update is safe + */ + if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { + if (xe_vma_is_userptr(vma)) { + lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || + (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && + lockdep_is_held(&xe_vm_resv(vm)->lock.base))); + + WARN_ON_ONCE(!mmu_interval_check_retry + (&to_userptr_vma(vma)->userptr.notifier, + to_userptr_vma(vma)->userptr.pages.notifier_seq)); + WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(vm), + DMA_RESV_USAGE_BOOKKEEP)); + + } else { + xe_bo_assert_held(xe_vma_bo(vma)); + } + } + + for_each_tile(tile, xe, id) + if (xe_pt_zap_ptes(tile, vma)) + tile_mask |= BIT(id); + + xe_device_wmb(xe); + + ret = xe_tlb_inval_range_tilemask_submit(xe, xe_vma_vm(vma)->usm.asid, + xe_vma_start(vma), xe_vma_end(vma), + tile_mask, batch); + + /* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ + WRITE_ONCE(vma->tile_invalidated, vma->tile_mask); + return ret; +} + +/** + * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock + * @vma: VMA to invalidate + * + * Walks a list of page tables leaves which it memset the entries owned by this + * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is + * complete. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_vm_invalidate_vma(struct xe_vma *vma) +{ + struct xe_tlb_inval_batch batch; + int ret; + + ret = xe_vm_invalidate_vma_submit(vma, &batch); + if (ret) + return ret; + + xe_tlb_inval_batch_wait(&batch); + return ret; +} + +int xe_vm_validate_protected(struct xe_vm *vm) +{ + struct drm_gpuva *gpuva; + int err = 0; + + if (!vm) + return -ENODEV; + + mutex_lock(&vm->snap_mutex); + + drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + struct xe_bo *bo = vma->gpuva.gem.obj ? + gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; + + if (!bo) + continue; + + if (xe_bo_is_protected(bo)) { + err = xe_pxp_bo_key_check(vm->xe->pxp, bo); + if (err) + break; + } + } + + mutex_unlock(&vm->snap_mutex); + return err; +} + +struct xe_vm_snapshot { + int uapi_flags; + unsigned long num_snaps; + struct { + u64 ofs, bo_ofs; + unsigned long len; +#define XE_VM_SNAP_FLAG_USERPTR BIT(0) +#define XE_VM_SNAP_FLAG_READ_ONLY BIT(1) +#define XE_VM_SNAP_FLAG_IS_NULL BIT(2) + unsigned long flags; + int uapi_mem_region; + u16 pat_index; + int cpu_caching; + struct xe_bo *bo; + void *data; + struct mm_struct *mm; + } snap[]; +}; + +struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) +{ + unsigned long num_snaps = 0, i; + struct xe_vm_snapshot *snap = NULL; + struct drm_gpuva *gpuva; + + if (!vm) + return NULL; + + mutex_lock(&vm->snap_mutex); + drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { + if (gpuva->flags & XE_VMA_DUMPABLE) + num_snaps++; + } + + if (num_snaps) + snap = kvzalloc(offsetof(struct xe_vm_snapshot, snap[num_snaps]), GFP_NOWAIT); + if (!snap) { + snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV); + goto out_unlock; + } + + if (vm->flags & XE_VM_FLAG_FAULT_MODE) + snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_FAULT_MODE; + if (vm->flags & XE_VM_FLAG_LR_MODE) + snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_LR_MODE; + if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE) + snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; + + snap->num_snaps = num_snaps; + i = 0; + drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + struct xe_bo *bo = vma->gpuva.gem.obj ? + gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; + + if (!(gpuva->flags & XE_VMA_DUMPABLE)) + continue; + + snap->snap[i].ofs = xe_vma_start(vma); + snap->snap[i].len = xe_vma_size(vma); + snap->snap[i].flags = xe_vma_read_only(vma) ? + XE_VM_SNAP_FLAG_READ_ONLY : 0; + snap->snap[i].pat_index = vma->attr.pat_index; + if (bo) { + snap->snap[i].cpu_caching = bo->cpu_caching; + snap->snap[i].bo = xe_bo_get(bo); + snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); + switch (bo->ttm.resource->mem_type) { + case XE_PL_SYSTEM: + case XE_PL_TT: + snap->snap[i].uapi_mem_region = 0; + break; + case XE_PL_VRAM0: + snap->snap[i].uapi_mem_region = 1; + break; + case XE_PL_VRAM1: + snap->snap[i].uapi_mem_region = 2; + break; + } + } else if (xe_vma_is_userptr(vma)) { + struct mm_struct *mm = + to_userptr_vma(vma)->userptr.notifier.mm; + + if (mmget_not_zero(mm)) + snap->snap[i].mm = mm; + else + snap->snap[i].data = ERR_PTR(-EFAULT); + + snap->snap[i].bo_ofs = xe_vma_userptr(vma); + snap->snap[i].flags |= XE_VM_SNAP_FLAG_USERPTR; + snap->snap[i].uapi_mem_region = 0; + } else if (xe_vma_is_null(vma)) { + snap->snap[i].flags |= XE_VM_SNAP_FLAG_IS_NULL; + snap->snap[i].uapi_mem_region = -1; + } else { + snap->snap[i].data = ERR_PTR(-ENOENT); + snap->snap[i].uapi_mem_region = -1; + } + i++; + } + +out_unlock: + mutex_unlock(&vm->snap_mutex); + return snap; +} + +void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap) +{ + if (IS_ERR_OR_NULL(snap)) + return; + + for (int i = 0; i < snap->num_snaps; i++) { + struct xe_bo *bo = snap->snap[i].bo; + int err; + + if (IS_ERR(snap->snap[i].data) || + snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL) + continue; + + snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER); + if (!snap->snap[i].data) { + snap->snap[i].data = ERR_PTR(-ENOMEM); + goto cleanup_bo; + } + + if (bo) { + err = xe_bo_read(bo, snap->snap[i].bo_ofs, + snap->snap[i].data, snap->snap[i].len); + } else { + void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; + + kthread_use_mm(snap->snap[i].mm); + if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) + err = 0; + else + err = -EFAULT; + kthread_unuse_mm(snap->snap[i].mm); + + mmput(snap->snap[i].mm); + snap->snap[i].mm = NULL; + } + + if (err) { + kvfree(snap->snap[i].data); + snap->snap[i].data = ERR_PTR(err); + } + +cleanup_bo: + xe_bo_put(bo); + snap->snap[i].bo = NULL; + } +} + +void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) +{ + unsigned long i, j; + + if (IS_ERR_OR_NULL(snap)) { + drm_printf(p, "[0].error: %li\n", PTR_ERR(snap)); + return; + } + + drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags); + for (i = 0; i < snap->num_snaps; i++) { + drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); + + drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx|pat_index=%d|cpu_caching=%d\n", + snap->snap[i].ofs, + snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? + "read_only" : "read_write", + snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ? + "null_sparse" : + snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ? + "userptr" : "bo", + snap->snap[i].uapi_mem_region == -1 ? 0 : + BIT(snap->snap[i].uapi_mem_region), + snap->snap[i].pat_index, + snap->snap[i].cpu_caching); + + if (IS_ERR(snap->snap[i].data)) { + drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, + PTR_ERR(snap->snap[i].data)); + continue; + } + + if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL) + continue; + + drm_printf(p, "[%llx].data: ", snap->snap[i].ofs); + + for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) { + u32 *val = snap->snap[i].data + j; + char dumped[ASCII85_BUFSZ]; + + drm_puts(p, ascii85_encode(*val, dumped)); + } + + drm_puts(p, "\n"); + + if (drm_coredump_printer_is_full(p)) + return; + } +} + +void xe_vm_snapshot_free(struct xe_vm_snapshot *snap) +{ + unsigned long i; + + if (IS_ERR_OR_NULL(snap)) + return; + + for (i = 0; i < snap->num_snaps; i++) { + if (!IS_ERR(snap->snap[i].data)) + kvfree(snap->snap[i].data); + xe_bo_put(snap->snap[i].bo); + if (snap->snap[i].mm) + mmput(snap->snap[i].mm); + } + kvfree(snap); +} + +/** + * xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations + * @xe: Pointer to the Xe device structure + * @vma: Pointer to the virtual memory area (VMA) structure + * @is_atomic: In pagefault path and atomic operation + * + * This function determines whether the given VMA needs to be migrated to + * VRAM in order to do atomic GPU operation. + * + * Return: + * 1 - Migration to VRAM is required + * 0 - Migration is not required + * -EACCES - Invalid access for atomic memory attr + * + */ +int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic) +{ + u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access : + vma->attr.atomic_access; + + if (!IS_DGFX(xe) || !is_atomic) + return false; + + /* + * NOTE: The checks implemented here are platform-specific. For + * instance, on a device supporting CXL atomics, these would ideally + * work universally without additional handling. + */ + switch (atomic_access) { + case DRM_XE_ATOMIC_DEVICE: + return !xe->info.has_device_atomics_on_smem; + + case DRM_XE_ATOMIC_CPU: + return -EACCES; + + case DRM_XE_ATOMIC_UNDEFINED: + case DRM_XE_ATOMIC_GLOBAL: + default: + return 1; + } +} + +static int xe_vm_alloc_vma(struct xe_vm *vm, + struct drm_gpuvm_map_req *map_req, + bool is_madvise) +{ + struct xe_vma_ops vops; + struct drm_gpuva_ops *ops = NULL; + struct drm_gpuva_op *__op; + unsigned int vma_flags = 0; + bool remap_op = false; + struct xe_vma_mem_attr tmp_attr = {}; + u16 default_pat; + int err; + + lockdep_assert_held_write(&vm->lock); + + if (is_madvise) + ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req); + else + ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req); + + if (IS_ERR(ops)) + return PTR_ERR(ops); + + if (list_empty(&ops->list)) { + err = 0; + goto free_ops; + } + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + struct xe_vma *vma = NULL; + + if (!is_madvise) { + if (__op->op == DRM_GPUVA_OP_UNMAP) { + vma = gpuva_to_vma(op->base.unmap.va); + XE_WARN_ON(!xe_vma_has_default_mem_attrs(vma)); + default_pat = vma->attr.default_pat_index; + vma_flags = vma->gpuva.flags; + } + + if (__op->op == DRM_GPUVA_OP_REMAP) { + vma = gpuva_to_vma(op->base.remap.unmap->va); + default_pat = vma->attr.default_pat_index; + vma_flags = vma->gpuva.flags; + } + + if (__op->op == DRM_GPUVA_OP_MAP) { + op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; + op->map.pat_index = default_pat; + } + } else { + if (__op->op == DRM_GPUVA_OP_REMAP) { + vma = gpuva_to_vma(op->base.remap.unmap->va); + xe_assert(vm->xe, !remap_op); + xe_assert(vm->xe, xe_vma_has_no_bo(vma)); + remap_op = true; + vma_flags = vma->gpuva.flags; + } + + if (__op->op == DRM_GPUVA_OP_MAP) { + xe_assert(vm->xe, remap_op); + remap_op = false; + /* + * In case of madvise ops DRM_GPUVA_OP_MAP is + * always after DRM_GPUVA_OP_REMAP, so ensure + * to propagate the flags from the vma we're + * unmapping. + */ + op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; + } + } + print_op(vm->xe, __op); + } + + xe_vma_ops_init(&vops, vm, NULL, NULL, 0); + + if (is_madvise) + vops.flags |= XE_VMA_OPS_FLAG_MADVISE; + else + vops.flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP; + + err = vm_bind_ioctl_ops_parse(vm, ops, &vops); + if (err) + goto unwind_ops; + + xe_vm_lock(vm, false); + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + struct xe_vma *vma; + + if (__op->op == DRM_GPUVA_OP_UNMAP) { + vma = gpuva_to_vma(op->base.unmap.va); + /* There should be no unmap for madvise */ + if (is_madvise) + XE_WARN_ON("UNEXPECTED UNMAP"); + + xe_vma_destroy(vma, NULL); + } else if (__op->op == DRM_GPUVA_OP_REMAP) { + vma = gpuva_to_vma(op->base.remap.unmap->va); + /* In case of madvise ops Store attributes for REMAP UNMAPPED + * VMA, so they can be assigned to newly MAP created vma. + */ + if (is_madvise) + xe_vma_mem_attr_copy(&tmp_attr, &vma->attr); + + xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); + } else if (__op->op == DRM_GPUVA_OP_MAP) { + vma = op->map.vma; + /* In case of madvise call, MAP will always be followed by REMAP. + * Therefore temp_attr will always have sane values, making it safe to + * copy them to new vma. + */ + if (is_madvise) + xe_vma_mem_attr_copy(&vma->attr, &tmp_attr); + } + } + + xe_vm_unlock(vm); + drm_gpuva_ops_free(&vm->gpuvm, ops); + xe_vma_mem_attr_fini(&tmp_attr); + return 0; + +unwind_ops: + vm_bind_ioctl_ops_unwind(vm, &ops, 1); +free_ops: + drm_gpuva_ops_free(&vm->gpuvm, ops); + return err; +} + +/** + * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops + * @vm: Pointer to the xe_vm structure + * @start: Starting input address + * @range: Size of the input range + * + * This function splits existing vma to create new vma for user provided input range + * + * Return: 0 if success + */ +int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) +{ + struct drm_gpuvm_map_req map_req = { + .map.va.addr = start, + .map.va.range = range, + }; + + lockdep_assert_held_write(&vm->lock); + + vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); + + return xe_vm_alloc_vma(vm, &map_req, true); +} + +static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma) +{ + return vma && xe_vma_is_cpu_addr_mirror(vma) && + xe_vma_has_default_mem_attrs(vma); +} + +/** + * xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs + * @vm: VM to search within + * @start: Input/output pointer to the starting address of the range + * @end: Input/output pointer to the end address of the range + * + * Given a range defined by @start and @range, this function checks the VMAs + * immediately before and after the range. If those neighboring VMAs are + * CPU-address-mirrored and have default memory attributes, the function + * updates @start and @range to include them. This extended range can then + * be used for merging or other operations that require a unified VMA. + * + * The function does not perform the merge itself; it only computes the + * mergeable boundaries. + */ +void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, u64 *start, u64 *end) +{ + struct xe_vma *prev, *next; + + lockdep_assert_held(&vm->lock); + + if (*start >= SZ_4K) { + prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K); + if (is_cpu_addr_vma_with_default_attr(prev)) + *start = xe_vma_start(prev); + } + + if (*end < vm->size) { + next = xe_vm_find_vma_by_addr(vm, *end + 1); + if (is_cpu_addr_vma_with_default_attr(next)) + *end = xe_vma_end(next); + } +} + +/** + * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma + * @vm: Pointer to the xe_vm structure + * @start: Starting input address + * @range: Size of the input range + * + * This function splits/merges existing vma to create new vma for user provided input range + * + * Return: 0 if success + */ +int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t range) +{ + struct drm_gpuvm_map_req map_req = { + .map.va.addr = start, + .map.va.range = range, + }; + + lockdep_assert_held_write(&vm->lock); + + vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx", + start, range); + + return xe_vm_alloc_vma(vm, &map_req, false); +} + +/** + * xe_vm_add_exec_queue() - Add exec queue to VM + * @vm: The VM. + * @q: The exec_queue + * + * Add exec queue to VM, skipped if the device does not have context based TLB + * invalidations. + */ +void xe_vm_add_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) +{ + struct xe_device *xe = vm->xe; + + /* User VMs and queues only */ + xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_KERNEL)); + xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); + xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); + xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_MIGRATE)); + xe_assert(xe, vm->xef); + xe_assert(xe, vm == q->vm); + + if (!xe->info.has_ctx_tlb_inval) + return; + + down_write(&vm->exec_queues.lock); + list_add(&q->vm_exec_queue_link, &vm->exec_queues.list[q->gt->info.id]); + ++vm->exec_queues.count[q->gt->info.id]; + up_write(&vm->exec_queues.lock); +} + +/** + * xe_vm_remove_exec_queue() - Remove exec queue from VM + * @vm: The VM. + * @q: The exec_queue + * + * Remove exec queue from VM, skipped if the device does not have context based + * TLB invalidations. + */ +void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) +{ + if (!vm->xe->info.has_ctx_tlb_inval) + return; + + down_write(&vm->exec_queues.lock); + if (!list_empty(&q->vm_exec_queue_link)) { + list_del(&q->vm_exec_queue_link); + --vm->exec_queues.count[q->gt->info.id]; + } + up_write(&vm->exec_queues.lock); +} diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h new file mode 100644 index 000000000..c5b900f38 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -0,0 +1,439 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_VM_H_ +#define _XE_VM_H_ + +#include "xe_assert.h" +#include "xe_bo_types.h" +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_vm_types.h" + +/** + * MAX_FAULTS_SAVED_PER_VM - Maximum number of faults each vm can store before future + * faults are discarded to prevent memory overuse + */ +#define MAX_FAULTS_SAVED_PER_VM 50 + +struct drm_device; +struct drm_printer; +struct drm_file; + +struct ttm_buffer_object; + +struct dma_fence; + +struct xe_exec_queue; +struct xe_file; +struct xe_pagefault; +struct xe_sync_entry; +struct xe_svm_range; +struct drm_exec; + +struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef); + +struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id); +int xe_vma_cmp_vma_cb(const void *key, const struct rb_node *node); + +static inline struct xe_vm *xe_vm_get(struct xe_vm *vm) +{ + drm_gpuvm_get(&vm->gpuvm); + return vm; +} + +static inline void xe_vm_put(struct xe_vm *vm) +{ + drm_gpuvm_put(&vm->gpuvm); +} + +int xe_vm_lock(struct xe_vm *vm, bool intr); + +void xe_vm_unlock(struct xe_vm *vm); + +static inline bool xe_vm_is_closed(struct xe_vm *vm) +{ + /* Only guaranteed not to change when vm->lock is held */ + return !vm->size; +} + +static inline bool xe_vm_is_banned(struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_BANNED; +} + +static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm) +{ + lockdep_assert_held(&vm->lock); + return xe_vm_is_closed(vm) || xe_vm_is_banned(vm); +} + +struct xe_vma * +xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range); + +bool xe_vma_has_default_mem_attrs(struct xe_vma *vma); + +void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, + u64 *start, + u64 *end); +/** + * xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs + * @vm: The vm + * + * Return: whether the vm populates unmapped areas with scratch PTEs + */ +static inline bool xe_vm_has_scratch(const struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_SCRATCH_PAGE; +} + +/** + * gpuvm_to_vm() - Return the embedding xe_vm from a struct drm_gpuvm pointer + * @gpuvm: The struct drm_gpuvm pointer + * + * Return: Pointer to the embedding struct xe_vm. + */ +static inline struct xe_vm *gpuvm_to_vm(struct drm_gpuvm *gpuvm) +{ + return container_of(gpuvm, struct xe_vm, gpuvm); +} + +static inline struct xe_vm *gpuva_to_vm(struct drm_gpuva *gpuva) +{ + return gpuvm_to_vm(gpuva->vm); +} + +static inline struct xe_vma *gpuva_to_vma(struct drm_gpuva *gpuva) +{ + return container_of(gpuva, struct xe_vma, gpuva); +} + +static inline struct xe_vma_op *gpuva_op_to_vma_op(struct drm_gpuva_op *op) +{ + return container_of(op, struct xe_vma_op, base); +} + +/** + * DOC: Provide accessors for vma members to facilitate easy change of + * implementation. + */ +static inline u64 xe_vma_start(struct xe_vma *vma) +{ + return vma->gpuva.va.addr; +} + +static inline u64 xe_vma_size(struct xe_vma *vma) +{ + return vma->gpuva.va.range; +} + +static inline u64 xe_vma_end(struct xe_vma *vma) +{ + return xe_vma_start(vma) + xe_vma_size(vma); +} + +static inline u64 xe_vma_bo_offset(struct xe_vma *vma) +{ + return vma->gpuva.gem.offset; +} + +static inline struct xe_bo *xe_vma_bo(struct xe_vma *vma) +{ + return !vma->gpuva.gem.obj ? NULL : + container_of(vma->gpuva.gem.obj, struct xe_bo, ttm.base); +} + +static inline struct xe_vm *xe_vma_vm(struct xe_vma *vma) +{ + return container_of(vma->gpuva.vm, struct xe_vm, gpuvm); +} + +static inline bool xe_vma_read_only(struct xe_vma *vma) +{ + return vma->gpuva.flags & XE_VMA_READ_ONLY; +} + +static inline u64 xe_vma_userptr(struct xe_vma *vma) +{ + return vma->gpuva.gem.offset; +} + +static inline bool xe_vma_is_null(struct xe_vma *vma) +{ + return vma->gpuva.flags & DRM_GPUVA_SPARSE; +} + +static inline bool xe_vma_is_cpu_addr_mirror(struct xe_vma *vma) +{ + return vma->gpuva.flags & XE_VMA_SYSTEM_ALLOCATOR; +} + +static inline bool xe_vma_has_no_bo(struct xe_vma *vma) +{ + return !xe_vma_bo(vma); +} + +static inline bool xe_vma_is_userptr(struct xe_vma *vma) +{ + return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma) && + !xe_vma_is_cpu_addr_mirror(vma); +} + +struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr); + +int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic); + +int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t addr, uint64_t size); + +int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t addr, uint64_t size); + +/** + * to_userptr_vma() - Return a pointer to an embedding userptr vma + * @vma: Pointer to the embedded struct xe_vma + * + * Return: Pointer to the embedding userptr vma + */ +static inline struct xe_userptr_vma *to_userptr_vma(struct xe_vma *vma) +{ + xe_assert(xe_vma_vm(vma)->xe, xe_vma_is_userptr(vma)); + return container_of(vma, struct xe_userptr_vma, vma); +} + +u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile); + +int xe_vm_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_vm_bind_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int xe_vm_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +void xe_vm_close_and_put(struct xe_vm *vm); + +static inline bool xe_vm_in_fault_mode(struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_FAULT_MODE; +} + +static inline bool xe_vm_in_lr_mode(struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_LR_MODE; +} + +static inline bool xe_vm_in_preempt_fence_mode(struct xe_vm *vm) +{ + return xe_vm_in_lr_mode(vm) && !xe_vm_in_fault_mode(vm); +} + +static inline bool xe_vm_allow_vm_eviction(struct xe_vm *vm) +{ + return !xe_vm_in_lr_mode(vm) || + (xe_vm_in_fault_mode(vm) && + !(vm->flags & XE_VM_FLAG_NO_VM_OVERCOMMIT)); +} + +int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q); +void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q); + +int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker); +struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, + u8 tile_mask); +struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, + struct xe_vma *vma, + struct xe_svm_range *range, + u8 tile_mask); +struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, + struct xe_svm_range *range); + +int xe_vm_invalidate_vma(struct xe_vma *vma); + +int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct xe_tlb_inval_batch *batch); + +int xe_vm_validate_protected(struct xe_vm *vm); + +static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm) +{ + xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); + queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work); +} + +/** + * xe_vm_reactivate_rebind() - Reactivate the rebind functionality on compute + * vms. + * @vm: The vm. + * + * If the rebind functionality on a compute vm was disabled due + * to nothing to execute. Reactivate it and run the rebind worker. + * This function should be called after submitting a batch to a compute vm. + */ +static inline void xe_vm_reactivate_rebind(struct xe_vm *vm) +{ + if (xe_vm_in_preempt_fence_mode(vm) && vm->preempt.rebind_deactivated) { + vm->preempt.rebind_deactivated = false; + xe_vm_queue_rebind_worker(vm); + } +} + +int xe_vm_lock_vma(struct drm_exec *exec, struct xe_vma *vma); + +int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, + unsigned int num_fences); + +struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo, + struct xe_exec_queue *q, u64 addr, + enum xe_cache_level cache_lvl); + +void xe_vm_resume_rebind_worker(struct xe_vm *vm); + +/** + * xe_vm_resv() - Return's the vm's reservation object + * @vm: The vm + * + * Return: Pointer to the vm's reservation object. + */ +static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm) +{ + return drm_gpuvm_resv(&vm->gpuvm); +} + +void xe_vm_kill(struct xe_vm *vm, bool unlocked); + +void xe_vm_add_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q); +void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q); + +/** + * xe_vm_assert_held(vm) - Assert that the vm's reservation object is held. + * @vm: The vm + */ +#define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm)) + +int xe_vm_drm_exec_lock(struct xe_vm *vm, struct drm_exec *exec); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) +#define vm_dbg drm_dbg +#else +__printf(2, 3) +static inline void vm_dbg(const struct drm_device *dev, + const char *format, ...) +{ /* noop */ } +#endif + +struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm); +void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap); +void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p); +void xe_vm_snapshot_free(struct xe_vm_snapshot *snap); + +void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf); + +/** + * xe_vm_set_validating() - Register this task as currently making bos resident + * @allow_res_evict: Allow eviction of buffer objects bound to @vm when + * validating. + * @vm: Pointer to the vm or NULL. + * + * Register this task as currently making bos resident for the vm. Intended + * to avoid eviction by the same task of shared bos bound to the vm. + * Call with the vm's resv lock held. + */ +static inline void xe_vm_set_validating(struct xe_vm *vm, bool allow_res_evict) +{ + if (vm && !allow_res_evict) { + xe_vm_assert_held(vm); + /* Pairs with READ_ONCE in xe_vm_is_validating() */ + WRITE_ONCE(vm->validation.validating, current); + } +} + +/** + * xe_vm_clear_validating() - Unregister this task as currently making bos resident + * @vm: Pointer to the vm or NULL + * @allow_res_evict: Eviction from @vm was allowed. Must be set to the same + * value as for xe_vm_set_validation(). + * + * Register this task as currently making bos resident for the vm. Intended + * to avoid eviction by the same task of shared bos bound to the vm. + * Call with the vm's resv lock held. + */ +static inline void xe_vm_clear_validating(struct xe_vm *vm, bool allow_res_evict) +{ + if (vm && !allow_res_evict) { + /* Pairs with READ_ONCE in xe_vm_is_validating() */ + WRITE_ONCE(vm->validation.validating, NULL); + } +} + +/** + * xe_vm_is_validating() - Whether bos bound to the vm are currently being made resident + * by the current task. + * @vm: Pointer to the vm. + * + * If this function returns %true, we should be in a vm resv locked region, since + * the current process is the same task that called xe_vm_set_validating(). + * The function asserts that that's indeed the case. + * + * Return: %true if the task is currently making bos resident, %false otherwise. + */ +static inline bool xe_vm_is_validating(struct xe_vm *vm) +{ + /* Pairs with WRITE_ONCE in xe_vm_is_validating() */ + if (READ_ONCE(vm->validation.validating) == current) { + xe_vm_assert_held(vm); + return true; + } + return false; +} + +/** + * xe_vm_set_validation_exec() - Accessor to set the drm_exec object + * @vm: The vm we want to register a drm_exec object with. + * @exec: The exec object we want to register. + * + * Set the drm_exec object used to lock the vm's resv. + */ +static inline void xe_vm_set_validation_exec(struct xe_vm *vm, struct drm_exec *exec) +{ + xe_vm_assert_held(vm); + xe_assert(vm->xe, !!exec ^ !!vm->validation._exec); + vm->validation._exec = exec; +} + +/** + * xe_vm_validation_exec() - Accessor to read the drm_exec object + * @vm: The vm we want to register a drm_exec object with. + * + * Return: The drm_exec object used to lock the vm's resv. The value + * is a valid pointer, %NULL, or one of the special values defined in + * xe_validation.h. + */ +static inline struct drm_exec *xe_vm_validation_exec(struct xe_vm *vm) +{ + xe_vm_assert_held(vm); + return vm->validation._exec; +} + +/** + * xe_vm_has_valid_gpu_mapping() - Advisory helper to check if VMA or SVM range has + * a valid GPU mapping + * @tile: The tile which the GPU mapping belongs to + * @tile_present: Tile present mask + * @tile_invalidated: Tile invalidated mask + * + * The READ_ONCEs pair with WRITE_ONCEs in either the TLB invalidation paths + * (xe_vm.c, xe_svm.c) or the binding paths (xe_pt.c). These are not reliable + * without the notifier lock in userptr or SVM cases, and not reliable without + * the BO dma-resv lock in the BO case. As such, they should only be used in + * opportunistic cases (e.g., skipping a page fault fix or not skipping a TLB + * invalidation) where it is harmless. + * + * Return: True is there are valid GPU pages, False otherwise + */ +#define xe_vm_has_valid_gpu_mapping(tile, tile_present, tile_invalidated) \ + ((READ_ONCE(tile_present) & ~READ_ONCE(tile_invalidated)) & BIT((tile)->id)) + +void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from); +#endif diff --git a/drivers/gpu/drm/xe/xe_vm_doc.h b/drivers/gpu/drm/xe/xe_vm_doc.h new file mode 100644 index 000000000..02e528837 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_doc.h @@ -0,0 +1,555 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_VM_DOC_H_ +#define _XE_VM_DOC_H_ + +/** + * DOC: Xe VM (user address space) + * + * VM creation + * =========== + * + * Allocate a physical page for root of the page table structure, create default + * bind engine, and return a handle to the user. + * + * Scratch page + * ------------ + * + * If the VM is created with the flag, DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE, set the + * entire page table structure defaults pointing to blank page allocated by the + * VM. Invalid memory access rather than fault just read / write to this page. + * + * VM bind (create GPU mapping for a BO or userptr) + * ================================================ + * + * Creates GPU mappings for a BO or userptr within a VM. VM binds uses the same + * in / out fence interface (struct drm_xe_sync) as execs which allows users to + * think of binds and execs as more or less the same operation. + * + * Operations + * ---------- + * + * DRM_XE_VM_BIND_OP_MAP - Create mapping for a BO + * DRM_XE_VM_BIND_OP_UNMAP - Destroy mapping for a BO / userptr + * DRM_XE_VM_BIND_OP_MAP_USERPTR - Create mapping for userptr + * + * Implementation details + * ~~~~~~~~~~~~~~~~~~~~~~ + * + * All bind operations are implemented via a hybrid approach of using the CPU + * and GPU to modify page tables. If a new physical page is allocated in the + * page table structure we populate that page via the CPU and insert that new + * page into the existing page table structure via a GPU job. Also any existing + * pages in the page table structure that need to be modified also are updated + * via the GPU job. As the root physical page is prealloced on VM creation our + * GPU job will always have at least 1 update. The in / out fences are passed to + * this job so again this is conceptually the same as an exec. + * + * Very simple example of few binds on an empty VM with 48 bits of address space + * and the resulting operations: + * + * .. code-block:: + * + * bind BO0 0x0-0x1000 + * alloc page level 3a, program PTE[0] to BO0 phys address (CPU) + * alloc page level 2, program PDE[0] page level 3a phys address (CPU) + * alloc page level 1, program PDE[0] page level 2 phys address (CPU) + * update root PDE[0] to page level 1 phys address (GPU) + * + * bind BO1 0x201000-0x202000 + * alloc page level 3b, program PTE[1] to BO1 phys address (CPU) + * update page level 2 PDE[1] to page level 3b phys address (GPU) + * + * bind BO2 0x1ff000-0x201000 + * update page level 3a PTE[511] to BO2 phys address (GPU) + * update page level 3b PTE[0] to BO2 phys address + 0x1000 (GPU) + * + * GPU bypass + * ~~~~~~~~~~ + * + * In the above example the steps using the GPU can be converted to CPU if the + * bind can be done immediately (all in-fences satisfied, VM dma-resv kernel + * slot is idle). + * + * Address space + * ------------- + * + * Depending on platform either 48 or 57 bits of address space is supported. + * + * Page sizes + * ---------- + * + * The minimum page size is either 4k or 64k depending on platform and memory + * placement (sysmem vs. VRAM). We enforce that binds must be aligned to the + * minimum page size. + * + * Larger pages (2M or 1GB) can be used for BOs in VRAM, the BO physical address + * is aligned to the larger pages size, and VA is aligned to the larger page + * size. Larger pages for userptrs / BOs in sysmem should be possible but is not + * yet implemented. + * + * Sync error handling mode + * ------------------------ + * + * In both modes during the bind IOCTL the user input is validated. In sync + * error handling mode the newly bound BO is validated (potentially moved back + * to a region of memory where is can be used), page tables are updated by the + * CPU and the job to do the GPU binds is created in the IOCTL itself. This step + * can fail due to memory pressure. The user can recover by freeing memory and + * trying this operation again. + * + * Async error handling mode + * ------------------------- + * + * In async error handling the step of validating the BO, updating page tables, + * and generating a job are deferred to an async worker. As this step can now + * fail after the IOCTL has reported success we need an error handling flow for + * which the user can recover from. + * + * The solution is for a user to register a user address with the VM which the + * VM uses to report errors to. The ufence wait interface can be used to wait on + * a VM going into an error state. Once an error is reported the VM's async + * worker is paused. While the VM's async worker is paused sync, + * DRM_XE_VM_BIND_OP_UNMAP operations are allowed (this can free memory). Once the + * uses believe the error state is fixed, the async worker can be resumed via + * XE_VM_BIND_OP_RESTART operation. When VM async bind work is restarted, the + * first operation processed is the operation that caused the original error. + * + * Bind queues / engines + * --------------------- + * + * Think of the case where we have two bind operations A + B and are submitted + * in that order. A has in fences while B has none. If using a single bind + * queue, B is now blocked on A's in fences even though it is ready to run. This + * example is a real use case for VK sparse binding. We work around this + * limitation by implementing bind engines. + * + * In the bind IOCTL the user can optionally pass in an engine ID which must map + * to an engine which is of the special class DRM_XE_ENGINE_CLASS_VM_BIND. + * Underneath this is a really virtual engine that can run on any of the copy + * hardware engines. The job(s) created each IOCTL are inserted into this + * engine's ring. In the example above if A and B have different bind engines B + * is free to pass A. If the engine ID field is omitted, the default bind queue + * for the VM is used. + * + * TODO: Explain race in issue 41 and how we solve it + * + * Array of bind operations + * ------------------------ + * + * The uAPI allows multiple binds operations to be passed in via a user array, + * of struct drm_xe_vm_bind_op, in a single VM bind IOCTL. This interface + * matches the VK sparse binding API. The implementation is rather simple, parse + * the array into a list of operations, pass the in fences to the first operation, + * and pass the out fences to the last operation. The ordered nature of a bind + * engine makes this possible. + * + * Munmap semantics for unbinds + * ---------------------------- + * + * Munmap allows things like: + * + * .. code-block:: + * + * 0x0000-0x2000 and 0x3000-0x5000 have mappings + * Munmap 0x1000-0x4000, results in mappings 0x0000-0x1000 and 0x4000-0x5000 + * + * To support this semantic in the above example we decompose the above example + * into 4 operations: + * + * .. code-block:: + * + * unbind 0x0000-0x2000 + * unbind 0x3000-0x5000 + * rebind 0x0000-0x1000 + * rebind 0x4000-0x5000 + * + * Why not just do a partial unbind of 0x1000-0x2000 and 0x3000-0x4000? This + * falls apart when using large pages at the edges and the unbind forces us to + * use a smaller page size. For simplity we always issue a set of unbinds + * unmapping anything in the range and at most 2 rebinds on the edges. + * + * Similar to an array of binds, in fences are passed to the first operation and + * out fences are signaled on the last operation. + * + * In this example there is a window of time where 0x0000-0x1000 and + * 0x4000-0x5000 are invalid but the user didn't ask for these addresses to be + * removed from the mapping. To work around this we treat any munmap style + * unbinds which require a rebind as a kernel operations (BO eviction or userptr + * invalidation). The first operation waits on the VM's + * DMA_RESV_USAGE_PREEMPT_FENCE slots (waits for all pending jobs on VM to + * complete / triggers preempt fences) and the last operation is installed in + * the VM's DMA_RESV_USAGE_KERNEL slot (blocks future jobs / resume compute mode + * VM). The caveat is all dma-resv slots must be updated atomically with respect + * to execs and compute mode rebind worker. To accomplish this, hold the + * vm->lock in write mode from the first operation until the last. + * + * Deferred binds in fault mode + * ---------------------------- + * + * If a VM is in fault mode (TODO: link to fault mode), new bind operations that + * create mappings are by default deferred to the page fault handler (first + * use). This behavior can be overridden by setting the flag + * DRM_XE_VM_BIND_FLAG_IMMEDIATE which indicates to creating the mapping + * immediately. + * + * User pointer + * ============ + * + * User pointers are user allocated memory (malloc'd, mmap'd, etc..) for which the + * user wants to create a GPU mapping. Typically in other DRM drivers a dummy BO + * was created and then a binding was created. We bypass creating a dummy BO in + * Xe and simply create a binding directly from the userptr. + * + * Invalidation + * ------------ + * + * Since this a core kernel managed memory the kernel can move this memory + * whenever it wants. We register an invalidation MMU notifier to alert Xe when + * a user pointer is about to move. The invalidation notifier needs to block + * until all pending users (jobs or compute mode engines) of the userptr are + * idle to ensure no faults. This done by waiting on all of VM's dma-resv slots. + * + * Rebinds + * ------- + * + * Either the next exec (non-compute) or rebind worker (compute mode) will + * rebind the userptr. The invalidation MMU notifier kicks the rebind worker + * after the VM dma-resv wait if the VM is in compute mode. + * + * Compute mode + * ============ + * + * A VM in compute mode enables long running workloads and ultra low latency + * submission (ULLS). ULLS is implemented via a continuously running batch + + * semaphores. This enables the user to insert jump to new batch commands + * into the continuously running batch. In both cases these batches exceed the + * time a dma fence is allowed to exist for before signaling, as such dma fences + * are not used when a VM is in compute mode. User fences (TODO: link user fence + * doc) are used instead to signal operation's completion. + * + * Preempt fences + * -------------- + * + * If the kernel decides to move memory around (either userptr invalidate, BO + * eviction, or mumap style unbind which results in a rebind) and a batch is + * running on an engine, that batch can fault or cause a memory corruption as + * page tables for the moved memory are no longer valid. To work around this we + * introduce the concept of preempt fences. When sw signaling is enabled on a + * preempt fence it tells the submission backend to kick that engine off the + * hardware and the preempt fence signals when the engine is off the hardware. + * Once all preempt fences are signaled for a VM the kernel can safely move the + * memory and kick the rebind worker which resumes all the engines execution. + * + * A preempt fence, for every engine using the VM, is installed into the VM's + * dma-resv DMA_RESV_USAGE_PREEMPT_FENCE slot. The same preempt fence, for every + * engine using the VM, is also installed into the same dma-resv slot of every + * external BO mapped in the VM. + * + * Rebind worker + * ------------- + * + * The rebind worker is very similar to an exec. It is responsible for rebinding + * evicted BOs or userptrs, waiting on those operations, installing new preempt + * fences, and finally resuming executing of engines in the VM. + * + * Flow + * ~~~~ + * + * .. code-block:: + * + * <----------------------------------------------------------------------| + * Check if VM is closed, if so bail out | + * Lock VM global lock in read mode | + * Pin userptrs (also finds userptr invalidated since last rebind worker) | + * Lock VM dma-resv and external BOs dma-resv | + * Validate BOs that have been evicted | + * Wait on and allocate new preempt fences for every engine using the VM | + * Rebind invalidated userptrs + evicted BOs | + * Wait on last rebind fence | + * Wait VM's DMA_RESV_USAGE_KERNEL dma-resv slot | + * Install preeempt fences and issue resume for every engine using the VM | + * Check if any userptrs invalidated since pin | + * Squash resume for all engines | + * Unlock all | + * Wait all VM's dma-resv slots | + * Retry ---------------------------------------------------------- + * Release all engines waiting to resume + * Unlock all + * + * Timeslicing + * ----------- + * + * In order to prevent an engine from continuously being kicked off the hardware + * and making no forward progress an engine has a period of time it allowed to + * run after resume before it can be kicked off again. This effectively gives + * each engine a timeslice. + * + * Handling multiple GTs + * ===================== + * + * If a GT has slower access to some regions and the page table structure are in + * the slow region, the performance on that GT could adversely be affected. To + * work around this we allow a VM page tables to be shadowed in multiple GTs. + * When VM is created, a default bind engine and PT table structure are created + * on each GT. + * + * Binds can optionally pass in a mask of GTs where a mapping should be created, + * if this mask is zero then default to all the GTs where the VM has page + * tables. + * + * The implementation for this breaks down into a bunch for_each_gt loops in + * various places plus exporting a composite fence for multi-GT binds to the + * user. + * + * Fault mode (unified shared memory) + * ================================== + * + * A VM in fault mode can be enabled on devices that support page faults. If + * page faults are enabled, using dma fences can potentially induce a deadlock: + * A pending page fault can hold up the GPU work which holds up the dma fence + * signaling, and memory allocation is usually required to resolve a page + * fault, but memory allocation is not allowed to gate dma fence signaling. As + * such, dma fences are not allowed when VM is in fault mode. Because dma-fences + * are not allowed, only long running workloads and ULLS are enabled on a faulting + * VM. + * + * Deferred VM binds + * ---------------- + * + * By default, on a faulting VM binds just allocate the VMA and the actual + * updating of the page tables is deferred to the page fault handler. This + * behavior can be overridden by setting the flag DRM_XE_VM_BIND_FLAG_IMMEDIATE in + * the VM bind which will then do the bind immediately. + * + * Page fault handler + * ------------------ + * + * Page faults are received in the G2H worker under the CT lock which is in the + * path of dma fences (no memory allocations are allowed, faults require memory + * allocations) thus we cannot process faults under the CT lock. Another issue + * is faults issue TLB invalidations which require G2H credits and we cannot + * allocate G2H credits in the G2H handlers without deadlocking. Lastly, we do + * not want the CT lock to be an outer lock of the VM global lock (VM global + * lock required to fault processing). + * + * To work around the above issue with processing faults in the G2H worker, we + * sink faults to a buffer which is large enough to sink all possible faults on + * the GT (1 per hardware engine) and kick a worker to process the faults. Since + * the page faults G2H are already received in a worker, kicking another worker + * adds more latency to a critical performance path. We add a fast path in the + * G2H irq handler which looks at first G2H and if it is a page fault we sink + * the fault to the buffer and kick the worker to process the fault. TLB + * invalidation responses are also in the critical path so these can also be + * processed in this fast path. + * + * Multiple buffers and workers are used and hashed over based on the ASID so + * faults from different VMs can be processed in parallel. + * + * The page fault handler itself is rather simple, flow is below. + * + * .. code-block:: + * + * Lookup VM from ASID in page fault G2H + * Lock VM global lock in read mode + * Lookup VMA from address in page fault G2H + * Check if VMA is valid, if not bail + * Check if VMA's BO has backing store, if not allocate + * <----------------------------------------------------------------------| + * If userptr, pin pages | + * Lock VM & BO dma-resv locks | + * If atomic fault, migrate to VRAM, else validate BO location | + * Issue rebind | + * Wait on rebind to complete | + * Check if userptr invalidated since pin | + * Drop VM & BO dma-resv locks | + * Retry ---------------------------------------------------------- + * Unlock all + * Issue blocking TLB invalidation | + * Send page fault response to GuC + * + * Access counters + * --------------- + * + * Access counters can be configured to trigger a G2H indicating the device is + * accessing VMAs in system memory frequently as hint to migrate those VMAs to + * VRAM. + * + * Same as the page fault handler, access counters G2H cannot be processed the + * G2H worker under the CT lock. Again we use a buffer to sink access counter + * G2H. Unlike page faults there is no upper bound so if the buffer is full we + * simply drop the G2H. Access counters are a best case optimization and it is + * safe to drop these unlike page faults. + * + * The access counter handler itself is rather simple flow is below. + * + * .. code-block:: + * + * Lookup VM from ASID in access counter G2H + * Lock VM global lock in read mode + * Lookup VMA from address in access counter G2H + * If userptr, bail nothing to do + * Lock VM & BO dma-resv locks + * Issue migration to VRAM + * Unlock all + * + * Notice no rebind is issued in the access counter handler as the rebind will + * be issued on next page fault. + * + * Caveats with eviction / user pointer invalidation + * ------------------------------------------------- + * + * In the case of eviction and user pointer invalidation on a faulting VM, there + * is no need to issue a rebind rather we just need to blow away the page tables + * for the VMAs and the page fault handler will rebind the VMAs when they fault. + * The caveat is to update / read the page table structure the VM global lock is + * needed. In both the case of eviction and user pointer invalidation locks are + * held which make acquiring the VM global lock impossible. To work around this + * every VMA maintains a list of leaf page table entries which should be written + * to zero to blow away the VMA's page tables. After writing zero to these + * entries a blocking TLB invalidate is issued. At this point it is safe for the + * kernel to move the VMA's memory around. This is a necessary lockless + * algorithm and is safe as leafs cannot be changed while either an eviction or + * userptr invalidation is occurring. + * + * Locking + * ======= + * + * VM locking protects all of the core data paths (bind operations, execs, + * evictions, and compute mode rebind worker) in Xe. + * + * Locks + * ----- + * + * VM global lock (vm->lock) - rw semaphore lock. Outer most lock which protects + * the list of userptrs mapped in the VM, the list of engines using this VM, and + * the array of external BOs mapped in the VM. When adding or removing any of the + * aforementioned state from the VM should acquire this lock in write mode. The VM + * bind path also acquires this lock in write while the exec / compute mode + * rebind worker acquires this lock in read mode. + * + * VM dma-resv lock (vm->gpuvm.r_obj->resv->lock) - WW lock. Protects VM dma-resv + * slots which is shared with any private BO in the VM. Expected to be acquired + * during VM binds, execs, and compute mode rebind worker. This lock is also + * held when private BOs are being evicted. + * + * external BO dma-resv lock (bo->ttm.base.resv->lock) - WW lock. Protects + * external BO dma-resv slots. Expected to be acquired during VM binds (in + * addition to the VM dma-resv lock). All external BO dma-locks within a VM are + * expected to be acquired (in addition to the VM dma-resv lock) during execs + * and the compute mode rebind worker. This lock is also held when an external + * BO is being evicted. + * + * Putting it all together + * ----------------------- + * + * 1. An exec and bind operation with the same VM can't be executing at the same + * time (vm->lock). + * + * 2. A compute mode rebind worker and bind operation with the same VM can't be + * executing at the same time (vm->lock). + * + * 3. We can't add / remove userptrs or external BOs to a VM while an exec with + * the same VM is executing (vm->lock). + * + * 4. We can't add / remove userptrs, external BOs, or engines to a VM while a + * compute mode rebind worker with the same VM is executing (vm->lock). + * + * 5. Evictions within a VM can't be happen while an exec with the same VM is + * executing (dma-resv locks). + * + * 6. Evictions within a VM can't be happen while a compute mode rebind worker + * with the same VM is executing (dma-resv locks). + * + * dma-resv usage + * ============== + * + * As previously stated to enforce the ordering of kernel ops (eviction, userptr + * invalidation, munmap style unbinds which result in a rebind), rebinds during + * execs, execs, and resumes in the rebind worker we use both the VMs and + * external BOs dma-resv slots. Let try to make this as clear as possible. + * + * Slot installation + * ----------------- + * + * 1. Jobs from kernel ops install themselves into the DMA_RESV_USAGE_KERNEL + * slot of either an external BO or VM (depends on if kernel op is operating on + * an external or private BO) + * + * 2. In non-compute mode, jobs from execs install themselves into the + * DMA_RESV_USAGE_BOOKKEEP slot of the VM + * + * 3. In non-compute mode, jobs from execs install themselves into the + * DMA_RESV_USAGE_WRITE slot of all external BOs in the VM + * + * 4. Jobs from binds install themselves into the DMA_RESV_USAGE_BOOKKEEP slot + * of the VM + * + * 5. Jobs from binds install themselves into the DMA_RESV_USAGE_BOOKKEEP slot + * of the external BO (if the bind is to an external BO, this is addition to #4) + * + * 6. Every engine using a compute mode VM has a preempt fence in installed into + * the DMA_RESV_USAGE_PREEMPT_FENCE slot of the VM + * + * 7. Every engine using a compute mode VM has a preempt fence in installed into + * the DMA_RESV_USAGE_PREEMPT_FENCE slot of all the external BOs in the VM + * + * Slot waiting + * ------------ + * + * 1. The execution of all jobs from kernel ops shall wait on all slots + * (DMA_RESV_USAGE_PREEMPT_FENCE) of either an external BO or VM (depends on if + * kernel op is operating on external or private BO) + * + * 2. In non-compute mode, the execution of all jobs from rebinds in execs shall + * wait on the DMA_RESV_USAGE_KERNEL slot of either an external BO or VM + * (depends on if the rebind is operatiing on an external or private BO) + * + * 3. In non-compute mode, the execution of all jobs from execs shall wait on the + * last rebind job + * + * 4. In compute mode, the execution of all jobs from rebinds in the rebind + * worker shall wait on the DMA_RESV_USAGE_KERNEL slot of either an external BO + * or VM (depends on if rebind is operating on external or private BO) + * + * 5. In compute mode, resumes in rebind worker shall wait on last rebind fence + * + * 6. In compute mode, resumes in rebind worker shall wait on the + * DMA_RESV_USAGE_KERNEL slot of the VM + * + * Putting it all together + * ----------------------- + * + * 1. New jobs from kernel ops are blocked behind any existing jobs from + * non-compute mode execs + * + * 2. New jobs from non-compute mode execs are blocked behind any existing jobs + * from kernel ops and rebinds + * + * 3. New jobs from kernel ops are blocked behind all preempt fences signaling in + * compute mode + * + * 4. Compute mode engine resumes are blocked behind any existing jobs from + * kernel ops and rebinds + * + * Future work + * =========== + * + * Support large pages for sysmem and userptr. + * + * Update page faults to handle BOs are page level grainularity (e.g. part of BO + * could be in system memory while another part could be in VRAM). + * + * Page fault handler likely we be optimized a bit more (e.g. Rebinds always + * wait on the dma-resv kernel slots of VM or BO, technically we only have to + * wait the BO moving. If using a job to do the rebind, we could not block in + * the page fault handler rather attach a callback to fence of the rebind job to + * signal page fault complete. Our handling of short circuting for atomic faults + * for bound VMAs could be better. etc...). We can tune all of this once we have + * benchmarks / performance number from workloads up and running. + */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c new file mode 100644 index 000000000..0474768a3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -0,0 +1,749 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_vm_madvise.h" + +#include +#include + +#include "xe_bo.h" +#include "xe_pat.h" +#include "xe_pt.h" +#include "xe_svm.h" +#include "xe_tlb_inval.h" +#include "xe_vm.h" + +struct xe_vmas_in_madvise_range { + u64 addr; + u64 range; + struct xe_vma **vmas; + int num_vmas; + bool has_bo_vmas; + bool has_svm_userptr_vmas; +}; + +/** + * struct xe_madvise_details - Argument to madvise_funcs + * @dpagemap: Reference-counted pointer to a struct drm_pagemap. + * @has_purged_bo: Track if any BO was purged (for purgeable state) + * @retained_ptr: User pointer for retained value (for purgeable state) + * + * The madvise IOCTL handler may, in addition to the user-space + * args, have additional info to pass into the madvise_func that + * handles the madvise type. Use a struct_xe_madvise_details + * for that and extend the struct as necessary. + */ +struct xe_madvise_details { + struct drm_pagemap *dpagemap; + bool has_purged_bo; + u64 retained_ptr; +}; + +static int get_vmas(struct xe_vm *vm, struct xe_vmas_in_madvise_range *madvise_range) +{ + u64 addr = madvise_range->addr; + u64 range = madvise_range->range; + + struct xe_vma **__vmas; + struct drm_gpuva *gpuva; + int max_vmas = 8; + + lockdep_assert_held(&vm->lock); + + madvise_range->num_vmas = 0; + madvise_range->vmas = kmalloc_objs(*madvise_range->vmas, max_vmas); + if (!madvise_range->vmas) + return -ENOMEM; + + vm_dbg(&vm->xe->drm, "VMA's in range: start=0x%016llx, end=0x%016llx", addr, addr + range); + + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, addr, addr + range) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + if (xe_vma_bo(vma)) + madvise_range->has_bo_vmas = true; + else if (xe_vma_is_cpu_addr_mirror(vma) || xe_vma_is_userptr(vma)) + madvise_range->has_svm_userptr_vmas = true; + + if (madvise_range->num_vmas == max_vmas) { + max_vmas <<= 1; + __vmas = krealloc(madvise_range->vmas, + max_vmas * sizeof(*madvise_range->vmas), + GFP_KERNEL); + if (!__vmas) { + kfree(madvise_range->vmas); + return -ENOMEM; + } + madvise_range->vmas = __vmas; + } + + madvise_range->vmas[madvise_range->num_vmas] = vma; + (madvise_range->num_vmas)++; + } + + if (!madvise_range->num_vmas) + kfree(madvise_range->vmas); + + vm_dbg(&vm->xe->drm, "madvise_range-num_vmas = %d\n", madvise_range->num_vmas); + + return 0; +} + +static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op, + struct xe_madvise_details *details) +{ + int i; + + xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC); + + for (i = 0; i < num_vmas; i++) { + struct xe_vma *vma = vmas[i]; + struct xe_vma_preferred_loc *loc = &vma->attr.preferred_loc; + + /*TODO: Extend attributes to bo based vmas */ + if ((loc->devmem_fd == op->preferred_mem_loc.devmem_fd && + loc->migration_policy == op->preferred_mem_loc.migration_policy) || + !xe_vma_is_cpu_addr_mirror(vma)) { + vma->skip_invalidation = true; + } else { + vma->skip_invalidation = false; + loc->devmem_fd = op->preferred_mem_loc.devmem_fd; + /* Till multi-device support is not added migration_policy + * is of no use and can be ignored. + */ + loc->migration_policy = op->preferred_mem_loc.migration_policy; + drm_pagemap_put(loc->dpagemap); + loc->dpagemap = NULL; + if (details->dpagemap) + loc->dpagemap = drm_pagemap_get(details->dpagemap); + } + } +} + +static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op, + struct xe_madvise_details *details) +{ + struct xe_bo *bo; + int i; + + xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC); + xe_assert(vm->xe, op->atomic.val <= DRM_XE_ATOMIC_CPU); + + for (i = 0; i < num_vmas; i++) { + if (xe_vma_is_userptr(vmas[i]) && + !(op->atomic.val == DRM_XE_ATOMIC_DEVICE && + xe->info.has_device_atomics_on_smem)) { + vmas[i]->skip_invalidation = true; + continue; + } + + if (vmas[i]->attr.atomic_access == op->atomic.val) { + vmas[i]->skip_invalidation = true; + } else { + vmas[i]->skip_invalidation = false; + vmas[i]->attr.atomic_access = op->atomic.val; + } + + bo = xe_vma_bo(vmas[i]); + if (!bo || bo->attr.atomic_access == op->atomic.val) + continue; + + vmas[i]->skip_invalidation = false; + xe_bo_assert_held(bo); + bo->attr.atomic_access = op->atomic.val; + + /* Invalidate cpu page table, so bo can migrate to smem in next access */ + if (xe_bo_is_vram(bo) && + (bo->attr.atomic_access == DRM_XE_ATOMIC_CPU || + bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL)) + ttm_bo_unmap_virtual(&bo->ttm); + } +} + +static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op, + struct xe_madvise_details *details) +{ + int i; + + xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PAT); + + for (i = 0; i < num_vmas; i++) { + if (vmas[i]->attr.pat_index == op->pat_index.val) { + vmas[i]->skip_invalidation = true; + } else { + vmas[i]->skip_invalidation = false; + vmas[i]->attr.pat_index = op->pat_index.val; + } + } +} + +/** + * madvise_purgeable - Handle purgeable buffer object advice + * @xe: XE device + * @vm: VM + * @vmas: Array of VMAs + * @num_vmas: Number of VMAs + * @op: Madvise operation + * @details: Madvise details for return values + * + * Handles DONTNEED/WILLNEED/PURGED states. Tracks if any BO was purged + * in details->has_purged_bo for later copy to userspace. + */ +static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op, + struct xe_madvise_details *details) +{ + int i; + + xe_assert(vm->xe, op->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE); + + for (i = 0; i < num_vmas; i++) { + struct xe_bo *bo = xe_vma_bo(vmas[i]); + + if (!bo) { + /* Purgeable state applies to BOs only, skip non-BO VMAs */ + vmas[i]->skip_invalidation = true; + continue; + } + + /* BO must be locked before modifying madv state */ + xe_bo_assert_held(bo); + + /* + * Once purged, always purged. Cannot transition back to WILLNEED. + * This matches i915 semantics where purged BOs are permanently invalid. + */ + if (xe_bo_is_purged(bo)) { + details->has_purged_bo = true; + vmas[i]->skip_invalidation = true; + continue; + } + + switch (op->purge_state_val.val) { + case DRM_XE_VMA_PURGEABLE_STATE_WILLNEED: + vmas[i]->skip_invalidation = true; + /* Only act on a real DONTNEED -> WILLNEED transition. */ + if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_DONTNEED) { + vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED; + xe_bo_willneed_get_locked(bo); + } + break; + case DRM_XE_VMA_PURGEABLE_STATE_DONTNEED: + /* + * Don't zap PTEs at DONTNEED time -- pages are still + * alive. The zap happens in xe_bo_move_notify() right + * before the shrinker frees them. + */ + vmas[i]->skip_invalidation = true; + + /* Only act on a real WILLNEED -> DONTNEED transition. */ + if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { + vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED; + xe_bo_willneed_put_locked(bo); + } + break; + default: + /* Should never hit - values validated in madvise_args_are_sane() */ + xe_assert(vm->xe, 0); + return; + } + } +} + +typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op, + struct xe_madvise_details *details); + +static const madvise_func madvise_funcs[] = { + [DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC] = madvise_preferred_mem_loc, + [DRM_XE_MEM_RANGE_ATTR_ATOMIC] = madvise_atomic, + [DRM_XE_MEM_RANGE_ATTR_PAT] = madvise_pat_index, + [DRM_XE_VMA_ATTR_PURGEABLE_STATE] = madvise_purgeable, +}; + +static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpuva *gpuva; + struct xe_tile *tile; + u8 id, tile_mask = 0; + + lockdep_assert_held_write(&vm->lock); + + /* Wait for pending binds */ + if (dma_resv_wait_timeout(xe_vm_resv(vm), DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT) <= 0) + XE_WARN_ON(1); + + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + if (vma->skip_invalidation || xe_vma_is_null(vma)) + continue; + + if (xe_vma_is_cpu_addr_mirror(vma)) { + tile_mask |= xe_svm_ranges_zap_ptes_in_range(vm, + xe_vma_start(vma), + xe_vma_end(vma)); + } else { + for_each_tile(tile, vm->xe, id) { + if (xe_pt_zap_ptes(tile, vma)) { + tile_mask |= BIT(id); + + /* + * WRITE_ONCE pairs with READ_ONCE + * in xe_vm_has_valid_gpu_mapping() + */ + WRITE_ONCE(vma->tile_invalidated, + vma->tile_invalidated | BIT(id)); + } + } + } + } + + return tile_mask; +} + +static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end) +{ + u8 tile_mask = xe_zap_ptes_in_madvise_range(vm, start, end); + struct xe_tlb_inval_batch batch; + int err; + + if (!tile_mask) + return 0; + + xe_device_wmb(vm->xe); + + err = xe_tlb_inval_range_tilemask_submit(vm->xe, vm->usm.asid, start, end, + tile_mask, &batch); + if (!err) + xe_tlb_inval_batch_wait(&batch); + + return err; +} + +/** + * madvise_range_needs_invalidation() - Check whether madvise needs invalidation + * @args: madvise ioctl arguments + * + * Purgeable state updates only touch VMA/BO metadata. PTEs stay valid and are + * zapped only if the BO is later purged. + * + * Return: true when the update needs PTE invalidation. + */ +static bool madvise_range_needs_invalidation(const struct drm_xe_madvise *args) +{ + return args->type != DRM_XE_VMA_ATTR_PURGEABLE_STATE; +} + +static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args) +{ + if (XE_IOCTL_DBG(xe, !args)) + return false; + + if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->start, SZ_4K))) + return false; + + if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->range, SZ_4K))) + return false; + + if (XE_IOCTL_DBG(xe, args->range < SZ_4K)) + return false; + + switch (args->type) { + case DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: + { + s32 fd = (s32)args->preferred_mem_loc.devmem_fd; + + if (XE_IOCTL_DBG(xe, fd < DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM)) + return false; + + if (XE_IOCTL_DBG(xe, fd <= DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && + args->preferred_mem_loc.region_instance != 0)) + return false; + + if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy > + DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES)) + return false; + + if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.reserved)) + return false; + break; + } + case DRM_XE_MEM_RANGE_ATTR_ATOMIC: + if (XE_IOCTL_DBG(xe, args->atomic.val > DRM_XE_ATOMIC_CPU)) + return false; + + if (XE_IOCTL_DBG(xe, args->atomic.pad)) + return false; + + if (XE_IOCTL_DBG(xe, args->atomic.reserved)) + return false; + + break; + case DRM_XE_MEM_RANGE_ATTR_PAT: + { + u16 pat_index, coh_mode; + + if (XE_IOCTL_DBG(xe, args->pat_index.val >= xe->pat.n_entries)) + return false; + + pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + if (XE_IOCTL_DBG(xe, !coh_mode)) + return false; + + if (XE_WARN_ON(coh_mode > XE_COH_2WAY)) + return false; + + if (XE_IOCTL_DBG(xe, args->pat_index.pad)) + return false; + + if (XE_IOCTL_DBG(xe, args->pat_index.reserved)) + return false; + break; + } + case DRM_XE_VMA_ATTR_PURGEABLE_STATE: + { + u32 val = args->purge_state_val.val; + + if (XE_IOCTL_DBG(xe, !(val == DRM_XE_VMA_PURGEABLE_STATE_WILLNEED || + val == DRM_XE_VMA_PURGEABLE_STATE_DONTNEED))) + return false; + + if (XE_IOCTL_DBG(xe, args->purge_state_val.pad)) + return false; + + break; + } + default: + if (XE_IOCTL_DBG(xe, 1)) + return false; + } + + if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return false; + + return true; +} + +static int xe_madvise_details_init(struct xe_vm *vm, const struct drm_xe_madvise *args, + struct xe_madvise_details *details) +{ + struct xe_device *xe = vm->xe; + + memset(details, 0, sizeof(*details)); + + /* Store retained pointer for purgeable state */ + if (args->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE) { + details->retained_ptr = args->purge_state_val.retained_ptr; + return 0; + } + + if (args->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC) { + int fd = args->preferred_mem_loc.devmem_fd; + struct drm_pagemap *dpagemap; + + if (fd <= 0) + return 0; + + dpagemap = xe_drm_pagemap_from_fd(args->preferred_mem_loc.devmem_fd, + args->preferred_mem_loc.region_instance); + if (XE_IOCTL_DBG(xe, IS_ERR(dpagemap))) + return PTR_ERR(dpagemap); + + /* Don't allow a foreign placement without a fast interconnect! */ + if (XE_IOCTL_DBG(xe, dpagemap->pagemap->owner != vm->svm.peer.owner)) { + drm_pagemap_put(dpagemap); + return -ENOLINK; + } + details->dpagemap = dpagemap; + } + + return 0; +} + +static void xe_madvise_details_fini(struct xe_madvise_details *details) +{ + drm_pagemap_put(details->dpagemap); +} + +static int xe_madvise_purgeable_retained_to_user(const struct xe_madvise_details *details) +{ + u32 retained; + + if (!details->retained_ptr) + return 0; + + retained = !details->has_purged_bo; + + if (put_user(retained, (u32 __user *)u64_to_user_ptr(details->retained_ptr))) + return -EFAULT; + + return 0; +} + +static bool check_pat_args_are_sane(struct xe_device *xe, + struct xe_vmas_in_madvise_range *madvise_range, + u16 pat_index) +{ + u16 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + int i; + + /* + * Using coh_none with CPU cached buffers is not allowed on iGPU. + * On iGPU the GPU shares the LLC with the CPU, so with coh_none + * the GPU bypasses CPU caches and reads directly from DRAM, + * potentially seeing stale sensitive data from previously freed + * pages. On dGPU this restriction does not apply, because the + * platform does not provide a non-coherent system memory access + * path that would violate the DMA coherency contract. + */ + if (coh_mode != XE_COH_NONE || IS_DGFX(xe)) + return true; + + for (i = 0; i < madvise_range->num_vmas; i++) { + struct xe_vma *vma = madvise_range->vmas[i]; + struct xe_bo *bo = xe_vma_bo(vma); + + if (bo) { + /* BO with WB caching + COH_NONE is not allowed */ + if (XE_IOCTL_DBG(xe, bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) + return false; + /* Imported dma-buf without caching info, assume cached */ + if (XE_IOCTL_DBG(xe, !bo->cpu_caching)) + return false; + } else if (XE_IOCTL_DBG(xe, xe_vma_is_cpu_addr_mirror(vma) || + xe_vma_is_userptr(vma))) + /* System memory (userptr/SVM) is always CPU cached */ + return false; + } + + return true; +} + +static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas, + int num_vmas, u32 atomic_val) +{ + struct xe_device *xe = vm->xe; + struct xe_bo *bo; + int i; + + for (i = 0; i < num_vmas; i++) { + bo = xe_vma_bo(vmas[i]); + if (!bo) + continue; + /* + * NOTE: The following atomic checks are platform-specific. For example, + * if a device supports CXL atomics, these may not be necessary or + * may behave differently. + */ + if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_CPU && + !(bo->flags & XE_BO_FLAG_SYSTEM))) + return false; + + if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_DEVICE && + !(bo->flags & XE_BO_FLAG_VRAM0) && + !(bo->flags & XE_BO_FLAG_VRAM1) && + !(bo->flags & XE_BO_FLAG_SYSTEM && + xe->info.has_device_atomics_on_smem))) + return false; + + if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_GLOBAL && + (!(bo->flags & XE_BO_FLAG_SYSTEM) || + (!(bo->flags & XE_BO_FLAG_VRAM0) && + !(bo->flags & XE_BO_FLAG_VRAM1))))) + return false; + } + return true; +} +/** + * xe_vm_madvise_ioctl - Handle MADVise ioctl for a VM + * @dev: DRM device pointer + * @data: Pointer to ioctl data (drm_xe_madvise*) + * @file: DRM file pointer + * + * Handles the MADVISE ioctl to provide memory advice for vma's within + * input range. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_madvise *args = data; + struct xe_vmas_in_madvise_range madvise_range = { + /* + * Userspace may pass canonical (sign-extended) addresses. + * Strip the sign extension to get the internal non-canonical + * form used by the GPUVM, matching xe_vm_bind_ioctl() behavior. + */ + .addr = xe_device_uncanonicalize_addr(xe, args->start), + .range = args->range, + }; + struct xe_madvise_details details; + u16 pat_index, coh_mode; + struct xe_vm *vm; + struct drm_exec exec; + int err, attr_type; + bool do_retained; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -EINVAL; + + if (!madvise_args_are_sane(vm->xe, args)) { + err = -EINVAL; + goto put_vm; + } + + /* Cache whether we need to write retained, and validate it's initialized to 0 */ + do_retained = args->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE && + args->purge_state_val.retained_ptr; + if (do_retained) { + u32 retained; + u32 __user *retained_ptr; + + retained_ptr = u64_to_user_ptr(args->purge_state_val.retained_ptr); + if (get_user(retained, retained_ptr)) { + err = -EFAULT; + goto put_vm; + } + + if (XE_IOCTL_DBG(xe, retained != 0)) { + err = -EINVAL; + goto put_vm; + } + } + + xe_svm_flush(vm); + + err = down_write_killable(&vm->lock); + if (err) + goto put_vm; + + if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { + err = -ENOENT; + goto unlock_vm; + } + + err = xe_madvise_details_init(vm, args, &details); + if (err) + goto unlock_vm; + + err = xe_vm_alloc_madvise_vma(vm, madvise_range.addr, args->range); + if (err) + goto madv_fini; + + err = get_vmas(vm, &madvise_range); + if (err || !madvise_range.num_vmas) + goto madv_fini; + + if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { + pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + if (XE_IOCTL_DBG(xe, madvise_range.has_svm_userptr_vmas && + xe_device_is_l2_flush_optimized(xe) && + (pat_index != 19 && coh_mode != XE_COH_2WAY))) { + err = -EINVAL; + goto free_vmas; + } + } + + if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { + if (!check_pat_args_are_sane(xe, &madvise_range, + args->pat_index.val)) { + err = -EINVAL; + goto free_vmas; + } + } + + if (madvise_range.has_bo_vmas) { + if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) { + if (!check_bo_args_are_sane(vm, madvise_range.vmas, + madvise_range.num_vmas, + args->atomic.val)) { + err = -EINVAL; + goto free_vmas; + } + } + + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT, 0); + drm_exec_until_all_locked(&exec) { + for (int i = 0; i < madvise_range.num_vmas; i++) { + struct xe_bo *bo = xe_vma_bo(madvise_range.vmas[i]); + + if (!bo) + continue; + + if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { + if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && + xe_device_is_l2_flush_optimized(xe) && + (pat_index != 19 && + coh_mode != XE_COH_2WAY))) { + err = -EINVAL; + goto err_fini; + } + } + + err = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + if (err) + goto err_fini; + } + } + } + + if (madvise_range.has_svm_userptr_vmas) { + err = xe_svm_notifier_lock_interruptible(vm); + if (err) + goto err_fini; + } + + attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs)); + + /* Ensure the madvise function exists for this type */ + if (!madvise_funcs[attr_type]) { + err = -EINVAL; + goto err_fini; + } + + madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args, + &details); + + if (madvise_range_needs_invalidation(args)) + err = xe_vm_invalidate_madvise_range(vm, madvise_range.addr, + madvise_range.addr + args->range); + + if (madvise_range.has_svm_userptr_vmas) + xe_svm_notifier_unlock(vm); + +err_fini: + if (madvise_range.has_bo_vmas) + drm_exec_fini(&exec); +free_vmas: + kfree(madvise_range.vmas); + madvise_range.vmas = NULL; +madv_fini: + xe_madvise_details_fini(&details); +unlock_vm: + up_write(&vm->lock); + + /* Write retained value to user after releasing all locks */ + if (!err && do_retained) + err = xe_madvise_purgeable_retained_to_user(&details); +put_vm: + xe_vm_put(vm); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h new file mode 100644 index 000000000..a3078f634 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_madvise.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_VM_MADVISE_H_ +#define _XE_VM_MADVISE_H_ + +struct drm_device; +struct drm_file; +struct xe_bo; + +int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h new file mode 100644 index 000000000..635ed29b9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -0,0 +1,556 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_VM_TYPES_H_ +#define _XE_VM_TYPES_H_ + +#include +#include +#include + +#include +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_pt_types.h" +#include "xe_range_fence.h" +#include "xe_tlb_inval_types.h" +#include "xe_userptr.h" + +struct drm_pagemap; + +struct xe_bo; +struct xe_pagefault; +struct xe_svm_range; +struct xe_sync_entry; +struct xe_user_fence; +struct xe_vm; +struct xe_vm_pgtable_update_op; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define TEST_VM_OPS_ERROR +#define FORCE_OP_ERROR BIT(31) + +#define FORCE_OP_ERROR_LOCK 0 +#define FORCE_OP_ERROR_PREPARE 1 +#define FORCE_OP_ERROR_RUN 2 +#define FORCE_OP_ERROR_COUNT 3 +#endif + +#define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS +#define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1) +#define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2) +#define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3) +#define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4) +#define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5) +#define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) +#define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) +#define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) +#define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9) +#define XE_VMA_MADV_AUTORESET (DRM_GPUVA_USERBITS << 10) + +/** + * struct xe_vma_mem_attr - memory attributes associated with vma + */ +struct xe_vma_mem_attr { + /** @preferred_loc: preferred memory_location */ + struct xe_vma_preferred_loc { + /** @preferred_loc.migration_policy: Pages migration policy */ + u32 migration_policy; + + /** + * @preferred_loc.devmem_fd: used for determining pagemap_fd + * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and + * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or + * closest device memory respectively. + */ + u32 devmem_fd; + /** + * @preferred_loc.dpagemap: Reference-counted pointer to the drm_pagemap preferred + * for migration on a SVM page-fault. The pointer is protected by the + * vm lock, and is %NULL if @devmem_fd should be consulted for special + * values. + */ + struct drm_pagemap *dpagemap; + } preferred_loc; + + /** + * @atomic_access: The atomic access type for the vma + * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE, + * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible + * values. These are defined in uapi/drm/xe_drm.h. + */ + u32 atomic_access; + + /** + * @default_pat_index: The pat index for VMA set during first bind by user. + */ + u16 default_pat_index; + + /** + * @pat_index: The pat index to use when encoding the PTEs for this vma. + * same as default_pat_index unless overwritten by madvise. + */ + u16 pat_index; + + /** + * @purgeable_state: Purgeable hint for this VMA mapping + * + * Per-VMA purgeable state from madvise. Valid states are WILLNEED (0) + * or DONTNEED (1). Shared BOs require all VMAs to be DONTNEED before + * the BO can be purged. PURGED state exists only at BO level. + * + * Protected by BO dma-resv lock. Set via DRM_IOCTL_XE_MADVISE. + */ + u32 purgeable_state; +}; + +struct xe_vma { + /** @gpuva: Base GPUVA object */ + struct drm_gpuva gpuva; + + /** + * @combined_links: links into lists which are mutually exclusive. + * Locking: vm lock in write mode OR vm lock in read mode and the vm's + * resv. + */ + union { + /** @rebind: link into VM if this VMA needs rebinding. */ + struct list_head rebind; + /** @destroy: link to contested list when VM is being closed. */ + struct list_head destroy; + } combined_links; + + union { + /** @destroy_cb: callback to destroy VMA when unbind job is done */ + struct dma_fence_cb destroy_cb; + /** @destroy_work: worker to destroy this BO */ + struct work_struct destroy_work; + }; + + /** + * @tile_invalidated: Tile mask of binding are invalidated for this VMA. + * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in + * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and + * the vm->resv. For stable reading, BO's resv or userptr + * vm->svm.gpusvm.notifier_lock in read mode is required. Can be + * opportunistically read with READ_ONCE outside of locks. + */ + u8 tile_invalidated; + + /** @tile_mask: Tile mask of where to create binding for this VMA */ + u8 tile_mask; + + /** + * @tile_present: Tile mask of binding are present for this VMA. + * protected by vm->lock, vm->resv and for userptrs, + * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading, + * but if reading is done under the vm->lock only, it needs to be held + * in write mode. + */ + u8 tile_present; + + /** @tile_staged: bind is staged for this VMA */ + u8 tile_staged; + + /** + * @skip_invalidation: Used in madvise to avoid invalidation + * if mem attributes doesn't change + */ + bool skip_invalidation; + + /** + * @ufence: The user fence that was provided with MAP. + * Needs to be signalled before UNMAP can be processed. + */ + struct xe_user_fence *ufence; + + /** + * @attr: The attributes of vma which determines the migration policy + * and encoding of the PTEs for this vma. + */ + struct xe_vma_mem_attr attr; +}; + +/** + * struct xe_userptr_vma - A userptr vma subclass + * @vma: The vma. + * @userptr: Additional userptr information. + */ +struct xe_userptr_vma { + struct xe_vma vma; + struct xe_userptr userptr; +}; + +struct xe_device; + +/** + * struct xe_vm_fault_entry - Elements of vm->faults.list + * @list: link into @xe_vm.faults.list + * @address: address of the fault + * @address_precision: precision of faulted address + * @access_type: type of address access that resulted in fault + * @fault_type: type of fault reported + * @fault_level: fault level of the fault + */ +struct xe_vm_fault_entry { + struct list_head list; + u64 address; + u32 address_precision; + u8 access_type; + u8 fault_type; + u8 fault_level; +}; + +struct xe_vm { + /** @gpuvm: base GPUVM used to track VMAs */ + struct drm_gpuvm gpuvm; + + /** @svm: Shared virtual memory state */ + struct { + /** @svm.gpusvm: base GPUSVM used to track fault allocations */ + struct drm_gpusvm gpusvm; + /** + * @svm.garbage_collector: Garbage collector which is used unmap + * SVM range's GPU bindings and destroy the ranges. + */ + struct { + /** @svm.garbage_collector.lock: Protect's range list */ + spinlock_t lock; + /** + * @svm.garbage_collector.range_list: List of SVM ranges + * in the garbage collector. + */ + struct list_head range_list; + /** + * @svm.garbage_collector.work: Worker which the + * garbage collector runs on. + */ + struct work_struct work; + } garbage_collector; + struct xe_pagemap *pagemaps[XE_MAX_TILES_PER_DEVICE]; + /** @svm.peer: Used for pagemap connectivity computations. */ + struct drm_pagemap_peer peer; + } svm; + + struct xe_device *xe; + + /* exec queue used for (un)binding vma's */ + struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; + + /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ + struct ttm_lru_bulk_move lru_bulk_move; + + u64 size; + + struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE]; + struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL]; + + /** + * @flags: flags for this VM, statically setup a creation time aside + * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely + */ +#define XE_VM_FLAG_64K BIT(0) +#define XE_VM_FLAG_LR_MODE BIT(1) +#define XE_VM_FLAG_MIGRATION BIT(2) +#define XE_VM_FLAG_SCRATCH_PAGE BIT(3) +#define XE_VM_FLAG_FAULT_MODE BIT(4) +#define XE_VM_FLAG_BANNED BIT(5) +#define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) +#define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) +#define XE_VM_FLAG_GSC BIT(8) +#define XE_VM_FLAG_NO_VM_OVERCOMMIT BIT(9) + unsigned long flags; + + /** + * @lock: outer most lock, protects objects of anything attached to this + * VM + */ + struct rw_semaphore lock; + /** + * @snap_mutex: Mutex used to guard insertions and removals from gpuva, + * so we can take a snapshot safely from devcoredump. + */ + struct mutex snap_mutex; + + /** + * @rebind_list: list of VMAs that need rebinding. Protected by the + * vm->lock in write mode, OR (the vm->lock in read mode and the + * vm resv). + */ + struct list_head rebind_list; + + /** + * @destroy_work: worker to destroy VM, needed as a dma_fence signaling + * from an irq context can be last put and the destroy needs to be able + * to sleep. + */ + struct work_struct destroy_work; + + /** + * @rftree: range fence tree to track updates to page table structure. + * Used to implement conflict tracking between independent bind engines. + */ + struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; + + const struct xe_pt_ops *pt_ops; + + /** @userptr: user pointer state */ + struct xe_userptr_vm userptr; + + /** @preempt: preempt state */ + struct { + /** + * @min_run_period_ms: The minimum run period before preempting + * an engine again + */ + unsigned int min_run_period_ms; + /** @exec_queues: list of exec queues attached to this VM */ + struct list_head exec_queues; + /** @num_exec_queues: number exec queues attached to this VM */ + int num_exec_queues; + /** + * @rebind_deactivated: Whether rebind has been temporarily deactivated + * due to no work available. Protected by the vm resv. + */ + bool rebind_deactivated; + /** + * @rebind_work: worker to rebind invalidated userptrs / evicted + * BOs + */ + struct work_struct rebind_work; + /** + * @preempt.pm_activate_link: Link to list of rebind workers to be + * kicked on resume. + */ + struct list_head pm_activate_link; + } preempt; + + /** @exec_queues: Manages list of exec queues attached to this VM, protected by lock. */ + struct { + /** + * @exec_queues.list: list of exec queues attached to this VM, + * per GT + */ + struct list_head list[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; + /** + * @exec_queues.count: count of exec queues attached to this VM, + * per GT + */ + int count[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; + /** @exec_queues.lock: lock to protect exec_queues list */ + struct rw_semaphore lock; + } exec_queues; + + /** @um: unified memory state */ + struct { + /** @asid: address space ID, unique to each VM */ + u32 asid; + /** + * @last_fault_vma: Last fault VMA, used for fast lookup when we + * get a flood of faults to the same VMA + */ + struct xe_vma *last_fault_vma; + } usm; + + /** @error_capture: allow to track errors */ + struct { + /** @capture_once: capture only one error per VM */ + bool capture_once; + } error_capture; + + /** @faults: List of all faults associated with this VM */ + struct { + /** @faults.lock: lock protecting @faults.list */ + spinlock_t lock; + /** @faults.list: list of xe_vm_fault_entry entries */ + struct list_head list; + /** @faults.len: length of @faults.list */ + unsigned int len; + } faults; + + /** + * @validation: Validation data only valid with the vm resv held. + * Note: This is really task state of the task holding the vm resv, + * and moving forward we should + * come up with a better way of passing this down the call- + * chain. + */ + struct { + /** + * @validation.validating: The task that is currently making bos resident. + * for this vm. + * Protected by the VM's resv for writing. Opportunistic reading can be done + * using READ_ONCE. Note: This is a workaround for the + * TTM eviction_valuable() callback not being passed a struct + * ttm_operation_context(). Future work might want to address this. + */ + struct task_struct *validating; + /** + * @validation.exec The drm_exec context used when locking the vm resv. + * Protected by the vm's resv. + */ + struct drm_exec *_exec; + } validation; + + /** + * @tlb_flush_seqno: Required TLB flush seqno for the next exec. + * protected by the vm resv. + */ + u64 tlb_flush_seqno; + /** @batch_invalidate_tlb: Always invalidate TLB before batch start */ + bool batch_invalidate_tlb; + /** @xef: Xe file handle for tracking this VM's drm client */ + struct xe_file *xef; +}; + +/** struct xe_vma_op_map - VMA map operation */ +struct xe_vma_op_map { + /** @vma: VMA to map */ + struct xe_vma *vma; + /** @vma_flags: VMA flags for this operation */ + unsigned int vma_flags; + /** @immediate: Immediate bind */ + bool immediate; + /** @invalidate_on_bind: Invalidate on bind */ + bool invalidate_on_bind; + /** @request_decompress: schedule decompression for GPU map */ + bool request_decompress; + /** @pat_index: The pat index to use for this operation. */ + u16 pat_index; +}; + +/** struct xe_vma_op_remap - VMA remap operation */ +struct xe_vma_op_remap { + /** @prev: VMA preceding part of a split mapping */ + struct xe_vma *prev; + /** @next: VMA subsequent part of a split mapping */ + struct xe_vma *next; + /** @start: start of the VMA unmap */ + u64 start; + /** @range: range of the VMA unmap */ + u64 range; + /** @old_start: Original start of the VMA we unmap */ + u64 old_start; + /** @old_range: Original range of the VMA we unmap */ + u64 old_range; + /** @skip_prev: skip prev rebind */ + bool skip_prev; + /** @skip_next: skip next rebind */ + bool skip_next; + /** @unmap_done: unmap operation in done */ + bool unmap_done; +}; + +/** struct xe_vma_op_prefetch - VMA prefetch operation */ +struct xe_vma_op_prefetch { + /** @region: memory region to prefetch to */ + u32 region; +}; + +/** struct xe_vma_op_map_range - VMA map range operation */ +struct xe_vma_op_map_range { + /** @vma: VMA to map (system allocator VMA) */ + struct xe_vma *vma; + /** @range: SVM range to map */ + struct xe_svm_range *range; +}; + +/** struct xe_vma_op_unmap_range - VMA unmap range operation */ +struct xe_vma_op_unmap_range { + /** @range: SVM range to unmap */ + struct xe_svm_range *range; +}; + +/** struct xe_vma_op_prefetch_range - VMA prefetch range operation */ +struct xe_vma_op_prefetch_range { + /** @range: xarray for SVM ranges data */ + struct xarray range; + /** @ranges_count: number of svm ranges to map */ + u32 ranges_count; + /** + * @dpagemap: Pointer to the dpagemap structure containing memory to prefetch. + * NULL if prefetch requested region is smem + */ + struct drm_pagemap *dpagemap; +}; + +/** enum xe_vma_op_flags - flags for VMA operation */ +enum xe_vma_op_flags { + /** @XE_VMA_OP_COMMITTED: VMA operation committed */ + XE_VMA_OP_COMMITTED = BIT(0), + /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */ + XE_VMA_OP_PREV_COMMITTED = BIT(1), + /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */ + XE_VMA_OP_NEXT_COMMITTED = BIT(2), +}; + +/** enum xe_vma_subop - VMA sub-operation */ +enum xe_vma_subop { + /** @XE_VMA_SUBOP_MAP_RANGE: Map range */ + XE_VMA_SUBOP_MAP_RANGE, + /** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */ + XE_VMA_SUBOP_UNMAP_RANGE, +}; + +/** struct xe_vma_op - VMA operation */ +struct xe_vma_op { + /** @base: GPUVA base operation */ + struct drm_gpuva_op base; + /** @link: async operation link */ + struct list_head link; + /** @flags: operation flags */ + enum xe_vma_op_flags flags; + /** @subop: user defined sub-operation */ + enum xe_vma_subop subop; + /** @tile_mask: Tile mask for operation */ + u8 tile_mask; + + union { + /** @map: VMA map operation specific data */ + struct xe_vma_op_map map; + /** @remap: VMA remap operation specific data */ + struct xe_vma_op_remap remap; + /** @prefetch: VMA prefetch operation specific data */ + struct xe_vma_op_prefetch prefetch; + /** @map_range: VMA map range operation specific data */ + struct xe_vma_op_map_range map_range; + /** @unmap_range: VMA unmap range operation specific data */ + struct xe_vma_op_unmap_range unmap_range; + /** @prefetch_range: VMA prefetch range operation specific data */ + struct xe_vma_op_prefetch_range prefetch_range; + }; +}; + +/** struct xe_vma_ops - VMA operations */ +struct xe_vma_ops { + /** @list: list of VMA operations */ + struct list_head list; + /** @vm: VM */ + struct xe_vm *vm; + /** @q: exec queue for VMA operations */ + struct xe_exec_queue *q; + /** @syncs: syncs these operation */ + struct xe_sync_entry *syncs; + /** @num_syncs: number of syncs */ + u32 num_syncs; + /** @pt_update_ops: page table update operations */ + struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; + /** @flag: signify the properties within xe_vma_ops*/ +#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) +#define XE_VMA_OPS_FLAG_MADVISE BIT(1) +#define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2) +#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3) +#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP BIT(4) + u32 flags; +#ifdef TEST_VM_OPS_ERROR + /** @inject_error: inject error to test error handling */ + bool inject_error; +#endif +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c new file mode 100644 index 000000000..7b4478fb1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2024 Intel Corporation + */ + +#include +#include + +#include +#include + +#include "regs/xe_bars.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" +#include "xe_assert.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt_mcr.h" +#include "xe_mmio.h" +#include "xe_sriov.h" +#include "xe_tile_sriov_vf.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vram.h" +#include "xe_vram_types.h" + +static bool resource_is_valid(struct pci_dev *pdev, int bar) +{ + if (!pci_resource_flags(pdev, bar)) + return false; + + if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET) + return false; + + if (!pci_resource_len(pdev, bar)) + return false; + + return true; +} + +static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region *lmem_bar) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + if (!resource_is_valid(pdev, LMEM_BAR)) { + drm_err(&xe->drm, "pci resource is not valid\n"); + return -ENXIO; + } + + lmem_bar->io_start = pci_resource_start(pdev, LMEM_BAR); + lmem_bar->io_size = pci_resource_len(pdev, LMEM_BAR); + if (!lmem_bar->io_size) + return -EIO; + + /* XXX: Need to change when xe link code is ready */ + lmem_bar->dpa_base = 0; + + /* set up a map to the total memory area. */ + lmem_bar->mapping = devm_ioremap_wc(&pdev->dev, lmem_bar->io_start, lmem_bar->io_size); + + return 0; +} + +static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) +{ + struct xe_device *xe = gt_to_xe(gt); + u64 offset; + u32 reg; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) + return -ETIMEDOUT; + + if (GRAPHICS_VER(xe) >= 20) { + u64 ccs_size = tile_size / 512; + u64 offset_hi, offset_lo; + u32 nodes, num_enabled; + + reg = xe_mmio_read32(>->mmio, MIRROR_FUSE3); + nodes = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, reg); + num_enabled = hweight32(nodes); /* Number of enabled l3 nodes */ + + reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); + offset_lo = REG_FIELD_GET(XE2_FLAT_CCS_BASE_LOWER_ADDR_MASK, reg); + + reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_UPPER); + offset_hi = REG_FIELD_GET(XE2_FLAT_CCS_BASE_UPPER_ADDR_MASK, reg); + + offset = offset_hi << 32; /* HW view bits 39:32 */ + offset |= offset_lo << 6; /* HW view bits 31:6 */ + offset *= num_enabled; /* convert to SW view */ + + /* + * Everything below this offset is handed to the VRAM + * allocator, so it has to be the *first* address the + * compression hardware owns, rounded down. Rounding it up + * publishes CCS storage as free memory. + */ + offset = round_down(offset, SZ_4K); + + /* + * CCS storage must not run into GSM. The old check compared + * the offset against GSMBASE - ccs_size for equality, which + * could not fail: that value is 128K aligned, so it agreed + * with the rounded-up offset even when the base was not 128K + * aligned - exactly the case this fixes. + */ + xe_assert_msg(xe, offset + ccs_size <= + xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE), + "CCS overlaps GSM.\n"); + } else { + reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR); + offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K; + } + + *poffset = offset; + + return 0; +} + +/* + * tile_vram_size() - Collect vram size and offset information + * @tile: tile to get info for + * @vram_size: available vram (size - device reserved portions) + * @tile_size: actual vram size + * @tile_offset: physical start point in the vram address space + * + * There are 4 places for size information: + * - io size (from pci_resource_len of LMEM bar) (only used for small bar and DG1) + * - TILEx size (actual vram size) + * - GSMBASE offset (TILEx - "stolen") + * - CSSBASE offset (TILEx - CSS space necessary) + * + * CSSBASE is always a lower/smaller offset then GSMBASE. + * + * The actual available size of memory is to the CCS or GSM base. + * NOTE: multi-tile bases will include the tile offset. + * + */ +static int tile_vram_size(struct xe_tile *tile, u64 *vram_size, + u64 *tile_size, u64 *tile_offset) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_gt *gt = tile->primary_gt; + u64 offset; + u32 reg; + + if (IS_SRIOV_VF(xe)) { + struct xe_tile *t; + int id; + + offset = 0; + for_each_tile(t, xe, id) + for_each_if(t->id < tile->id) + offset += xe_tile_sriov_vf_lmem(t); + + *tile_size = xe_tile_sriov_vf_lmem(tile); + *vram_size = *tile_size; + *tile_offset = offset; + + return 0; + } + + /* actual size */ + if (unlikely(xe->info.platform == XE_DG1)) { + *tile_size = pci_resource_len(to_pci_dev(xe->drm.dev), LMEM_BAR); + *tile_offset = 0; + } else { + reg = xe_mmio_read32(&tile->mmio, SG_TILE_ADDR_RANGE(tile->id)); + *tile_size = (u64)REG_FIELD_GET(GENMASK(17, 8), reg) * SZ_1G; + *tile_offset = (u64)REG_FIELD_GET(GENMASK(7, 1), reg) * SZ_1G; + } + + /* minus device usage */ + if (xe->info.has_flat_ccs) { + int ret = get_flat_ccs_offset(gt, *tile_size, &offset); + + if (ret) + return ret; + } else { + offset = xe_mmio_read64_2x32(&tile->mmio, GSMBASE); + } + + /* remove the tile offset so we have just the available size */ + *vram_size = offset - *tile_offset; + + return 0; +} + +static void vram_fini(void *arg) +{ + struct xe_device *xe = arg; + struct xe_tile *tile; + int id; + + xe->mem.vram->mapping = NULL; + + for_each_tile(tile, xe, id) { + tile->mem.vram->mapping = NULL; + if (tile->mem.kernel_vram) + tile->mem.kernel_vram->mapping = NULL; + } +} + +struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement) +{ + struct xe_vram_region *vram; + struct drm_device *drm = &xe->drm; + + xe_assert(xe, id < xe->info.tile_count); + + vram = drmm_kzalloc(drm, sizeof(*vram), GFP_KERNEL); + if (!vram) + return NULL; + + vram->xe = xe; + vram->id = id; + vram->placement = placement; +#if defined(CONFIG_DRM_XE_PAGEMAP) + vram->migrate = xe->tiles[id].migrate; +#endif + return vram; +} + +static void print_vram_region_info(struct xe_device *xe, struct xe_vram_region *vram) +{ + struct drm_device *drm = &xe->drm; + + if (vram->io_size < vram->usable_size) + drm_info(drm, "Small BAR device\n"); + + drm_info(drm, + "VRAM[%u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", + vram->id, &vram->actual_physical_size, &vram->usable_size, &vram->io_size); + drm_info(drm, "VRAM[%u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", + vram->id, &vram->dpa_base, vram->dpa_base + (u64)vram->actual_physical_size, + &vram->io_start, vram->io_start + (u64)vram->io_size); +} + +static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, + struct xe_vram_region *lmem_bar, u64 offset, u64 usable_size, + u64 region_size, resource_size_t remain_io_size) +{ + /* Check if VRAM region is already initialized */ + if (vram->mapping) + return 0; + + vram->actual_physical_size = region_size; + vram->io_start = lmem_bar->io_start + offset; + vram->io_size = min_t(u64, usable_size, remain_io_size); + + if (!vram->io_size) { + drm_err(&xe->drm, "Tile without any CPU visible VRAM. Aborting.\n"); + return -ENODEV; + } + + vram->dpa_base = lmem_bar->dpa_base + offset; + vram->mapping = lmem_bar->mapping + offset; + vram->usable_size = usable_size; + + print_vram_region_info(xe, vram); + + return 0; +} + +/** + * xe_map_resource_to_region - Map ttm resource to vram memory region + * @res: The ttm resource + * + * Get vram memory region using vram memory manager managing this resource + * + * Returns: pointer to xe_vram_region + */ +struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res) +{ + struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); + struct ttm_resource_manager *mgr; + struct xe_ttm_vram_mgr *vram_mgr; + + xe_assert(xe, mem_type_is_vram(res->mem_type)); + mgr = ttm_manager_type(&xe->ttm, res->mem_type); + vram_mgr = to_xe_ttm_vram_mgr(mgr); + + return container_of(vram_mgr, struct xe_vram_region, ttm); +} + +/** + * xe_vram_probe() - Probe VRAM configuration + * @xe: the &xe_device + * + * Collect VRAM size and offset information for all tiles. + * + * Return: 0 on success, error code on failure + */ +int xe_vram_probe(struct xe_device *xe) +{ + struct xe_tile *tile; + struct xe_vram_region lmem_bar; + resource_size_t remain_io_size; + u64 available_size = 0; + u64 total_size = 0; + int err; + u8 id; + + if (!IS_DGFX(xe)) + return 0; + + err = determine_lmem_bar_size(xe, &lmem_bar); + if (err) + return err; + drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &lmem_bar.io_start, &lmem_bar.io_size); + + remain_io_size = lmem_bar.io_size; + + for_each_tile(tile, xe, id) { + u64 region_size; + u64 usable_size; + u64 tile_offset; + + err = tile_vram_size(tile, &usable_size, ®ion_size, &tile_offset); + if (err) + return err; + + total_size += region_size; + available_size += usable_size; + + err = vram_region_init(xe, tile->mem.vram, &lmem_bar, tile_offset, usable_size, + region_size, remain_io_size); + if (err) + return err; + + if (total_size > lmem_bar.io_size) { + drm_info(&xe->drm, "VRAM: %pa is larger than resource %pa\n", + &total_size, &lmem_bar.io_size); + } + + remain_io_size -= min_t(u64, tile->mem.vram->actual_physical_size, remain_io_size); + } + + err = vram_region_init(xe, xe->mem.vram, &lmem_bar, 0, available_size, total_size, + lmem_bar.io_size); + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, vram_fini, xe); +} + +/** + * xe_vram_region_io_start - Get the IO start of a VRAM region + * @vram: the VRAM region + * + * Return: the IO start of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram) +{ + return vram ? vram->io_start : 0; +} + +/** + * xe_vram_region_io_size - Get the IO size of a VRAM region + * @vram: the VRAM region + * + * Return: the IO size of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram) +{ + return vram ? vram->io_size : 0; +} + +/** + * xe_vram_region_dpa_base - Get the DPA base of a VRAM region + * @vram: the VRAM region + * + * Return: the DPA base of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram) +{ + return vram ? vram->dpa_base : 0; +} + +/** + * xe_vram_region_usable_size - Get the usable size of a VRAM region + * @vram: the VRAM region + * + * Return: the usable size of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram) +{ + return vram ? vram->usable_size : 0; +} + +/** + * xe_vram_region_actual_physical_size - Get the actual physical size of a VRAM region + * @vram: the VRAM region + * + * Return: the actual physical size of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram) +{ + return vram ? vram->actual_physical_size : 0; +} +EXPORT_SYMBOL_IF_KUNIT(xe_vram_region_actual_physical_size); diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h new file mode 100644 index 000000000..dd1c8bf17 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vram.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_VRAM_H_ +#define _XE_VRAM_H_ + +#include + +struct xe_device; +struct xe_vram_region; +struct ttm_resource; + +struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res); +int xe_vram_probe(struct xe_device *xe); + +struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement); + +resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram); + +#endif diff --git a/drivers/gpu/drm/xe/xe_vram_freq.c b/drivers/gpu/drm/xe/xe_vram_freq.c new file mode 100644 index 000000000..8717367cc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vram_freq.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ +#include +#include + +#include "xe_device_types.h" +#include "xe_pcode.h" +#include "xe_pcode_api.h" +#include "xe_tile_sysfs.h" +#include "xe_vram_freq.h" + +/** + * DOC: Xe VRAM freq + * + * Provides sysfs entries for vram frequency in tile + * + * device/tile#/memory/freq0/max_freq - This is maximum frequency. This value is read-only as it + * is the fixed fuse point P0. It is not the system + * configuration. + * device/tile#/memory/freq0/min_freq - This is minimum frequency. This value is read-only as it + * is the fixed fuse point PN. It is not the system + * configuration. + */ + +static struct xe_tile *dev_to_tile(struct device *dev) +{ + return kobj_to_tile(dev->kobj.parent); +} + +static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct xe_tile *tile = dev_to_tile(dev); + u32 val = 0, mbox; + int err; + + mbox = REG_FIELD_PREP(PCODE_MB_COMMAND, PCODE_FREQUENCY_CONFIG) + | REG_FIELD_PREP(PCODE_MB_PARAM1, PCODE_MBOX_FC_SC_READ_FUSED_P0) + | REG_FIELD_PREP(PCODE_MB_PARAM2, PCODE_MBOX_DOMAIN_HBM); + + err = xe_pcode_read(tile, mbox, &val, NULL); + if (err) + return err; + + /* data_out - Fused P0 for domain ID in units of 50 MHz */ + val *= 50; + + return sysfs_emit(buf, "%u\n", val); +} +static DEVICE_ATTR_RO(max_freq); + +static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct xe_tile *tile = dev_to_tile(dev); + u32 val = 0, mbox; + int err; + + mbox = REG_FIELD_PREP(PCODE_MB_COMMAND, PCODE_FREQUENCY_CONFIG) + | REG_FIELD_PREP(PCODE_MB_PARAM1, PCODE_MBOX_FC_SC_READ_FUSED_PN) + | REG_FIELD_PREP(PCODE_MB_PARAM2, PCODE_MBOX_DOMAIN_HBM); + + err = xe_pcode_read(tile, mbox, &val, NULL); + if (err) + return err; + + /* data_out - Fused Pn for domain ID in units of 50 MHz */ + val *= 50; + + return sysfs_emit(buf, "%u\n", val); +} +static DEVICE_ATTR_RO(min_freq); + +static struct attribute *freq_attrs[] = { + &dev_attr_max_freq.attr, + &dev_attr_min_freq.attr, + NULL +}; + +static const struct attribute_group freq_group_attrs = { + .name = "freq0", + .attrs = freq_attrs, +}; + +static void vram_freq_sysfs_fini(void *arg) +{ + struct kobject *kobj = arg; + + sysfs_remove_group(kobj, &freq_group_attrs); + kobject_put(kobj); +} + +/** + * xe_vram_freq_sysfs_init - Initialize vram frequency sysfs component + * @tile: Xe Tile object + * + * It needs to be initialized after the main tile component is ready + * + * Returns: 0 on success, negative error code on error. + */ +int xe_vram_freq_sysfs_init(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct kobject *kobj; + int err; + + if (xe->info.platform != XE_PVC) + return 0; + + kobj = kobject_create_and_add("memory", tile->sysfs); + if (!kobj) + return -ENOMEM; + + err = sysfs_create_group(kobj, &freq_group_attrs); + if (err) { + kobject_put(kobj); + return err; + } + + return devm_add_action_or_reset(xe->drm.dev, vram_freq_sysfs_fini, kobj); +} diff --git a/drivers/gpu/drm/xe/xe_vram_freq.h b/drivers/gpu/drm/xe/xe_vram_freq.h new file mode 100644 index 000000000..bf726bc58 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vram_freq.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_VRAM_FREQ_H_ +#define _XE_VRAM_FREQ_H_ + +struct xe_tile; + +int xe_vram_freq_sysfs_init(struct xe_tile *tile); + +#endif /* _XE_VRAM_FREQ_H_ */ diff --git a/drivers/gpu/drm/xe/xe_vram_types.h b/drivers/gpu/drm/xe/xe_vram_types.h new file mode 100644 index 000000000..646e3c12a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vram_types.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_VRAM_TYPES_H_ +#define _XE_VRAM_TYPES_H_ + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) +#include +#endif + +#include "xe_ttm_vram_mgr_types.h" + +struct xe_device; +struct xe_migrate; + +/** + * struct xe_vram_region - memory region structure + * This is used to describe a memory region in xe + * device, such as HBM memory or CXL extension memory. + */ +struct xe_vram_region { + /** @xe: Back pointer to xe device */ + struct xe_device *xe; + /** + * @id: VRAM region instance id + * + * The value should be unique for VRAM region. + */ + u8 id; + /** @io_start: IO start address of this VRAM instance */ + resource_size_t io_start; + /** + * @io_size: IO size of this VRAM instance + * + * This represents how much of this VRAM we can access + * via the CPU through the VRAM BAR. This can be smaller + * than @usable_size, in which case only part of VRAM is CPU + * accessible (typically the first 256M). This + * configuration is known as small-bar. + */ + resource_size_t io_size; + /** @dpa_base: This memory regions's DPA (device physical address) base */ + resource_size_t dpa_base; + /** + * @usable_size: usable size of VRAM + * + * Usable size of VRAM excluding reserved portions + * (e.g stolen mem) + */ + resource_size_t usable_size; + /** + * @actual_physical_size: Actual VRAM size + * + * Actual VRAM size including reserved portions + * (e.g stolen mem) + */ + resource_size_t actual_physical_size; + /** @mapping: pointer to VRAM mappable space */ + void __iomem *mapping; + /** @ttm: VRAM TTM manager */ + struct xe_ttm_vram_mgr ttm; + /** @placement: TTM placement dedicated for this region */ + u32 placement; +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + /** @migrate: Back pointer to migrate */ + struct xe_migrate *migrate; + /** @dpagemap_cache: drm_pagemap cache. */ + struct drm_pagemap_cache *dpagemap_cache; +#endif +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_vsec.c b/drivers/gpu/drm/xe/xe_vsec.c new file mode 100644 index 000000000..a9baf0bfe --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vsec.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright © 2024 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_device_types.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "xe_pm.h" +#include "xe_vsec.h" + +#include "regs/xe_pmt.h" + +/* PMT GUID value for BMG devices. NOTE: this is NOT a PCI id */ +#define BMG_DEVICE_ID 0xE2F8 + +static struct intel_vsec_header bmg_telemetry = { + .rev = 1, + .length = 0x10, + .id = VSEC_ID_TELEMETRY, + .num_entries = 2, + .entry_size = 4, + .tbir = 0, + .offset = BMG_DISCOVERY_OFFSET, +}; + +static struct intel_vsec_header bmg_crashlog = { + .rev = 1, + .length = 0x10, + .id = VSEC_ID_CRASHLOG, + .num_entries = 2, + .entry_size = 6, + .tbir = 0, + .offset = BMG_DISCOVERY_OFFSET + 0x60, +}; + +static struct intel_vsec_header *bmg_capabilities[] = { + &bmg_telemetry, + &bmg_crashlog, + NULL +}; + +enum xe_vsec { + XE_VSEC_UNKNOWN = 0, + XE_VSEC_BMG, +}; + +static struct intel_vsec_platform_info xe_vsec_info[] = { + [XE_VSEC_BMG] = { + .caps = VSEC_CAP_TELEMETRY | VSEC_CAP_CRASHLOG, + .headers = bmg_capabilities, + }, + { } +}; + +/* + * The GUID will have the following bits to decode: + * [0:3] - {Telemetry space iteration number (0,1,..)} + * [4:7] - Segment (SEGMENT_INDEPENDENT-0, Client-1, Server-2) + * [8:11] - SOC_SKU + * [12:27] – Device ID – changes for each down bin SKU’s + * [28:29] - Capability Type (Crashlog-0, Telemetry Aggregator-1, Watcher-2) + * [30:31] - Record-ID (0-PUNIT, 1-OOBMSM_0, 2-OOBMSM_1) + */ +#define GUID_TELEM_ITERATION GENMASK(3, 0) +#define GUID_SEGMENT GENMASK(7, 4) +#define GUID_SOC_SKU GENMASK(11, 8) +#define GUID_DEVICE_ID GENMASK(27, 12) +#define GUID_CAP_TYPE GENMASK(29, 28) +#define GUID_RECORD_ID GENMASK(31, 30) + +#define PUNIT_TELEMETRY_OFFSET 0x0200 +#define PUNIT_WATCHER_OFFSET 0x14A0 +#define OOBMSM_0_WATCHER_OFFSET 0x18D8 +#define OOBMSM_1_TELEMETRY_OFFSET 0x1000 + +enum record_id { + PUNIT, + OOBMSM_0, + OOBMSM_1, +}; + +enum capability { + CRASHLOG, + TELEMETRY, + WATCHER, +}; + +static int xe_guid_decode(u32 guid, int *index, u32 *offset) +{ + u32 record_id = FIELD_GET(GUID_RECORD_ID, guid); + u32 cap_type = FIELD_GET(GUID_CAP_TYPE, guid); + u32 device_id = FIELD_GET(GUID_DEVICE_ID, guid); + + if (device_id != BMG_DEVICE_ID) + return -ENODEV; + + if (cap_type > WATCHER) + return -EINVAL; + + *offset = 0; + + if (cap_type == CRASHLOG) { + *index = record_id == PUNIT ? 2 : 4; + return 0; + } + + switch (record_id) { + case PUNIT: + *index = 0; + if (cap_type == TELEMETRY) + *offset = PUNIT_TELEMETRY_OFFSET; + else + *offset = PUNIT_WATCHER_OFFSET; + break; + + case OOBMSM_0: + *index = 1; + if (cap_type == WATCHER) + *offset = OOBMSM_0_WATCHER_OFFSET; + break; + + case OOBMSM_1: + *index = 1; + if (cap_type == TELEMETRY) + *offset = OOBMSM_1_TELEMETRY_OFFSET; + break; + default: + return -EINVAL; + } + + return 0; +} + +int xe_pmt_telem_read(struct device *dev, u32 guid, u64 *data, loff_t user_offset, + u32 count) +{ + struct xe_device *xe = kdev_to_xe_device(dev); + void __iomem *telem_addr = xe->mmio.regs + BMG_TELEMETRY_OFFSET; + u32 mem_region; + u32 offset; + int ret; + + ret = xe_guid_decode(guid, &mem_region, &offset); + if (ret) + return ret; + + telem_addr += offset + user_offset; + + guard(mutex)(&xe->pmt.lock); + + if (!xe->soc_remapper.set_telem_region) + return -ENODEV; + + /* indicate that we are not at an appropriate power level */ + if (!xe_pm_runtime_get_if_active(xe)) + return -ENODATA; + + /* set SoC re-mapper index register based on GUID memory region */ + xe->soc_remapper.set_telem_region(xe, mem_region); + + memcpy_fromio(data, telem_addr, count); + xe_pm_runtime_put(xe); + + return count; +} + +static struct pmt_callbacks xe_pmt_cb = { + .read_telem = xe_pmt_telem_read, +}; + +static const int vsec_platforms[] = { + [XE_BATTLEMAGE] = XE_VSEC_BMG, +}; + +static enum xe_vsec get_platform_info(struct xe_device *xe) +{ + if (xe->info.platform > XE_BATTLEMAGE) + return XE_VSEC_UNKNOWN; + + return vsec_platforms[xe->info.platform]; +} + +/** + * xe_vsec_init - Initialize resources and add intel_vsec auxiliary + * interface + * @xe: valid xe instance + */ +void xe_vsec_init(struct xe_device *xe) +{ + struct intel_vsec_platform_info *info; + struct device *dev = xe->drm.dev; + enum xe_vsec platform; + + platform = get_platform_info(xe); + if (platform == XE_VSEC_UNKNOWN) + return; + + info = &xe_vsec_info[platform]; + if (!info->headers) + return; + + switch (platform) { + case XE_VSEC_BMG: + info->priv_data = &xe_pmt_cb; + break; + default: + break; + } + + /* + * Register a VSEC. Cleanup is handled using device managed + * resources. + */ + intel_vsec_register(dev, info); +} +MODULE_IMPORT_NS("INTEL_VSEC"); diff --git a/drivers/gpu/drm/xe/xe_vsec.h b/drivers/gpu/drm/xe/xe_vsec.h new file mode 100644 index 000000000..a25b4e6e6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vsec.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright © 2024 Intel Corporation */ + +#ifndef _XE_VSEC_H_ +#define _XE_VSEC_H_ + +#include + +struct device; +struct xe_device; + +void xe_vsec_init(struct xe_device *xe); +int xe_pmt_telem_read(struct device *dev, u32 guid, u64 *data, loff_t user_offset, u32 count); + +#endif diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c new file mode 100644 index 000000000..139434946 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -0,0 +1,1032 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_wa.h" + +#include +#include +#include +#include + +#include +#include + +#include "regs/xe_engine_regs.h" +#include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" +#include "regs/xe_regs.h" +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gt_types.h" +#include "xe_hw_engine_types.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "xe_rtp.h" +#include "xe_sriov.h" +#include "xe_step.h" + +/** + * DOC: Hardware workarounds + * + * Hardware workarounds are register programming documented to be executed in + * the driver that fall outside of the normal programming sequences for a + * platform. There are some basic categories of workarounds, depending on + * how/when they are applied: + * + * - LRC workarounds: workarounds that touch registers that are + * saved/restored to/from the HW context image. The list is emitted (via Load + * Register Immediate commands) once when initializing the device and saved in + * the default context. That default context is then used on every context + * creation to have a "primed golden context", i.e. a context image that + * already contains the changes needed to all the registers. See + * drivers/gpu/drm/xe/xe_lrc.c for default context handling. + * + * - Engine workarounds: the list of these WAs is applied whenever the specific + * engine is reset. It's also possible that a set of engine classes share a + * common power domain and they are reset together. This happens on some + * platforms with render and compute engines. In this case (at least) one of + * them need to keeep the workaround programming: the approach taken in the + * driver is to tie those workarounds to the first compute/render engine that + * is registered. When executing with GuC submission, engine resets are + * outside of kernel driver control, hence the list of registers involved is + * written once, on engine initialization, and then passed to GuC, that + * saves/restores their values before/after the reset takes place. See + * drivers/gpu/drm/xe/xe_guc_ads.c for reference. + * + * - GT workarounds: the list of these WAs is applied whenever these registers + * revert to their default values: on GPU reset, suspend/resume [1]_, etc. + * + * - Register whitelist: some workarounds need to be implemented in userspace, + * but need to touch privileged registers. The whitelist in the kernel + * instructs the hardware to allow the access to happen. From the kernel side, + * this is just a special case of a MMIO workaround (as we write the list of + * these to/be-whitelisted registers to some special HW registers). + * + * - Workaround batchbuffers: buffers that get executed automatically by the + * hardware on every HW context restore. These buffers are created and + * programmed in the default context so the hardware always go through those + * programming sequences when switching contexts. The support for workaround + * batchbuffers is enabled via these hardware mechanisms: + * + * #. INDIRECT_CTX (also known as **mid context restore bb**): A batchbuffer + * and an offset are provided in the default context, pointing the hardware + * to jump to that location when that offset is reached in the context + * restore. When a context is being restored, this is executed after the + * ring context, in the middle (or beginning) of the engine context image. + * + * #. BB_PER_CTX_PTR (also known as **post context restore bb**): A + * batchbuffer is provided in the default context, pointing the hardware to + * a buffer to continue executing after the engine registers are restored + * in a context restore sequence. + * + * Below is the timeline for a context restore sequence: + * + * .. code:: + * + * INDIRECT_CTX_OFFSET + * |----------->| + * .------------.------------.-------------.------------.--------------.-----------. + * |Ring | Engine | Mid-context | Engine | Post-context | Ring | + * |Restore | Restore (1)| BB Restore | Restore (2)| BB Restore | Execution | + * `------------'------------'-------------'------------'--------------'-----------' + * + * - Other/OOB: There are WAs that, due to their nature, cannot be applied from + * a central place. Those are peppered around the rest of the code, as needed. + * There's a central place to control which workarounds are enabled: + * drivers/gpu/drm/xe/xe_wa_oob.rules for GT workarounds and + * drivers/gpu/drm/xe/xe_device_wa_oob.rules for device/SoC workarounds. + * These files only record which workarounds are enabled: during early device + * initialization those rules are evaluated and recorded by the driver. Then + * later the driver checks with ``XE_GT_WA()`` and ``XE_DEVICE_WA()`` to + * implement them. + * + * .. [1] Technically, some registers are powercontext saved & restored, so they + * survive a suspend/resume. In practice, writing them again is not too + * costly and simplifies things, so it's the approach taken in the driver. + * + * .. note:: + * Hardware workarounds in xe work the same way as in i915, with the + * difference of how they are maintained in the code. In xe it uses the + * xe_rtp infrastructure so the workarounds can be kept in tables, following + * a more declarative approach rather than procedural. + * + * .. note:: + * When a workaround applies to every single known IP version in a range, + * the preferred handling is to use a single range-based RTP entry rather + * than individual entries for each version, even if some of the intermediate + * version numbers are currently unused. If a new intermediate IP version + * appears in the future and is enabled in the driver, any existing + * range-based entries that contain the new version number will need to be + * analyzed to determine whether their workarounds should apply to the new + * version, or whether any existing range based entries needs to be split + * into two entries that do not include the new intermediate version. + */ + +#undef XE_REG_MCR +#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) + +__diag_push(); +__diag_ignore_all("-Woverride-init", "Allow field overrides in table"); + +VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_was = XE_RTP_TABLE_SR( + /* Workarounds applying over a range of IPs */ + + { XE_RTP_NAME("14011060649"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1200, 1255), + ENGINE_CLASS(VIDEO_DECODE), + FUNC(xe_rtp_match_even_instance)), + XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + }, + { XE_RTP_NAME("14011059788"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + XE_RTP_ACTIONS(SET(DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE)) + }, + { XE_RTP_NAME("14015795083"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260)), + XE_RTP_ACTIONS(CLR(MISCCPCTL, DOP_CLOCK_GATE_RENDER_ENABLE)) + }, + { XE_RTP_NAME("16021867713"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, 3002), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + }, + { XE_RTP_NAME("14019449301"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1301, 2000), ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + }, + { XE_RTP_NAME("16028005424"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), OR, + MEDIA_VERSION_RANGE(1301, 3500)), + XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) + }, + { XE_RTP_NAME("14026578760"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, 3511), OR, + MEDIA_VERSION(3503)), + XE_RTP_ACTIONS(SET(GAMSTLB_CTRL, DIS_PEND_GPA_LINK)) + }, + + /* DG1 */ + + { XE_RTP_NAME("1409420604"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_ACTIONS(SET(SUBSLICE_UNIT_LEVEL_CLKGATE2, CPSSUNIT_CLKGATE_DIS)) + }, + { XE_RTP_NAME("1408615072"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_ACTIONS(SET(UNSLICE_UNIT_LEVEL_CLKGATE2, VSUNIT_CLKGATE2_DIS)) + }, + + /* DG2 */ + + { XE_RTP_NAME("22010523718"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10)), + XE_RTP_ACTIONS(SET(UNSLICE_UNIT_LEVEL_CLKGATE, CG3DDISCFEG_CLKGATE_DIS)) + }, + { XE_RTP_NAME("14011006942"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10)), + XE_RTP_ACTIONS(SET(SUBSLICE_UNIT_LEVEL_CLKGATE, DSS_ROUTER_CLKGATE_DIS)) + }, + { XE_RTP_NAME("14014830051"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(CLR(SARB_CHICKEN1, COMP_CKN_IN)) + }, + { XE_RTP_NAME("18018781329"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(RENDER_MOD_CTRL, FORCE_MISS_FTLB), + SET(COMP_MOD_CTRL, FORCE_MISS_FTLB), + SET(XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB), + SET(XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB)) + }, + { XE_RTP_NAME("1509235366"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(XEHP_GAMCNTRL_CTRL, + INVALIDATION_BROADCAST_MODE_DIS | + GLOBAL_INVALIDATION_MODE)) + }, + + /* PVC */ + + { XE_RTP_NAME("18018781329"), + XE_RTP_RULES(PLATFORM(PVC)), + XE_RTP_ACTIONS(SET(RENDER_MOD_CTRL, FORCE_MISS_FTLB), + SET(COMP_MOD_CTRL, FORCE_MISS_FTLB), + SET(XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB), + SET(XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB)) + }, + { XE_RTP_NAME("16016694945"), + XE_RTP_RULES(PLATFORM(PVC)), + XE_RTP_ACTIONS(SET(XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC)) + }, + + /* Xe_LPG */ + + { XE_RTP_NAME("14018575942"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)), + XE_RTP_ACTIONS(SET(COMP_MOD_CTRL, FORCE_MISS_FTLB)) + }, + { XE_RTP_NAME("22016670082"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)), + XE_RTP_ACTIONS(SET(SQCNT1, ENFORCE_RAR)) + }, + + /* Xe_LPM+ */ + + { XE_RTP_NAME("22016670082"), + XE_RTP_RULES(MEDIA_VERSION(1300)), + XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR)) + }, + + /* Xe2_LPM */ + + { XE_RTP_NAME("14017421178"), + XE_RTP_RULES(MEDIA_VERSION(2000), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + }, + + /* Xe2_HPG */ + + { XE_RTP_NAME("16025250150"), + XE_RTP_RULES(GRAPHICS_VERSION(2001)), + XE_RTP_ACTIONS(FIELD_SET(LSN_VC_REG2, + LSN_LNI_WGT_MASK | LSN_LNE_WGT_MASK | + LSN_DIM_X_WGT_MASK | LSN_DIM_Y_WGT_MASK | + LSN_DIM_Z_WGT_MASK, + LSN_LNI_WGT(1) | LSN_LNE_WGT(1) | + LSN_DIM_X_WGT(1) | LSN_DIM_Y_WGT(1) | + LSN_DIM_Z_WGT(1)), + SET(LSC_CHICKEN_BIT_0_UDW, L3_128B_256B_WRT_DIS)) + }, + + /* Xe3_LPG */ + + { XE_RTP_NAME("14021871409"), + XE_RTP_RULES(GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0)), + XE_RTP_ACTIONS(SET(UNSLCGCTL9454, LSCFE_CLKGATE_DIS)) + }, + + /* Xe3_LPM */ + + { XE_RTP_NAME("16021865536"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(3000, 3002), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + }, + { XE_RTP_NAME("14021486841"), + XE_RTP_RULES(MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)), + XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), + }, + + /* Xe3P_LPG */ + + { XE_RTP_NAME("14025160223"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)), + XE_RTP_ACTIONS(SET(MMIOATSREQLIMIT_GAM_WALK_3D, + DIS_ATS_WRONLY_PG)) + }, + { XE_RTP_NAME("14026144927, 16029437861, 14026127056"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)), + XE_RTP_ACTIONS(SET(L3SQCREG2, L3_SQ_DISABLE_COAMA_2WAY_COH | + L3_SQ_DISABLE_COAMA)) + }, + { XE_RTP_NAME("14025635424"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)), + XE_RTP_ACTIONS(SET(GAMSTLB_CTRL2, STLB_SINGLE_BANK_MODE)) + }, + { XE_RTP_NAME("16028005424"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)), + XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) + }, +); +EXPORT_SYMBOL_IF_KUNIT(gt_was); + +static const struct xe_rtp_table_sr engine_was = XE_RTP_TABLE_SR( + /* Workarounds applying over a range of IPs */ + + { XE_RTP_NAME("22010931296, 18011464164, 14010919138"), + XE_RTP_RULES(GRAPHICS_VERSION(1200), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(FF_THREAD_MODE(RENDER_RING_BASE), + FF_TESSELATION_DOP_GATE_DISABLE)) + }, + { XE_RTP_NAME("1409804808"), + XE_RTP_RULES(GRAPHICS_VERSION(1200), + ENGINE_CLASS(RENDER), + IS_INTEGRATED), + XE_RTP_ACTIONS(SET(ROW_CHICKEN2, PUSH_CONST_DEREF_HOLD_DIS)) + }, + { XE_RTP_NAME("14010229206, 1409085225"), + XE_RTP_RULES(GRAPHICS_VERSION(1200), + ENGINE_CLASS(RENDER), + IS_INTEGRATED), + XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH)) + }, + { XE_RTP_NAME("1606931601"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_EARLY_READ)) + }, + { XE_RTP_NAME("14010826681, 1606700617, 22010271021, 18019627453"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1255), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(CS_DEBUG_MODE1(RENDER_RING_BASE), + FF_DOP_CLOCK_GATE_DISABLE)) + }, + { XE_RTP_NAME("1406941453"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(SAMPLER_MODE, ENABLE_SMALLPL)) + }, + { XE_RTP_NAME("FtrPerCtxtPreemptionGranularityControl"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1250), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(FF_SLICE_CS_CHICKEN1(RENDER_RING_BASE), + FFSC_PERCTX_PREEMPT_CTRL)) + }, + { XE_RTP_NAME("18032247524"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE)) + }, + { XE_RTP_NAME("16018712365"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS)) + }, + { XE_RTP_NAME("14020338487"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS)) + }, + { XE_RTP_NAME("14018471104"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL)) + }, + /* + * Although this workaround isn't required for the RCS, disabling these + * reports has no impact for our driver or the GuC, so we go ahead and + * apply this to all engines for simplicity. + */ + { XE_RTP_NAME("16021639441"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), OR, + MEDIA_VERSION_RANGE(1301, 2000)), + XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), + GHWSP_CSB_REPORT_DIS | + PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("14021402888"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE)) + }, + { XE_RTP_NAME("13012615864"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS)) + }, + { XE_RTP_NAME("18041344222"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3000), + FUNC(xe_rtp_match_first_render_or_compute), + FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)), + XE_RTP_ACTIONS(SET(TDL_CHICKEN, EUSTALL_PERF_SAMPLING_DISABLE)) + }, + + /* TGL */ + + { XE_RTP_NAME("1607297627, 1607030317, 1607186500"), + XE_RTP_RULES(PLATFORM(TIGERLAKE), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(RING_PSMI_CTL(RENDER_RING_BASE), + WAIT_FOR_EVENT_POWER_DOWN_DISABLE | + RC_SEMA_IDLE_MSG_DISABLE)) + }, + + /* RKL */ + + { XE_RTP_NAME("1607297627, 1607030317, 1607186500"), + XE_RTP_RULES(PLATFORM(ROCKETLAKE), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(RING_PSMI_CTL(RENDER_RING_BASE), + WAIT_FOR_EVENT_POWER_DOWN_DISABLE | + RC_SEMA_IDLE_MSG_DISABLE)) + }, + + /* ADL-P */ + + { XE_RTP_NAME("1607297627, 1607030317, 1607186500"), + XE_RTP_RULES(PLATFORM(ALDERLAKE_P), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(RING_PSMI_CTL(RENDER_RING_BASE), + WAIT_FOR_EVENT_POWER_DOWN_DISABLE | + RC_SEMA_IDLE_MSG_DISABLE)) + }, + + /* DG2 */ + + { XE_RTP_NAME("22013037850"), + XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, + DISABLE_128B_EVICTION_COMMAND_UDW)) + }, + { XE_RTP_NAME("22014226127"), + XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE)) + }, + { XE_RTP_NAME("18017747507"), + XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(VFG_PREEMPTION_CHICKEN, + POLYGON_TRIFAN_LINELOOP_DISABLE)) + }, + { XE_RTP_NAME("22012826095, 22013059131"), + XE_RTP_RULES(SUBPLATFORM(DG2, G11), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(FIELD_SET(LSC_CHICKEN_BIT_0_UDW, + MAXREQS_PER_BANK, + REG_FIELD_PREP(MAXREQS_PER_BANK, 2))) + }, + { XE_RTP_NAME("22013059131"), + XE_RTP_RULES(SUBPLATFORM(DG2, G11), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, FORCE_1_SUB_MESSAGE_PER_FRAGMENT)) + }, + { XE_RTP_NAME("14015227452"), + XE_RTP_RULES(PLATFORM(DG2), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE)) + }, + { XE_RTP_NAME("18028616096"), + XE_RTP_RULES(PLATFORM(DG2), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, UGM_FRAGMENT_THRESHOLD_TO_3)) + }, + { XE_RTP_NAME("22015475538"), + XE_RTP_RULES(PLATFORM(DG2), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8)) + }, + { XE_RTP_NAME("22012654132"), + XE_RTP_RULES(SUBPLATFORM(DG2, G11), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(CACHE_MODE_SS, ENABLE_PREFETCH_INTO_IC, + /* + * Register can't be read back for verification on + * DG2 due to Wa_14012342262 + */ + .read_mask = 0)) + }, + { XE_RTP_NAME("1509727124"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(SAMPLER_MODE, SC_DISABLE_POWER_OPTIMIZATION_EBB)) + }, + { XE_RTP_NAME("22012856258"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_READ_SUPPRESSION)) + }, + { XE_RTP_NAME("22010960976, 14013347512"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(CLR(XEHP_HDC_CHICKEN0, + LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK)) + }, + { XE_RTP_NAME("14015150844"), + XE_RTP_RULES(PLATFORM(DG2), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(XEHP_HDC_CHICKEN0, DIS_ATOMIC_CHAINING_TYPED_WRITES, + XE_RTP_NOCHECK)) + }, + + /* PVC */ + + { XE_RTP_NAME("22014226127"), + XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE)) + }, + { XE_RTP_NAME("14015227452"), + XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE)) + }, + { XE_RTP_NAME("18020744125"), + XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute), + ENGINE_CLASS(COMPUTE)), + XE_RTP_ACTIONS(SET(RING_HWSTAM(RENDER_RING_BASE), ~0)) + }, + + /* Xe_LPG */ + + { XE_RTP_NAME("14017856879"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_FIX_EOT1_FLUSH)) + }, + { XE_RTP_NAME("14015150844"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(XEHP_HDC_CHICKEN0, DIS_ATOMIC_CHAINING_TYPED_WRITES, + XE_RTP_NOCHECK)) + }, + { XE_RTP_NAME("14020495402"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_TDL_SVHS_GATING)) + }, + + /* Xe2_LPG */ + + { XE_RTP_NAME("18034896535, 16021540221"), /* 16021540221: GRAPHICS_STEP(A0, B0) */ + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH)) + }, + { XE_RTP_NAME("16018610683"), + XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, SLM_WMTP_RESTORE)) + }, + + /* Xe2_HPG */ + + { XE_RTP_NAME("16018737384"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2999), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS)) + }, + { XE_RTP_NAME("14019811474"), + XE_RTP_RULES(GRAPHICS_VERSION(2001), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, WR_REQ_CHAINING_DIS)) + }, + { XE_RTP_NAME("14021821874, 14022954250"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, STK_ID_RESTRICT)) + }, + + /* Xe3_LPG */ + + { XE_RTP_NAME("18034896535"), + XE_RTP_RULES(GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH)) + }, + { XE_RTP_NAME("16024792527"), + XE_RTP_RULES(GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(FIELD_SET(SAMPLER_MODE, SMP_WAIT_FETCH_MERGING_COUNTER, + SMP_FORCE_128B_OVERFETCH)) + }, + { XE_RTP_NAME("14023061436"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_CHICKEN, QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE)) + }, + { XE_RTP_NAME("16023105232"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1301, 3000), OR, + GRAPHICS_VERSION_RANGE(2001, 3001)), + XE_RTP_ACTIONS(SET(RING_PSMI_CTL(0), RC_SEMA_IDLE_MSG_DISABLE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + + /* Xe3p_LPG*/ + + { XE_RTP_NAME("22021149932, 14026290593"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, SAMPLER_LD_LSC_DISABLE)) + }, + { XE_RTP_NAME("14025676848, 14026270459"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE)) + }, + { XE_RTP_NAME("16028951944"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN5, CPSS_AWARE_DIS)) + }, + + /* Xe3p_XPC */ + + { XE_RTP_NAME("14026999295"), + XE_RTP_RULES(GRAPHICS_VERSION(3511), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_EU_GRF_POISON_TO_LSC)) + }, + { XE_RTP_NAME("18044193044"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_CHICKEN, BIT_APQ_OPT_DIS)) + }, +); + +static const struct xe_rtp_table_sr lrc_was = XE_RTP_TABLE_SR( + { XE_RTP_NAME("16011163337"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + /* read verification is ignored due to 1608008084. */ + XE_RTP_ACTIONS(FIELD_SET_NO_READ_MASK(FF_MODE2, + FF_MODE2_GS_TIMER_MASK, + FF_MODE2_GS_TIMER_224)) + }, + { XE_RTP_NAME("1604555607"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + /* read verification is ignored due to 1608008084. */ + XE_RTP_ACTIONS(FIELD_SET_NO_READ_MASK(FF_MODE2, + FF_MODE2_TDS_TIMER_MASK, + FF_MODE2_TDS_TIMER_128)) + }, + { XE_RTP_NAME("1409342910, 14010698770, 14010443199, 1408979724, 1409178076, 1409207793, 1409217633, 1409252684, 1409347922, 1409142259"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN3, + DISABLE_CPS_AWARE_COLOR_PIPE)) + }, + { XE_RTP_NAME("WaDisableGPGPUMidThreadPreemption"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(RENDER_RING_BASE), + PREEMPT_GPGPU_LEVEL_MASK, + PREEMPT_GPGPU_THREAD_GROUP_LEVEL)) + }, + { XE_RTP_NAME("1806527549"), + XE_RTP_RULES(GRAPHICS_VERSION(1200)), + XE_RTP_ACTIONS(SET(HIZ_CHICKEN, HZ_DEPTH_TEST_LE_GE_OPT_DISABLE)) + }, + { XE_RTP_NAME("1606376872"), + XE_RTP_RULES(GRAPHICS_VERSION(1200)), + XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, DISABLE_TDC_LOAD_BALANCING_CALC)) + }, + { XE_RTP_NAME("14019877138"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) + }, + { XE_RTP_NAME("14019386621"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE)) + }, + { XE_RTP_NAME("14019988906"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) + }, + { XE_RTP_NAME("18033852989"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST)) + }, + { XE_RTP_NAME("15016589081"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX)) + }, + { XE_RTP_NAME("14026781792"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3510), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(FF_MODE, DIS_TE_PATCH_CTRL)) + }, + + /* DG1 */ + + { XE_RTP_NAME("1409044764"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_ACTIONS(CLR(COMMON_SLICE_CHICKEN3, + DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN)) + }, + { XE_RTP_NAME("22010493298"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_ACTIONS(SET(HIZ_CHICKEN, + DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE)) + }, + + /* DG2 */ + + { XE_RTP_NAME("16013271637"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(XEHP_SLICE_COMMON_ECO_CHICKEN1, + MSC_MSAA_REODER_BUF_BYPASS_DISABLE)) + }, + { XE_RTP_NAME("14014947963"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(FIELD_SET(VF_PREEMPTION, + PREEMPTION_VERTEX_COUNT, + 0x4000)) + }, + { XE_RTP_NAME("18018764978"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(XEHP_PSS_MODE2, + SCOREBOARD_STALL_FLUSH_CONTROL)) + }, + { XE_RTP_NAME("18019271663"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE)) + }, + + /* PVC */ + + { XE_RTP_NAME("16017236439"), + XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COPY), + FUNC(xe_rtp_match_even_instance)), + XE_RTP_ACTIONS(SET(BCS_SWCTRL(0), + BCS_SWCTRL_DISABLE_256B, + XE_RTP_ACTION_FLAG(ENGINE_BASE))), + }, + + /* Xe_LPG */ + + { XE_RTP_NAME("18019271663"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)), + XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE)) + }, + + /* Xe2_LPG */ + + { XE_RTP_NAME("14021567978"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(CHICKEN_RASTER_2, TBIMR_FAST_CLIP)) + }, + { XE_RTP_NAME("14020756599"), + XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER), OR, + MEDIA_VERSION_ANY_GT(2000), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) + }, + { XE_RTP_NAME("14021490052"), + XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(FF_MODE, + DIS_MESH_PARTIAL_AUTOSTRIP | + DIS_MESH_AUTOSTRIP), + SET(VFLSKPD, + DIS_PARTIAL_AUTOSTRIP | + DIS_AUTOSTRIP)) + }, + + /* Xe2_HPG */ + + { XE_RTP_NAME("14020756599"), + XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) + }, + { XE_RTP_NAME("14021490052"), + XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(FF_MODE, + DIS_MESH_PARTIAL_AUTOSTRIP | + DIS_MESH_AUTOSTRIP), + SET(VFLSKPD, + DIS_PARTIAL_AUTOSTRIP | + DIS_AUTOSTRIP)) + }, + { XE_RTP_NAME("22021007897"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE)) + }, + + /* Xe3_LPG */ + { XE_RTP_NAME("14021490052"), + XE_RTP_RULES(GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(FF_MODE, + DIS_MESH_PARTIAL_AUTOSTRIP | + DIS_MESH_AUTOSTRIP), + SET(VFLSKPD, + DIS_PARTIAL_AUTOSTRIP | + DIS_AUTOSTRIP)) + }, + { XE_RTP_NAME("22021007897"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE)) + }, + { XE_RTP_NAME("14024681466"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_SLICE_COMMON_ECO_CHICKEN1, FAST_CLEAR_VALIGN_FIX)) + }, + { XE_RTP_NAME("15016589081"), + XE_RTP_RULES(GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX)) + }, +); + +static const struct xe_rtp_entry oob_was_entries[] = { +#include +}; + +static_assert(ARRAY_SIZE(oob_was_entries) == _XE_WA_OOB_COUNT); + +VISIBLE_IF_KUNIT __maybe_unused const struct xe_rtp_table oob_was = { + .entries = oob_was_entries, + .n_entries = ARRAY_SIZE(oob_was_entries), +}; +EXPORT_SYMBOL_IF_KUNIT(oob_was); + +static const struct xe_rtp_entry device_oob_was_entries[] = { +#include +}; + +static_assert(ARRAY_SIZE(device_oob_was_entries) == _XE_DEVICE_WA_OOB_COUNT); + +VISIBLE_IF_KUNIT __maybe_unused const struct xe_rtp_table device_oob_was = { + .entries = device_oob_was_entries, + .n_entries = ARRAY_SIZE(device_oob_was_entries), +}; +EXPORT_SYMBOL_IF_KUNIT(device_oob_was); + +__diag_pop(); + +/** + * xe_wa_process_device_oob - process OOB workaround table + * @xe: device instance to process workarounds for + * + * process OOB workaround table for this device, marking in @xe the + * workarounds that are active. + */ + +void xe_wa_process_device_oob(struct xe_device *xe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(xe); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, xe->wa_active.oob, device_oob_was.n_entries); + + xe->wa_active.oob_initialized = true; + xe_rtp_process(&ctx, &device_oob_was); +} + +/** + * xe_wa_process_gt_oob - process GT OOB workaround table + * @gt: GT instance to process workarounds for + * + * Process OOB workaround table for this platform, marking in @gt the + * workarounds that are active. + */ +void xe_wa_process_gt_oob(struct xe_gt *gt) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.oob, + oob_was.n_entries); + gt->wa_active.oob_initialized = true; + xe_rtp_process(&ctx, &oob_was); +} + +/** + * xe_wa_process_gt - process GT workaround table + * @gt: GT instance to process workarounds for + * + * Process GT workaround table for this platform, saving in @gt all the + * workarounds that need to be applied at the GT level. + */ +void xe_wa_process_gt(struct xe_gt *gt) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.gt, + gt_was.n_entries); + xe_rtp_process_to_sr(&ctx, >_was, >->reg_sr, false); +} +EXPORT_SYMBOL_IF_KUNIT(xe_wa_process_gt); + +/** + * xe_wa_process_engine - process engine workaround table + * @hwe: engine instance to process workarounds for + * + * Process engine workaround table for this platform, saving in @hwe all the + * workarounds that need to be applied at the engine level that match this + * engine. + */ +void xe_wa_process_engine(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.engine, + engine_was.n_entries); + xe_rtp_process_to_sr(&ctx, &engine_was, &hwe->reg_sr, false); +} + +/** + * xe_wa_process_lrc - process context workaround table + * @hwe: engine instance to process workarounds for + * + * Process context workaround table for this platform, saving in @hwe all the + * workarounds that need to be applied on context restore. These are workarounds + * touching registers that are part of the HW context image. + */ +void xe_wa_process_lrc(struct xe_hw_engine *hwe) +{ + struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + + xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.lrc, + lrc_was.n_entries); + xe_rtp_process_to_sr(&ctx, &lrc_was, &hwe->reg_lrc, true); +} + +/** + * xe_wa_device_init - initialize device with workaround oob bookkeeping + * @xe: Xe device instance to initialize + * + * Returns 0 for success, negative with error code otherwise + */ +int xe_wa_device_init(struct xe_device *xe) +{ + unsigned long *p; + + p = drmm_kzalloc(&xe->drm, + sizeof(*p) * BITS_TO_LONGS(device_oob_was.n_entries), + GFP_KERNEL); + + if (!p) + return -ENOMEM; + + xe->wa_active.oob = p; + + return 0; +} + +/** + * xe_wa_gt_init - initialize gt with workaround bookkeeping + * @gt: GT instance to initialize + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_wa_gt_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + size_t n_oob, n_lrc, n_engine, n_gt, total; + unsigned long *p; + + n_gt = BITS_TO_LONGS(gt_was.n_entries); + n_engine = BITS_TO_LONGS(engine_was.n_entries); + n_lrc = BITS_TO_LONGS(lrc_was.n_entries); + n_oob = BITS_TO_LONGS(oob_was.n_entries); + total = n_gt + n_engine + n_lrc + n_oob; + + p = drmm_kzalloc(&xe->drm, sizeof(*p) * total, GFP_KERNEL); + if (!p) + return -ENOMEM; + + gt->wa_active.gt = p; + p += n_gt; + gt->wa_active.engine = p; + p += n_engine; + gt->wa_active.lrc = p; + p += n_lrc; + gt->wa_active.oob = p; + + return 0; +} +ALLOW_ERROR_INJECTION(xe_wa_gt_init, ERRNO); /* See xe_pci_probe() */ + +void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p) +{ + size_t idx; + + drm_printf(p, "Device OOB Workarounds\n"); + for_each_set_bit(idx, xe->wa_active.oob, device_oob_was.n_entries) + if (device_oob_was.entries[idx].name) + drm_printf_indent(p, 1, "%s\n", device_oob_was.entries[idx].name); +} + +/** + * xe_wa_gt_dump() - Dump GT workarounds into a drm printer. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Return: always 0. + */ +int xe_wa_gt_dump(struct xe_gt *gt, struct drm_printer *p) +{ + size_t idx; + + drm_printf(p, "GT Workarounds\n"); + for_each_set_bit(idx, gt->wa_active.gt, gt_was.n_entries) + drm_printf_indent(p, 1, "%s\n", gt_was.entries[idx].name); + + drm_puts(p, "\n"); + drm_printf(p, "Engine Workarounds\n"); + for_each_set_bit(idx, gt->wa_active.engine, engine_was.n_entries) + drm_printf_indent(p, 1, "%s\n", engine_was.entries[idx].name); + + drm_puts(p, "\n"); + drm_printf(p, "LRC Workarounds\n"); + for_each_set_bit(idx, gt->wa_active.lrc, lrc_was.n_entries) + drm_printf_indent(p, 1, "%s\n", lrc_was.entries[idx].name); + + drm_puts(p, "\n"); + drm_printf(p, "OOB Workarounds\n"); + for_each_set_bit(idx, gt->wa_active.oob, oob_was.n_entries) + if (oob_was.entries[idx].name) + drm_printf_indent(p, 1, "%s\n", oob_was.entries[idx].name); + return 0; +} + +/* + * Apply tile (non-GT, non-display) workarounds. Think very carefully before + * adding anything to this function; most workarounds should be implemented + * elsewhere. The programming here is primarily for sgunit/soc workarounds, + * which are relatively rare. Since the registers these workarounds target are + * outside the GT, they should only need to be applied once at device + * probe/resume; they will not lose their values on any kind of GT or engine + * reset. + * + * TODO: We may want to move this over to xe_rtp in the future once we have + * enough workarounds to justify the work. + */ +void xe_wa_apply_tile_workarounds(struct xe_tile *tile) +{ + struct xe_mmio *mmio = &tile->mmio; + + if (IS_SRIOV_VF(tile->xe)) + return; + + if (XE_DEVICE_WA(tile->xe, 22010954014)) + xe_mmio_rmw32(mmio, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS); +} diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h new file mode 100644 index 000000000..f4da2b271 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wa.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WA_H_ +#define _XE_WA_H_ + +#include +#include "xe_assert.h" + +struct drm_printer; +struct xe_gt; +struct xe_hw_engine; +struct xe_tile; + +int xe_wa_device_init(struct xe_device *xe); +int xe_wa_gt_init(struct xe_gt *gt); +void xe_wa_process_device_oob(struct xe_device *xe); +void xe_wa_process_gt_oob(struct xe_gt *gt); +void xe_wa_process_gt(struct xe_gt *gt); +void xe_wa_process_engine(struct xe_hw_engine *hwe); +void xe_wa_process_lrc(struct xe_hw_engine *hwe); +void xe_wa_apply_tile_workarounds(struct xe_tile *tile); +void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p); +int xe_wa_gt_dump(struct xe_gt *gt, struct drm_printer *p); + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +extern const struct xe_rtp_table_sr gt_was; +extern __maybe_unused const struct xe_rtp_table oob_was; +extern __maybe_unused const struct xe_rtp_table device_oob_was; +#endif + +/** + * XE_GT_WA - Out-of-band GT workarounds, to be queried and called as needed. + * @gt__: gt instance + * @id__: XE_OOB_, as generated by build system in generated/xe_wa_oob.h + */ +#define XE_GT_WA(gt__, id__) ({ \ + xe_gt_assert(gt__, (gt__)->wa_active.oob_initialized); \ + test_bit(XE_WA_OOB_ ## id__, (gt__)->wa_active.oob); \ +}) + +/** + * XE_DEVICE_WA - Out-of-band Device workarounds, to be queried and called + * as needed. + * @xe__: xe_device + * @id__: XE_DEVICE_WA_OOB_, as generated by build system in generated/xe_device_wa_oob.h + */ +#define XE_DEVICE_WA(xe__, id__) ({ \ + xe_assert(xe__, (xe__)->wa_active.oob_initialized); \ + test_bit(XE_DEVICE_WA_OOB_ ## id__, (xe__)->wa_active.oob); \ +}) + +#define XE_DEVICE_WA_DISABLE(xe__, id__) ({ \ + xe_assert(xe__, (xe__)->wa_active.oob_initialized); \ + clear_bit(XE_DEVICE_WA_OOB_ ## id__, (xe__)->wa_active.oob); \ +}) + +#endif diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules new file mode 100644 index 000000000..dd69ad07f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -0,0 +1,74 @@ +1607983814 GRAPHICS_VERSION_RANGE(1200, 1210) +16010904313 GRAPHICS_VERSION_RANGE(1200, 1210) +18022495364 GRAPHICS_VERSION_RANGE(1200, 1210) +22012773006 GRAPHICS_VERSION_RANGE(1200, 1250) +14014475959 PLATFORM(DG2) +22011391025 PLATFORM(DG2) +22012727170 SUBPLATFORM(DG2, G11) +22012727685 SUBPLATFORM(DG2, G11) +22016596838 PLATFORM(PVC) +18020744125 PLATFORM(PVC) +1409600907 GRAPHICS_VERSION_RANGE(1200, 1250) +22014953428 SUBPLATFORM(DG2, G10) + SUBPLATFORM(DG2, G12) +16017236439 PLATFORM(PVC) +14019821291 MEDIA_VERSION_RANGE(1300, 2000) +14015076503 MEDIA_VERSION(1300) +14018913170 GRAPHICS_VERSION_RANGE(1270, 1274) + MEDIA_VERSION(1300) + PLATFORM(DG2) +14018094691 GRAPHICS_VERSION_RANGE(2001, 2002) + GRAPHICS_VERSION(2004) +18024947630 GRAPHICS_VERSION(2001) + GRAPHICS_VERSION(2004) + MEDIA_VERSION(2000) +16022287689 GRAPHICS_VERSION(2001) + GRAPHICS_VERSION(2004) +13011645652 GRAPHICS_VERSION_RANGE(2004, 3005) +14022293748 GRAPHICS_VERSION_RANGE(2001, 3005) +22019794406 GRAPHICS_VERSION_RANGE(2001, 3005) +22019338487 MEDIA_VERSION(2000) + GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_not_sriov_vf) + MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_not_sriov_vf) +16023588340 GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_not_sriov_vf) +14019789679 GRAPHICS_VERSION(1255) + GRAPHICS_VERSION_RANGE(1270, 2004) +14022866841 GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0) + MEDIA_VERSION(3000), MEDIA_STEP(A0, B0) +16021333562 GRAPHICS_VERSION_RANGE(1200, 1274) + MEDIA_VERSION(1300) +14016712196 GRAPHICS_VERSION(1255) + GRAPHICS_VERSION_RANGE(1270, 1274) +14015568240 GRAPHICS_VERSION_RANGE(1255, 1260) +18013179988 GRAPHICS_VERSION(1255) + GRAPHICS_VERSION_RANGE(1270, 1274) +1508761755 GRAPHICS_VERSION(1255) +16023105232 GRAPHICS_VERSION_RANGE(2001, 3005) + MEDIA_VERSION_RANGE(1301, 3002) +16026508708 GRAPHICS_VERSION_RANGE(1200, 3005) + MEDIA_VERSION_RANGE(1300, 3002) +14020001231 GRAPHICS_VERSION_RANGE(2001,2004), FUNC(xe_rtp_match_psmi_enabled) + MEDIA_VERSION_RANGE(2000, 3002), FUNC(xe_rtp_match_psmi_enabled) +16023683509 MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled) + MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_psmi_enabled) +14025515070 GRAPHICS_VERSION(2004) + MEDIA_VERSION_RANGE(1301, 3000) + MEDIA_VERSION(3002) + GRAPHICS_VERSION_RANGE(3000, 3001) + GRAPHICS_VERSION_RANGE(3003, 3005) + MEDIA_VERSION(3500) + GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0) + +15015404425_disable PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER) +16026007364 MEDIA_VERSION(3000) +14020316580 MEDIA_VERSION(1301) + +14025883347 MEDIA_VERSION_RANGE(1301, 3500) + GRAPHICS_VERSION_RANGE(2004, 3005) +16029380221 MEDIA_VERSION(3500) +22022079272 MEDIA_VERSION(3503) + GRAPHICS_VERSION(3510) + GRAPHICS_VERSION(3511) +16029897822 MEDIA_VERSION(3500) + GRAPHICS_VERSION(3510) +14027054324 GRAPHICS_VERSION(3511) diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c new file mode 100644 index 000000000..51eb940ce --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_wait_user_fence.h" + +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_macros.h" +#include "xe_exec_queue.h" + +static int do_compare(u64 addr, u64 value, u64 mask, u16 op) +{ + u64 rvalue; + int err; + bool passed; + + err = copy_from_user(&rvalue, u64_to_user_ptr(addr), sizeof(rvalue)); + if (err) + return -EFAULT; + + switch (op) { + case DRM_XE_UFENCE_WAIT_OP_EQ: + passed = (rvalue & mask) == (value & mask); + break; + case DRM_XE_UFENCE_WAIT_OP_NEQ: + passed = (rvalue & mask) != (value & mask); + break; + case DRM_XE_UFENCE_WAIT_OP_GT: + passed = (rvalue & mask) > (value & mask); + break; + case DRM_XE_UFENCE_WAIT_OP_GTE: + passed = (rvalue & mask) >= (value & mask); + break; + case DRM_XE_UFENCE_WAIT_OP_LT: + passed = (rvalue & mask) < (value & mask); + break; + case DRM_XE_UFENCE_WAIT_OP_LTE: + passed = (rvalue & mask) <= (value & mask); + break; + default: + XE_WARN_ON("Not possible"); + return -EINVAL; + } + + return passed ? 0 : 1; +} + +#define VALID_FLAGS DRM_XE_UFENCE_WAIT_FLAG_ABSTIME +#define MAX_OP DRM_XE_UFENCE_WAIT_OP_LTE + +static long to_jiffies_timeout(struct xe_device *xe, + struct drm_xe_wait_user_fence *args) +{ + unsigned long long t; + long timeout; + + /* + * For negative timeout we want to wait "forever" by setting + * MAX_SCHEDULE_TIMEOUT. But we have to assign this value also + * to args->timeout to avoid being zeroed on the signal delivery + * (see arithmetics after wait). + */ + if (args->timeout < 0) { + args->timeout = MAX_SCHEDULE_TIMEOUT; + return MAX_SCHEDULE_TIMEOUT; + } + + if (args->timeout == 0) + return 0; + + /* + * Save the timeout to an u64 variable because nsecs_to_jiffies + * might return a value that overflows s32 variable. + */ + if (args->flags & DRM_XE_UFENCE_WAIT_FLAG_ABSTIME) + t = drm_timeout_abs_to_jiffies(args->timeout); + else + t = nsecs_to_jiffies(args->timeout); + + /* + * Anything greater then MAX_SCHEDULE_TIMEOUT is meaningless, + * also we don't want to cap it at MAX_SCHEDULE_TIMEOUT because + * apparently user doesn't mean to wait forever, otherwise the + * args->timeout should have been set to a negative value. + */ + if (t > MAX_SCHEDULE_TIMEOUT) + timeout = MAX_SCHEDULE_TIMEOUT - 1; + else + timeout = t; + + return timeout ?: 1; +} + +int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + DEFINE_WAIT_FUNC(w_wait, woken_wake_function); + struct drm_xe_wait_user_fence *args = data; + struct xe_exec_queue *q = NULL; + u64 addr = args->addr; + int err = 0; + long timeout; + ktime_t start; + + if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad) || + XE_IOCTL_DBG(xe, args->pad2) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->flags & ~VALID_FLAGS)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->op > MAX_OP)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, addr & 0x7)) + return -EINVAL; + + if (args->exec_queue_id) { + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) + return -ENOENT; + } + + timeout = to_jiffies_timeout(xe, args); + + start = ktime_get(); + + add_wait_queue(&xe->ufence_wq, &w_wait); + for (;;) { + err = do_compare(addr, args->value, args->mask, args->op); + if (err <= 0) + break; + + if (signal_pending(current)) { + err = -ERESTARTSYS; + break; + } + + if (q) { + if (q->ops->reset_status(q)) { + drm_info(&xe->drm, "exec queue reset detected\n"); + err = -EIO; + break; + } + } + + if (!timeout) { + LNL_FLUSH_WORKQUEUE(xe->ordered_wq); + err = do_compare(addr, args->value, args->mask, + args->op); + if (err <= 0) { + drm_dbg(&xe->drm, "LNL_FLUSH_WORKQUEUE resolved ufence timeout\n"); + break; + } + err = -ETIME; + break; + } + + timeout = wait_woken(&w_wait, TASK_INTERRUPTIBLE, timeout); + } + remove_wait_queue(&xe->ufence_wq, &w_wait); + + if (!(args->flags & DRM_XE_UFENCE_WAIT_FLAG_ABSTIME)) { + args->timeout -= ktime_to_ns(ktime_sub(ktime_get(), start)); + if (args->timeout < 0) + args->timeout = 0; + } + + if (q) + xe_exec_queue_put(q); + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.h b/drivers/gpu/drm/xe/xe_wait_user_fence.h new file mode 100644 index 000000000..0e268978f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WAIT_USER_FENCE_H_ +#define _XE_WAIT_USER_FENCE_H_ + +struct drm_device; +struct drm_file; + +int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c new file mode 100644 index 000000000..fe65ed246 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wopcm.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_wopcm.h" + +#include + +#include "regs/xe_guc_regs.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt_types.h" +#include "xe_mmio.h" +#include "xe_uc_fw.h" + +/** + * DOC: Write Once Protected Content Memory (WOPCM) Layout + * + * The layout of the WOPCM will be fixed after writing to GuC WOPCM size and + * offset registers whose values are calculated and determined by HuC/GuC + * firmware size and set of hardware requirements/restrictions as shown below: + * + * :: + * + * +=========> +====================+ <== WOPCM Top + * ^ | HW contexts RSVD | + * | +===> +====================+ <== GuC WOPCM Top + * | ^ | | + * | | | | + * | | | | + * | GuC | | + * | WOPCM | | + * | Size +--------------------+ + * WOPCM | | GuC FW RSVD | + * | | +--------------------+ + * | | | GuC Stack RSVD | + * | | +------------------- + + * | v | GuC WOPCM RSVD | + * | +===> +====================+ <== GuC WOPCM base + * | | WOPCM RSVD | + * | +------------------- + <== HuC Firmware Top + * v | HuC FW | + * +=========> +====================+ <== WOPCM Base + * + * GuC accessible WOPCM starts at GuC WOPCM base and ends at GuC WOPCM top. + * The top part of the WOPCM is reserved for hardware contexts (e.g. RC6 + * context). + */ + +/* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */ +/* FIXME: Larger size require for some platforms, do a proper probe sooner or later */ +#define DGFX_WOPCM_SIZE SZ_4M +#define LNL_WOPCM_SIZE SZ_8M +#define MTL_WOPCM_SIZE SZ_4M +#define WOPCM_SIZE SZ_2M + +/* 16KB WOPCM (RSVD WOPCM) is reserved from HuC firmware top. */ +#define WOPCM_RESERVED_SIZE SZ_16K + +/* 16KB reserved at the beginning of GuC WOPCM. */ +#define GUC_WOPCM_RESERVED SZ_16K +/* 8KB from GUC_WOPCM_RESERVED is reserved for GuC stack. */ +#define GUC_WOPCM_STACK_RESERVED SZ_8K + +/* GuC WOPCM Offset value needs to be aligned to 16KB. */ +#define GUC_WOPCM_OFFSET_ALIGNMENT (1UL << GUC_WOPCM_OFFSET_SHIFT) + +/* 36KB WOPCM reserved at the end of WOPCM */ +#define WOPCM_HW_CTX_RESERVED (SZ_32K + SZ_4K) + +static inline struct xe_gt *wopcm_to_gt(struct xe_wopcm *wopcm) +{ + return container_of(wopcm, struct xe_gt, uc.wopcm); +} + +static inline struct xe_device *wopcm_to_xe(struct xe_wopcm *wopcm) +{ + return gt_to_xe(wopcm_to_gt(wopcm)); +} + +static u32 context_reserved_size(void) +{ + return WOPCM_HW_CTX_RESERVED; +} + +static bool __check_layout(struct xe_device *xe, u32 wopcm_size, + u32 guc_wopcm_base, u32 guc_wopcm_size, + u32 guc_fw_size, u32 huc_fw_size) +{ + const u32 ctx_rsvd = context_reserved_size(); + u32 size; + + size = wopcm_size - ctx_rsvd; + if (unlikely(guc_wopcm_base >= size || + guc_wopcm_size > size - guc_wopcm_base)) { + drm_err(&xe->drm, + "WOPCM: invalid GuC region layout: %uK + %uK > %uK\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K, + size / SZ_1K); + return false; + } + + size = guc_fw_size + GUC_WOPCM_RESERVED + GUC_WOPCM_STACK_RESERVED; + if (unlikely(guc_wopcm_size < size)) { + drm_err(&xe->drm, "WOPCM: no space for %s: %uK < %uK\n", + xe_uc_fw_type_repr(XE_UC_FW_TYPE_GUC), + guc_wopcm_size / SZ_1K, size / SZ_1K); + return false; + } + + size = huc_fw_size + WOPCM_RESERVED_SIZE; + if (unlikely(guc_wopcm_base < size)) { + drm_err(&xe->drm, "WOPCM: no space for %s: %uK < %uK\n", + xe_uc_fw_type_repr(XE_UC_FW_TYPE_HUC), + guc_wopcm_base / SZ_1K, size / SZ_1K); + return false; + } + + return true; +} + +static bool __wopcm_regs_locked(struct xe_gt *gt, + u32 *guc_wopcm_base, u32 *guc_wopcm_size) +{ + u32 reg_base = xe_mmio_read32(>->mmio, DMA_GUC_WOPCM_OFFSET); + u32 reg_size = xe_mmio_read32(>->mmio, GUC_WOPCM_SIZE); + + if (!(reg_size & GUC_WOPCM_SIZE_LOCKED) || + !(reg_base & GUC_WOPCM_OFFSET_VALID)) + return false; + + *guc_wopcm_base = reg_base & GUC_WOPCM_OFFSET_MASK; + *guc_wopcm_size = reg_size & GUC_WOPCM_SIZE_MASK; + return true; +} + +static int __wopcm_init_regs(struct xe_device *xe, struct xe_gt *gt, + struct xe_wopcm *wopcm) +{ + u32 base = wopcm->guc.base; + u32 size = wopcm->guc.size; + u32 huc_agent = xe_uc_fw_is_available(>->uc.huc.fw) ? HUC_LOADING_AGENT_GUC : 0; + u32 mask; + int err; + + XE_WARN_ON(!(base & GUC_WOPCM_OFFSET_MASK)); + XE_WARN_ON(base & ~GUC_WOPCM_OFFSET_MASK); + XE_WARN_ON(!(size & GUC_WOPCM_SIZE_MASK)); + XE_WARN_ON(size & ~GUC_WOPCM_SIZE_MASK); + + mask = GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED; + err = xe_mmio_write32_and_verify(>->mmio, GUC_WOPCM_SIZE, size, mask, + size | GUC_WOPCM_SIZE_LOCKED); + if (err) + goto err_out; + + mask = GUC_WOPCM_OFFSET_MASK | GUC_WOPCM_OFFSET_VALID | huc_agent; + err = xe_mmio_write32_and_verify(>->mmio, DMA_GUC_WOPCM_OFFSET, + base | huc_agent, mask, + base | huc_agent | + GUC_WOPCM_OFFSET_VALID); + if (err) + goto err_out; + + return 0; + +err_out: + drm_notice(&xe->drm, "Failed to init uC WOPCM registers!\n"); + drm_notice(&xe->drm, "%s(%#x)=%#x\n", "DMA_GUC_WOPCM_OFFSET", + DMA_GUC_WOPCM_OFFSET.addr, + xe_mmio_read32(>->mmio, DMA_GUC_WOPCM_OFFSET)); + drm_notice(&xe->drm, "%s(%#x)=%#x\n", "GUC_WOPCM_SIZE", + GUC_WOPCM_SIZE.addr, + xe_mmio_read32(>->mmio, GUC_WOPCM_SIZE)); + + return err; +} + +u32 xe_wopcm_size(struct xe_device *xe) +{ + if (xe->info.platform >= XE_LUNARLAKE) + return LNL_WOPCM_SIZE; + else if (IS_DGFX(xe)) + return DGFX_WOPCM_SIZE; + else if (xe->info.platform == XE_METEORLAKE) + return MTL_WOPCM_SIZE; + else + return WOPCM_SIZE; +} + +static u32 max_wopcm_size(struct xe_device *xe) +{ + if (xe->info.platform == XE_NOVALAKE_P) + return SZ_16M; + else + return SZ_8M; +} + +/** + * xe_wopcm_init() - Initialize the WOPCM structure. + * @wopcm: pointer to xe_wopcm. + * + * This function will partition WOPCM space based on GuC and HuC firmware sizes + * and will allocate max remaining for use by GuC. This function will also + * enforce platform dependent hardware restrictions on GuC WOPCM offset and + * size. It will fail the WOPCM init if any of these checks fail, so that the + * following WOPCM registers setup and GuC firmware uploading would be aborted. + */ +int xe_wopcm_init(struct xe_wopcm *wopcm) +{ + struct xe_device *xe = wopcm_to_xe(wopcm); + struct xe_gt *gt = wopcm_to_gt(wopcm); + u32 guc_fw_size = xe_uc_fw_get_upload_size(>->uc.guc.fw); + u32 huc_fw_size = xe_uc_fw_get_upload_size(>->uc.huc.fw); + u32 ctx_rsvd = context_reserved_size(); + u32 guc_wopcm_base; + u32 guc_wopcm_size; + bool locked; + int ret = 0; + + if (!guc_fw_size) + return -EINVAL; + + wopcm->size = xe_wopcm_size(xe); + drm_dbg(&xe->drm, "WOPCM: %uK\n", wopcm->size / SZ_1K); + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + XE_WARN_ON(guc_fw_size >= wopcm->size); + XE_WARN_ON(huc_fw_size >= wopcm->size); + XE_WARN_ON(ctx_rsvd + WOPCM_RESERVED_SIZE >= wopcm->size); + + locked = __wopcm_regs_locked(gt, &guc_wopcm_base, &guc_wopcm_size); + if (locked) { + drm_dbg(&xe->drm, "GuC WOPCM is already locked [%uK, %uK)\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); + /* + * When the GuC wopcm base and size are preprogrammed by + * BIOS/IFWI, check against the max allowed wopcm size to + * validate if the programmed values align to the wopcm layout. + * + * FIXME: This is giving the maximum overall WOPCM size and not + * the size relative to each GT. + */ + wopcm->size = max_wopcm_size(xe); + + goto check; + } + + /* + * Aligned value of guc_wopcm_base will determine available WOPCM space + * for HuC firmware and mandatory reserved area. + */ + guc_wopcm_base = huc_fw_size + WOPCM_RESERVED_SIZE; + guc_wopcm_base = ALIGN(guc_wopcm_base, GUC_WOPCM_OFFSET_ALIGNMENT); + + /* + * Need to clamp guc_wopcm_base now to make sure the following math is + * correct. Formal check of whole WOPCM layout will be done below. + */ + guc_wopcm_base = min(guc_wopcm_base, wopcm->size - ctx_rsvd); + + /* Aligned remainings of usable WOPCM space can be assigned to GuC. */ + guc_wopcm_size = wopcm->size - ctx_rsvd - guc_wopcm_base; + guc_wopcm_size &= GUC_WOPCM_SIZE_MASK; + + drm_dbg(&xe->drm, "Calculated GuC WOPCM [%uK, %uK)\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); + +check: + if (__check_layout(xe, wopcm->size, guc_wopcm_base, guc_wopcm_size, + guc_fw_size, huc_fw_size)) { + wopcm->guc.base = guc_wopcm_base; + wopcm->guc.size = guc_wopcm_size; + XE_WARN_ON(!wopcm->guc.base); + XE_WARN_ON(!wopcm->guc.size); + } else { + drm_notice(&xe->drm, "Unsuccessful WOPCM partitioning\n"); + return -E2BIG; + } + + if (!locked) + ret = __wopcm_init_regs(xe, gt, wopcm); + + return ret; +} +ALLOW_ERROR_INJECTION(xe_wopcm_init, ERRNO); /* See xe_pci_probe() */ diff --git a/drivers/gpu/drm/xe/xe_wopcm.h b/drivers/gpu/drm/xe/xe_wopcm.h new file mode 100644 index 000000000..0197a2824 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wopcm.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WOPCM_H_ +#define _XE_WOPCM_H_ + +#include "xe_wopcm_types.h" + +struct xe_device; + +int xe_wopcm_init(struct xe_wopcm *wopcm); +u32 xe_wopcm_size(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_wopcm_types.h b/drivers/gpu/drm/xe/xe_wopcm_types.h new file mode 100644 index 000000000..99d34837c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wopcm_types.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WOPCM_TYPES_H_ +#define _XE_WOPCM_TYPES_H_ + +#include + +/** + * struct xe_wopcm - Overall WOPCM info and WOPCM regions. + */ +struct xe_wopcm { + /** @size: Size of overall WOPCM */ + u32 size; + /** @guc: GuC WOPCM Region info */ + struct { + /** @guc.base: GuC WOPCM base which is offset from WOPCM base */ + u32 base; + /** @guc.size: Size of the GuC WOPCM region */ + u32 size; + } guc; +}; + +#endif -- cgit v1.3.1