From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- drivers/gpu/drm/scheduler/.kunitconfig | 12 + drivers/gpu/drm/scheduler/Makefile | 27 + drivers/gpu/drm/scheduler/gpu_scheduler_trace.h | 159 +++ drivers/gpu/drm/scheduler/sched_entity.c | 661 ++++++++++ drivers/gpu/drm/scheduler/sched_fence.c | 242 ++++ drivers/gpu/drm/scheduler/sched_internal.h | 154 +++ drivers/gpu/drm/scheduler/sched_main.c | 1355 +++++++++++++++++++++ drivers/gpu/drm/scheduler/sched_rq.c | 420 +++++++ drivers/gpu/drm/scheduler/tests/Makefile | 8 + drivers/gpu/drm/scheduler/tests/mock_scheduler.c | 381 ++++++ drivers/gpu/drm/scheduler/tests/sched_tests.h | 224 ++++ drivers/gpu/drm/scheduler/tests/tests_basic.c | 922 ++++++++++++++ drivers/gpu/drm/scheduler/tests/tests_scheduler.c | 874 +++++++++++++ 13 files changed, 5439 insertions(+) create mode 100644 drivers/gpu/drm/scheduler/.kunitconfig create mode 100644 drivers/gpu/drm/scheduler/Makefile create mode 100644 drivers/gpu/drm/scheduler/gpu_scheduler_trace.h create mode 100644 drivers/gpu/drm/scheduler/sched_entity.c create mode 100644 drivers/gpu/drm/scheduler/sched_fence.c create mode 100644 drivers/gpu/drm/scheduler/sched_internal.h create mode 100644 drivers/gpu/drm/scheduler/sched_main.c create mode 100644 drivers/gpu/drm/scheduler/sched_rq.c create mode 100644 drivers/gpu/drm/scheduler/tests/Makefile create mode 100644 drivers/gpu/drm/scheduler/tests/mock_scheduler.c create mode 100644 drivers/gpu/drm/scheduler/tests/sched_tests.h create mode 100644 drivers/gpu/drm/scheduler/tests/tests_basic.c create mode 100644 drivers/gpu/drm/scheduler/tests/tests_scheduler.c (limited to 'drivers/gpu/drm/scheduler') diff --git a/drivers/gpu/drm/scheduler/.kunitconfig b/drivers/gpu/drm/scheduler/.kunitconfig new file mode 100644 index 000000000..cece53609 --- /dev/null +++ b/drivers/gpu/drm/scheduler/.kunitconfig @@ -0,0 +1,12 @@ +CONFIG_KUNIT=y +CONFIG_DRM=y +CONFIG_DRM_SCHED_KUNIT_TEST=y +CONFIG_EXPERT=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_ATOMIC_SLEEP=y +CONFIG_LOCK_DEBUGGING_SUPPORT=y +CONFIG_PROVE_LOCKING=y +CONFIG_LOCKDEP=y +CONFIG_DEBUG_LOCKDEP=y +CONFIG_DEBUG_LIST=y diff --git a/drivers/gpu/drm/scheduler/Makefile b/drivers/gpu/drm/scheduler/Makefile new file mode 100644 index 000000000..74e75eff6 --- /dev/null +++ b/drivers/gpu/drm/scheduler/Makefile @@ -0,0 +1,27 @@ +# +# Copyright 2017 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# +gpu-sched-y := sched_main.o sched_fence.o sched_entity.o sched_rq.o + +obj-$(CONFIG_DRM_SCHED) += gpu-sched.o + +obj-$(CONFIG_DRM_SCHED_KUNIT_TEST) += tests/ diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h new file mode 100644 index 000000000..261713dd7 --- /dev/null +++ b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h @@ -0,0 +1,159 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#if !defined(_GPU_SCHED_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _GPU_SCHED_TRACE_H_ + +#include +#include +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM gpu_scheduler +#define TRACE_INCLUDE_FILE gpu_scheduler_trace + +/** + * DOC: uAPI trace events + * + * ``drm_sched_job_queue``, ``drm_sched_job_run``, ``drm_sched_job_add_dep``, + * ``drm_sched_job_done`` and ``drm_sched_job_unschedulable`` are considered + * stable uAPI. + * + * Common trace events attributes: + * + * * ``dev`` - the dev_name() of the device running the job. + * + * * ``ring`` - the hardware ring running the job. Together with ``dev`` it + * uniquely identifies where the job is going to be executed. + * + * * ``fence`` - the &struct dma_fence.context and the &struct dma_fence.seqno of + * &struct drm_sched_fence.finished + * + * All the events depends on drm_sched_job_arm() having been called already for + * the job because they use &struct drm_sched_job.sched or + * &struct drm_sched_job.s_fence. + */ + +DECLARE_EVENT_CLASS(drm_sched_job, + TP_PROTO(struct drm_sched_job *sched_job, struct drm_sched_entity *entity), + TP_ARGS(sched_job, entity), + TP_STRUCT__entry( + __string(name, sched_job->sched->name) + __field(u32, job_count) + __field(int, hw_job_count) + __string(dev, dev_name(sched_job->sched->dev)) + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, client_id) + ), + + TP_fast_assign( + __assign_str(name); + __entry->job_count = spsc_queue_count(&entity->job_queue); + __entry->hw_job_count = atomic_read( + &sched_job->sched->credit_count); + __assign_str(dev); + __entry->fence_context = sched_job->s_fence->finished.context; + __entry->fence_seqno = sched_job->s_fence->finished.seqno; + __entry->client_id = sched_job->s_fence->drm_client_id; + ), + TP_printk("dev=%s, fence=%llu:%llu, ring=%s, job count:%u, hw job count:%d, client_id:%llu", + __get_str(dev), + __entry->fence_context, __entry->fence_seqno, __get_str(name), + __entry->job_count, __entry->hw_job_count, __entry->client_id) +); + +DEFINE_EVENT(drm_sched_job, drm_sched_job_queue, + TP_PROTO(struct drm_sched_job *sched_job, struct drm_sched_entity *entity), + TP_ARGS(sched_job, entity) +); + +DEFINE_EVENT(drm_sched_job, drm_sched_job_run, + TP_PROTO(struct drm_sched_job *sched_job, struct drm_sched_entity *entity), + TP_ARGS(sched_job, entity) +); + +TRACE_EVENT(drm_sched_job_done, + TP_PROTO(struct drm_sched_fence *fence), + TP_ARGS(fence), + TP_STRUCT__entry( + __field(u64, fence_context) + __field(u64, fence_seqno) + ), + + TP_fast_assign( + __entry->fence_context = fence->finished.context; + __entry->fence_seqno = fence->finished.seqno; + ), + TP_printk("fence=%llu:%llu signaled", + __entry->fence_context, __entry->fence_seqno) +); + +TRACE_EVENT(drm_sched_job_add_dep, + TP_PROTO(struct drm_sched_job *sched_job, struct dma_fence *fence), + TP_ARGS(sched_job, fence), + TP_STRUCT__entry( + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, ctx) + __field(u64, seqno) + ), + + TP_fast_assign( + __entry->fence_context = sched_job->s_fence->finished.context; + __entry->fence_seqno = sched_job->s_fence->finished.seqno; + __entry->ctx = fence->context; + __entry->seqno = fence->seqno; + ), + TP_printk("fence=%llu:%llu depends on fence=%llu:%llu", + __entry->fence_context, __entry->fence_seqno, + __entry->ctx, __entry->seqno) +); + +TRACE_EVENT(drm_sched_job_unschedulable, + TP_PROTO(struct drm_sched_job *sched_job, struct dma_fence *fence), + TP_ARGS(sched_job, fence), + TP_STRUCT__entry( + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, ctx) + __field(u64, seqno) + ), + + TP_fast_assign( + __entry->fence_context = sched_job->s_fence->finished.context; + __entry->fence_seqno = sched_job->s_fence->finished.seqno; + __entry->ctx = fence->context; + __entry->seqno = fence->seqno; + ), + TP_printk("fence=%llu:%llu depends on unsignalled fence=%llu:%llu", + __entry->fence_context, __entry->fence_seqno, + __entry->ctx, __entry->seqno) +); + +#endif /* _GPU_SCHED_TRACE_H_ */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/scheduler +#include diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c new file mode 100644 index 000000000..a4a7efdbf --- /dev/null +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -0,0 +1,661 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include + +#include +#include + +#include "sched_internal.h" + +#include "gpu_scheduler_trace.h" + +/** + * drm_sched_entity_stats_release - Entity stats kref release function + * @kref: Entity stats embedded kref pointer + */ +void drm_sched_entity_stats_release(struct kref *kref) +{ + struct drm_sched_entity_stats *stats = + container_of(kref, typeof(*stats), kref); + + kfree(stats); +} + +/** + * drm_sched_entity_stats_new - Allocate a new struct drm_sched_entity_stats object + * + * Return: Pointer to newly allocated struct drm_sched_entity_stats object. + */ +static struct drm_sched_entity_stats *drm_sched_entity_stats_new(void) +{ + struct drm_sched_entity_stats *stats; + + stats = kzalloc_obj(*stats); + if (!stats) + return NULL; + + kref_init(&stats->kref); + spin_lock_init(&stats->lock); + ewma_drm_sched_avgtime_init(&stats->avg_job_us); + + return stats; +} + +/** + * drm_sched_entity_stats_job_add_gpu_time - Account job execution time to entity + * @job: Scheduler job to account. + * + * Accounts the execution time of @job to its respective entity stats object. + * + * Return: Job's real duration in micro seconds. + */ +ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job) +{ + struct drm_sched_entity_stats *stats = job->entity_stats; + struct drm_sched_fence *s_fence = job->s_fence; + ktime_t start, end, duration; + + start = dma_fence_timestamp(&s_fence->scheduled); + end = dma_fence_timestamp(&s_fence->finished); + duration = ktime_sub(end, start); + + spin_lock(&stats->lock); + stats->runtime = ktime_add(stats->runtime, duration); + ewma_drm_sched_avgtime_add(&stats->avg_job_us, ktime_to_us(duration)); + spin_unlock(&stats->lock); + + return duration; +} + +/** + * drm_sched_entity_init - Init a context entity used by scheduler when + * submit to HW ring. + * + * @entity: scheduler entity to init + * @priority: priority of the entity + * @sched_list: the list of drm scheds on which jobs from this + * entity can be submitted + * @num_sched_list: number of drm sched in sched_list + * @guilty: atomic_t set to 1 when a job on this queue + * is found to be guilty causing a timeout + * + * Note that the &sched_list must have at least one element to schedule the entity. + * + * For changing @priority later on at runtime see + * drm_sched_entity_set_priority(). For changing the set of schedulers + * @sched_list at runtime see drm_sched_entity_modify_sched(). + * + * An entity is cleaned up by calling drm_sched_entity_fini(). See also + * drm_sched_entity_destroy(). + * + * Returns 0 on success or a negative error code on failure. + */ +int drm_sched_entity_init(struct drm_sched_entity *entity, + enum drm_sched_priority priority, + struct drm_gpu_scheduler **sched_list, + unsigned int num_sched_list, + atomic_t *guilty) +{ + if (!entity || !sched_list || !num_sched_list || !sched_list[0]) + return -EINVAL; + + memset(entity, 0, sizeof(struct drm_sched_entity)); + + entity->stats = drm_sched_entity_stats_new(); + if (!entity->stats) + return -ENOMEM; + + INIT_LIST_HEAD(&entity->list); + entity->guilty = guilty; + entity->priority = priority; + entity->last_user = current->group_leader; + entity->rq_priority = drm_sched_policy == DRM_SCHED_POLICY_FAIR ? + DRM_SCHED_PRIORITY_KERNEL : priority; + entity->num_sched_list = num_sched_list; + entity->sched_list = num_sched_list > 1 ? sched_list : NULL; + RCU_INIT_POINTER(entity->last_scheduled, NULL); + RB_CLEAR_NODE(&entity->rb_tree_node); + + if (!sched_list[0]->sched_rq) { + /* Since every entry covered by num_sched_list + * should be non-NULL and therefore we warn drivers + * not to do this and to fix their DRM calling order. + */ + pr_warn("%s: called with uninitialized scheduler\n", __func__); + } else { + enum drm_sched_priority p = entity->priority; + + /* + * The "priority" of an entity cannot exceed the number of + * run-queues of a scheduler. Protect against num_rqs being 0, + * by converting to signed. Choose the lowest priority + * available. + */ + if (p >= sched_list[0]->num_user_rqs) { + dev_err(sched_list[0]->dev, "entity with out-of-bounds priority:%u num_user_rqs:%u\n", + p, sched_list[0]->num_user_rqs); + p = max_t(s32, + (s32)sched_list[0]->num_user_rqs - 1, + (s32)DRM_SCHED_PRIORITY_KERNEL); + entity->priority = p; + } + entity->rq = sched_list[0]->sched_rq[entity->rq_priority]; + } + + init_completion(&entity->entity_idle); + + /* We start in an idle state. */ + complete_all(&entity->entity_idle); + + spin_lock_init(&entity->lock); + spsc_queue_init(&entity->job_queue); + + atomic_set(&entity->fence_seq, 0); + entity->fence_context = dma_fence_context_alloc(2); + + return 0; +} +EXPORT_SYMBOL(drm_sched_entity_init); + +/** + * drm_sched_entity_modify_sched - Modify sched of an entity + * @entity: scheduler entity to init + * @sched_list: the list of new drm scheds which will replace + * existing entity->sched_list + * @num_sched_list: number of drm sched in sched_list + * + * Note that this must be called under the same common lock for @entity as + * drm_sched_job_arm() and drm_sched_entity_push_job(), or the driver needs to + * guarantee through some other means that this is never called while new jobs + * can be pushed to @entity. + */ +void drm_sched_entity_modify_sched(struct drm_sched_entity *entity, + struct drm_gpu_scheduler **sched_list, + unsigned int num_sched_list) +{ + WARN_ON(!num_sched_list || !sched_list); + + spin_lock(&entity->lock); + entity->sched_list = sched_list; + entity->num_sched_list = num_sched_list; + spin_unlock(&entity->lock); +} +EXPORT_SYMBOL(drm_sched_entity_modify_sched); + +static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity) +{ + rmb(); /* for list_empty to work without lock */ + + if (list_empty(&entity->list) || + spsc_queue_count(&entity->job_queue) == 0 || + entity->stopped) + return true; + + return false; +} + +/** + * drm_sched_entity_error - return error of last scheduled job + * @entity: scheduler entity to check + * + * Opportunistically return the error of the last scheduled job. Result can + * change any time when new jobs are pushed to the hw. + */ +int drm_sched_entity_error(struct drm_sched_entity *entity) +{ + struct dma_fence *fence; + int r; + + rcu_read_lock(); + fence = rcu_dereference(entity->last_scheduled); + r = fence ? fence->error : 0; + rcu_read_unlock(); + + return r; +} +EXPORT_SYMBOL(drm_sched_entity_error); + +static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, + struct dma_fence_cb *cb); + +static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) +{ + struct drm_sched_job *job = container_of(wrk, typeof(*job), work); + struct dma_fence *f; + unsigned long index; + + /* Wait for all dependencies to avoid data corruptions */ + xa_for_each(&job->dependencies, index, f) { + struct drm_sched_fence *s_fence = to_drm_sched_fence(f); + + if (s_fence && f == &s_fence->scheduled) { + /* The dependencies array had a reference on the scheduled + * fence, and the finished fence refcount might have + * dropped to zero. Use dma_fence_get_rcu() so we get + * a NULL fence in that case. + */ + f = dma_fence_get_rcu(&s_fence->finished); + + /* Now that we have a reference on the finished fence, + * we can release the reference the dependencies array + * had on the scheduled fence. + */ + dma_fence_put(&s_fence->scheduled); + } + + xa_erase(&job->dependencies, index); + if (f && !dma_fence_add_callback(f, &job->finish_cb, + drm_sched_entity_kill_jobs_cb)) + return; + + dma_fence_put(f); + } + + drm_sched_fence_scheduled(job->s_fence, NULL); + drm_sched_fence_finished(job->s_fence, -ESRCH); + WARN_ON(job->s_fence->parent); + job->sched->ops->free_job(job); +} + +/* Signal the scheduler finished fence when the entity in question is killed. */ +static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, + struct dma_fence_cb *cb) +{ + struct drm_sched_job *job = container_of(cb, struct drm_sched_job, + finish_cb); + + dma_fence_put(f); + + INIT_WORK(&job->work, drm_sched_entity_kill_jobs_work); + schedule_work(&job->work); +} + +/** + * drm_sched_entity_kill - kill an entity's pending jobs and remove it + * @entity: the entity to remove + * + * Removes the entity from the scheduler's run queue and kills all pending jobs. + * + * This function should be used over drm_sched_entity_flush() if it is not + * desired to actually wait for all pending jobs to finish. + */ +void drm_sched_entity_kill(struct drm_sched_entity *entity) +{ + struct drm_sched_job *job; + struct dma_fence *prev; + + spin_lock(&entity->lock); + entity->stopped = true; + drm_sched_rq_remove_entity(entity->rq, entity); + spin_unlock(&entity->lock); + + /* Make sure this entity is not used by the scheduler at the moment */ + wait_for_completion(&entity->entity_idle); + + /* The entity is guaranteed to not be used by the scheduler */ + prev = rcu_dereference_check(entity->last_scheduled, true); + dma_fence_get(prev); + while ((job = drm_sched_entity_queue_pop(entity))) { + struct drm_sched_fence *s_fence = job->s_fence; + + dma_fence_get(&s_fence->finished); + if (!prev || + dma_fence_add_callback(prev, &job->finish_cb, + drm_sched_entity_kill_jobs_cb)) { + /* + * Adding callback above failed. + * dma_fence_put() checks for NULL. + */ + dma_fence_put(prev); + drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); + } + + prev = &s_fence->finished; + } + dma_fence_put(prev); +} +EXPORT_SYMBOL(drm_sched_entity_kill); + +/** + * drm_sched_entity_flush - Flush a context entity + * + * @entity: scheduler entity + * @timeout: time to wait in for Q to become empty in jiffies. + * + * Splitting drm_sched_entity_fini() into two functions, The first one does the + * waiting, removes the entity from the runqueue and returns an error when the + * process was killed. + * + * Returns the remaining time in jiffies left from the input timeout + */ +long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) +{ + struct drm_gpu_scheduler *sched; + struct task_struct *last_user; + long ret = timeout; + + sched = entity->rq->sched; + /* + * The client will not queue more jobs during this fini - consume + * existing queued ones, or discard them on SIGKILL. + */ + if (current->flags & PF_EXITING) { + if (timeout) + ret = wait_event_timeout( + sched->job_scheduled, + drm_sched_entity_is_idle(entity), + timeout); + } else { + wait_event_killable(sched->job_scheduled, + drm_sched_entity_is_idle(entity)); + } + + /* For a killed process disallow further enqueueing of jobs. */ + last_user = cmpxchg(&entity->last_user, current->group_leader, NULL); + if (last_user == current->group_leader && + (current->flags & PF_EXITING) && (current->exit_code == SIGKILL)) + drm_sched_entity_kill(entity); + + return ret; +} +EXPORT_SYMBOL(drm_sched_entity_flush); + +/** + * drm_sched_entity_fini - Destroy a context entity + * + * @entity: scheduler entity + * + * Cleanups up @entity which has been initialized by drm_sched_entity_init(). + * + * If there are potentially job still in flight or getting newly queued + * drm_sched_entity_flush() must be called first. This function then goes over + * the entity and signals all jobs with an error code if the process was killed. + */ +void drm_sched_entity_fini(struct drm_sched_entity *entity) +{ + /* + * If consumption of existing jobs wasn't completed forcefully remove + * them. Also makes sure that the scheduler won't touch this entity any + * more. + */ + drm_sched_entity_kill(entity); + + if (entity->dependency) { + dma_fence_remove_callback(entity->dependency, &entity->cb); + dma_fence_put(entity->dependency); + entity->dependency = NULL; + } + + dma_fence_put(rcu_dereference_check(entity->last_scheduled, true)); + RCU_INIT_POINTER(entity->last_scheduled, NULL); + drm_sched_entity_stats_put(entity->stats); +} +EXPORT_SYMBOL(drm_sched_entity_fini); + +/** + * drm_sched_entity_destroy - Destroy a context entity + * @entity: scheduler entity + * + * Calls drm_sched_entity_flush() and drm_sched_entity_fini() as a + * convenience wrapper. + */ +void drm_sched_entity_destroy(struct drm_sched_entity *entity) +{ + drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY); + drm_sched_entity_fini(entity); +} +EXPORT_SYMBOL(drm_sched_entity_destroy); + +/* + * drm_sched_entity_wakeup - callback to clear the entity's dependency and + * wake up the scheduler + */ +static void drm_sched_entity_wakeup(struct dma_fence *f, + struct dma_fence_cb *cb) +{ + struct drm_sched_entity *entity = + container_of(cb, struct drm_sched_entity, cb); + + entity->dependency = NULL; + dma_fence_put(f); + drm_sched_wakeup(entity->rq->sched); +} + +/** + * drm_sched_entity_set_priority - Sets priority of the entity + * + * @entity: scheduler entity + * @priority: scheduler priority + * + * Update the priority of runqueues used for the entity. + */ +void drm_sched_entity_set_priority(struct drm_sched_entity *entity, + enum drm_sched_priority priority) +{ + spin_lock(&entity->lock); + entity->priority = priority; + spin_unlock(&entity->lock); +} +EXPORT_SYMBOL(drm_sched_entity_set_priority); + +/* + * Add a callback to the current dependency of the entity to wake up the + * scheduler when the entity becomes available. + */ +static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity, + struct drm_sched_job *sched_job) +{ + struct drm_gpu_scheduler *sched = entity->rq->sched; + struct dma_fence *fence = entity->dependency; + struct drm_sched_fence *s_fence; + + if (fence->context == entity->fence_context || + fence->context == entity->fence_context + 1) { + /* + * Fence is a scheduled/finished fence from a job + * which belongs to the same entity, we can ignore + * fences from ourself + */ + dma_fence_put(entity->dependency); + return false; + } + + s_fence = to_drm_sched_fence(fence); + if (!fence->error && s_fence && s_fence->sched == sched && + !test_bit(DRM_SCHED_FENCE_DONT_PIPELINE, &fence->flags)) { + + /* + * Fence is from the same scheduler, only need to wait for + * it to be scheduled + */ + fence = dma_fence_get(&s_fence->scheduled); + dma_fence_put(entity->dependency); + entity->dependency = fence; + } + + if (trace_drm_sched_job_unschedulable_enabled() && + !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &entity->dependency->flags)) + trace_drm_sched_job_unschedulable(sched_job, entity->dependency); + + if (!dma_fence_add_callback(entity->dependency, &entity->cb, + drm_sched_entity_wakeup)) + return true; + + dma_fence_put(entity->dependency); + return false; +} + +static struct dma_fence * +drm_sched_job_dependency(struct drm_sched_job *job, + struct drm_sched_entity *entity) +{ + struct dma_fence *f; + + /* We keep the fence around, so we can iterate over all dependencies + * in drm_sched_entity_kill_jobs_cb() to ensure all deps are signaled + * before killing the job. + */ + f = xa_load(&job->dependencies, job->last_dependency); + if (f) { + job->last_dependency++; + return dma_fence_get(f); + } + + if (job->sched->ops->prepare_job) + return job->sched->ops->prepare_job(job, entity); + + return NULL; +} + +struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) +{ + struct drm_sched_job *sched_job; + + sched_job = drm_sched_entity_queue_peek(entity); + if (!sched_job) + return NULL; + + while ((entity->dependency = + drm_sched_job_dependency(sched_job, entity))) { + if (drm_sched_entity_add_dependency_cb(entity, sched_job)) + return NULL; + } + + /* skip jobs from entity that marked guilty */ + if (entity->guilty && atomic_read(entity->guilty)) + dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED); + + dma_fence_put(rcu_dereference_check(entity->last_scheduled, true)); + rcu_assign_pointer(entity->last_scheduled, + dma_fence_get(&sched_job->s_fence->finished)); + + /* + * If the queue is empty we allow drm_sched_entity_select_rq() to + * locklessly access ->last_scheduled. This only works if we set the + * pointer before we dequeue and if we a write barrier here. + */ + smp_wmb(); + + spsc_queue_pop(&entity->job_queue); + + drm_sched_rq_pop_entity(entity); + + /* Jobs and entities might have different lifecycles. Since we're + * removing the job from the entities queue, set the jobs entity pointer + * to NULL to prevent any future access of the entity through this job. + */ + sched_job->entity = NULL; + + return sched_job; +} + +void drm_sched_entity_select_rq(struct drm_sched_entity *entity) +{ + struct dma_fence *fence; + struct drm_gpu_scheduler *sched; + struct drm_sched_rq *rq; + + /* single possible engine and already selected */ + if (!entity->sched_list) + return; + + /* queue non-empty, stay on the same engine */ + if (spsc_queue_count(&entity->job_queue)) + return; + + /* + * Only when the queue is empty are we guaranteed that + * drm_sched_run_job_work() cannot change entity->last_scheduled. To + * enforce ordering we need a read barrier here. See + * drm_sched_entity_pop_job() for the other side. + */ + smp_rmb(); + + fence = rcu_dereference_check(entity->last_scheduled, true); + + /* stay on the same engine if the previous job hasn't finished */ + if (fence && !dma_fence_is_signaled(fence)) + return; + + spin_lock(&entity->lock); + sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list); + rq = sched ? sched->sched_rq[entity->rq_priority] : NULL; + if (rq != entity->rq) { + drm_sched_rq_remove_entity(entity->rq, entity); + entity->rq = rq; + } + + if (entity->num_sched_list == 1) + entity->sched_list = NULL; + + spin_unlock(&entity->lock); +} + +/** + * drm_sched_entity_push_job - Submit a job to the entity's job queue + * @sched_job: job to submit + * + * Note: To guarantee that the order of insertion to queue matches the job's + * fence sequence number this function should be called with drm_sched_job_arm() + * under common lock for the struct drm_sched_entity that was set up for + * @sched_job in drm_sched_job_init(). + */ +void drm_sched_entity_push_job(struct drm_sched_job *sched_job) +{ + struct drm_sched_entity *entity = sched_job->entity; + bool first; + ktime_t submit_ts; + + trace_drm_sched_job_queue(sched_job, entity); + + if (trace_drm_sched_job_add_dep_enabled()) { + struct dma_fence *entry; + unsigned long index; + + xa_for_each(&sched_job->dependencies, index, entry) + trace_drm_sched_job_add_dep(sched_job, entry); + } + atomic_inc(entity->rq->sched->score); + WRITE_ONCE(entity->last_user, current->group_leader); + + /* + * After the sched_job is pushed into the entity queue, it may be + * completed and freed up at any time. We can no longer access it. + * Make sure to set the submit_ts first, to avoid a race. + */ + sched_job->submit_ts = submit_ts = ktime_get(); + first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); + + /* first job wakes up scheduler */ + if (first) { + struct drm_gpu_scheduler *sched; + + sched = drm_sched_rq_add_entity(entity, submit_ts); + if (sched) + drm_sched_wakeup(sched); + } +} +EXPORT_SYMBOL(drm_sched_entity_push_job); diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c new file mode 100644 index 000000000..096fe28aa --- /dev/null +++ b/drivers/gpu/drm/scheduler/sched_fence.c @@ -0,0 +1,242 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include +#include +#include + +#include + +#include "sched_internal.h" + +static struct kmem_cache *sched_fence_slab; + +static int __init drm_sched_fence_slab_init(void) +{ + sched_fence_slab = KMEM_CACHE(drm_sched_fence, SLAB_HWCACHE_ALIGN); + if (!sched_fence_slab) + return -ENOMEM; + + return 0; +} + +static void __exit drm_sched_fence_slab_fini(void) +{ + rcu_barrier(); + kmem_cache_destroy(sched_fence_slab); +} + +static void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, + struct dma_fence *fence) +{ + /* + * smp_store_release() to ensure another thread racing us + * in drm_sched_fence_set_deadline_finished() sees the + * fence's parent set before test_bit() + */ + smp_store_release(&s_fence->parent, dma_fence_get(fence)); + if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, + &s_fence->finished.flags)) + dma_fence_set_deadline(fence, s_fence->deadline); +} + +void drm_sched_fence_scheduled(struct drm_sched_fence *fence, + struct dma_fence *parent) +{ + /* Set the parent before signaling the scheduled fence, such that, + * any waiter expecting the parent to be filled after the job has + * been scheduled (which is the case for drivers delegating waits + * to some firmware) doesn't have to busy wait for parent to show + * up. + */ + if (!IS_ERR_OR_NULL(parent)) + drm_sched_fence_set_parent(fence, parent); + + dma_fence_signal(&fence->scheduled); +} + +void drm_sched_fence_finished(struct drm_sched_fence *fence, int result) +{ + if (result) + dma_fence_set_error(&fence->finished, result); + dma_fence_signal(&fence->finished); +} + +static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence) +{ + return "drm_sched"; +} + +static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f) +{ + struct drm_sched_fence *fence = to_drm_sched_fence(f); + return (const char *)fence->sched->name; +} + +static void drm_sched_fence_free_rcu(struct rcu_head *rcu) +{ + struct dma_fence *f = container_of(rcu, struct dma_fence, rcu); + struct drm_sched_fence *fence = to_drm_sched_fence(f); + + if (!WARN_ON_ONCE(!fence)) + kmem_cache_free(sched_fence_slab, fence); +} + +/** + * drm_sched_fence_free - free up an uninitialized fence + * + * @fence: fence to free + * + * Free up the fence memory. Should only be used if drm_sched_fence_init() + * has not been called yet. + */ +void drm_sched_fence_free(struct drm_sched_fence *fence) +{ + /* This function should not be called if the fence has been initialized. */ + if (!WARN_ON_ONCE(fence->sched)) + kmem_cache_free(sched_fence_slab, fence); +} + +/** + * drm_sched_fence_release_scheduled - callback that fence can be freed + * + * @f: fence + * + * This function is called when the reference count becomes zero. + * It just RCU schedules freeing up the fence. + */ +static void drm_sched_fence_release_scheduled(struct dma_fence *f) +{ + struct drm_sched_fence *fence = to_drm_sched_fence(f); + + dma_fence_put(fence->parent); + call_rcu(&fence->finished.rcu, drm_sched_fence_free_rcu); +} + +/** + * drm_sched_fence_release_finished - drop extra reference + * + * @f: fence + * + * Drop the extra reference from the scheduled fence to the base fence. + */ +static void drm_sched_fence_release_finished(struct dma_fence *f) +{ + struct drm_sched_fence *fence = to_drm_sched_fence(f); + + dma_fence_put(&fence->scheduled); +} + +static void drm_sched_fence_set_deadline_finished(struct dma_fence *f, + ktime_t deadline) +{ + struct drm_sched_fence *fence = to_drm_sched_fence(f); + struct dma_fence *parent; + unsigned long flags; + + dma_fence_lock_irqsave(f, flags); + + /* If we already have an earlier deadline, keep it: */ + if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags) && + ktime_before(fence->deadline, deadline)) { + dma_fence_unlock_irqrestore(f, flags); + return; + } + + fence->deadline = deadline; + set_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags); + + dma_fence_unlock_irqrestore(f, flags); + + /* + * smp_load_aquire() to ensure that if we are racing another + * thread calling drm_sched_fence_set_parent(), that we see + * the parent set before it calls test_bit(HAS_DEADLINE_BIT) + */ + parent = smp_load_acquire(&fence->parent); + if (parent) + dma_fence_set_deadline(parent, deadline); +} + +static const struct dma_fence_ops drm_sched_fence_ops_scheduled = { + .get_driver_name = drm_sched_fence_get_driver_name, + .get_timeline_name = drm_sched_fence_get_timeline_name, + .release = drm_sched_fence_release_scheduled, +}; + +static const struct dma_fence_ops drm_sched_fence_ops_finished = { + .get_driver_name = drm_sched_fence_get_driver_name, + .get_timeline_name = drm_sched_fence_get_timeline_name, + .release = drm_sched_fence_release_finished, + .set_deadline = drm_sched_fence_set_deadline_finished, +}; + +struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f) +{ + if (rcu_access_pointer(f->ops) == &drm_sched_fence_ops_scheduled) + return container_of(f, struct drm_sched_fence, scheduled); + + if (rcu_access_pointer(f->ops) == &drm_sched_fence_ops_finished) + return container_of(f, struct drm_sched_fence, finished); + + return NULL; +} +EXPORT_SYMBOL(to_drm_sched_fence); + +struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *entity, + void *owner, + u64 drm_client_id) +{ + struct drm_sched_fence *fence = NULL; + + fence = kmem_cache_zalloc(sched_fence_slab, GFP_KERNEL); + if (fence == NULL) + return NULL; + + fence->owner = owner; + fence->drm_client_id = drm_client_id; + spin_lock_init(&fence->lock); + + return fence; +} + +void drm_sched_fence_init(struct drm_sched_fence *fence, + struct drm_sched_entity *entity) +{ + unsigned seq; + + fence->sched = entity->rq->sched; + seq = atomic_inc_return(&entity->fence_seq); + dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled, + &fence->lock, entity->fence_context, seq); + dma_fence_init(&fence->finished, &drm_sched_fence_ops_finished, + &fence->lock, entity->fence_context + 1, seq); +} + +module_init(drm_sched_fence_slab_init); +module_exit(drm_sched_fence_slab_fini); + +MODULE_DESCRIPTION("DRM GPU scheduler"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/scheduler/sched_internal.h b/drivers/gpu/drm/scheduler/sched_internal.h new file mode 100644 index 000000000..a901801fc --- /dev/null +++ b/drivers/gpu/drm/scheduler/sched_internal.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: MIT */ + +#ifndef _DRM_GPU_SCHEDULER_INTERNAL_H_ +#define _DRM_GPU_SCHEDULER_INTERNAL_H_ + +#include +#include +#include + +/** + * struct drm_sched_entity_stats - execution stats for an entity. + * @kref: reference count for the object. + * @lock: lock guarding the @runtime updates. + * @runtime: time entity spent on the GPU. + * @prev_runtime: previous @runtime used to get the runtime delta. + * @vruntime: virtual runtime as accumulated by the fair algorithm. + * @avg_job_us: average job duration. + * + * Because jobs and entities have decoupled lifetimes, ie. we cannot access the + * entity once the job has been de-queued, and we do need know how much GPU time + * each entity has spent, we need to track this in a separate object which is + * reference counted by both entities and jobs. + */ +struct drm_sched_entity_stats { + struct kref kref; + spinlock_t lock; /* Protects the below fields. */ + ktime_t runtime; + ktime_t prev_runtime; + ktime_t vruntime; + + struct ewma_drm_sched_avgtime avg_job_us; +}; + +/* Used to choose between FIFO and RR job-scheduling */ +extern int drm_sched_policy; + +#define DRM_SCHED_POLICY_RR 0 +#define DRM_SCHED_POLICY_FIFO 1 +#define DRM_SCHED_POLICY_FAIR 2 + +bool drm_sched_can_queue(struct drm_gpu_scheduler *sched, + struct drm_sched_entity *entity); +void drm_sched_wakeup(struct drm_gpu_scheduler *sched); + +void drm_sched_rq_init(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq); + +struct drm_gpu_scheduler * +drm_sched_rq_add_entity(struct drm_sched_entity *entity, ktime_t ts); +void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, + struct drm_sched_entity *entity); +void drm_sched_rq_pop_entity(struct drm_sched_entity *entity); + +struct drm_sched_entity * +drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq); + +void drm_sched_entity_select_rq(struct drm_sched_entity *entity); +struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity); + +struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *s_entity, + void *owner, u64 drm_client_id); +void drm_sched_fence_init(struct drm_sched_fence *fence, + struct drm_sched_entity *entity); +void drm_sched_fence_free(struct drm_sched_fence *fence); + +void drm_sched_fence_scheduled(struct drm_sched_fence *fence, + struct dma_fence *parent); +void drm_sched_fence_finished(struct drm_sched_fence *fence, int result); + +/** + * drm_sched_entity_queue_pop - Low level helper for popping queued jobs + * + * @entity: scheduler entity + * + * Low level helper for popping queued jobs. + * + * Returns: The job dequeued or NULL. + */ +static inline struct drm_sched_job * +drm_sched_entity_queue_pop(struct drm_sched_entity *entity) +{ + struct spsc_node *node; + + node = spsc_queue_pop(&entity->job_queue); + if (!node) + return NULL; + + return container_of(node, struct drm_sched_job, queue_node); +} + +/** + * drm_sched_entity_queue_peek - Low level helper for peeking at the job queue + * + * @entity: scheduler entity + * + * Low level helper for peeking at the job queue + * + * Returns: The job at the head of the queue or NULL. + */ +static inline struct drm_sched_job * +drm_sched_entity_queue_peek(struct drm_sched_entity *entity) +{ + struct spsc_node *node; + + node = spsc_queue_peek(&entity->job_queue); + if (!node) + return NULL; + + return container_of(node, struct drm_sched_job, queue_node); +} + +/* Return true if entity could provide a job. */ +static inline bool +drm_sched_entity_is_ready(struct drm_sched_entity *entity) +{ + if (!spsc_queue_count(&entity->job_queue)) + return false; + + if (READ_ONCE(entity->dependency)) + return false; + + return true; +} + +void drm_sched_entity_stats_release(struct kref *kref); + +/** + * drm_sched_entity_stats_get - Obtain a reference count on &struct drm_sched_entity_stats object + * @stats: struct drm_sched_entity_stats pointer + * + * Return: struct drm_sched_entity_stats pointer + */ +static inline struct drm_sched_entity_stats * +drm_sched_entity_stats_get(struct drm_sched_entity_stats *stats) +{ + kref_get(&stats->kref); + + return stats; +} + +/** + * drm_sched_entity_stats_put - Release a reference count on &struct drm_sched_entity_stats object + * @stats: struct drm_sched_entity_stats pointer + */ +static inline void +drm_sched_entity_stats_put(struct drm_sched_entity_stats *stats) +{ + kref_put(&stats->kref, drm_sched_entity_stats_release); +} + +ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job); + +#endif diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c new file mode 100644 index 000000000..6cb6f9546 --- /dev/null +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -0,0 +1,1355 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +/** + * DOC: Overview + * + * The GPU scheduler provides entities which allow userspace to push jobs + * into software queues which are then scheduled on a hardware run queue. + * The software queues have a priority among them. The scheduler selects the entities + * from the run queue using a FIFO. The scheduler provides dependency handling + * features among jobs. The driver is supposed to provide callback functions for + * backend operations to the scheduler like submitting a job to hardware run queue, + * returning the dependencies of a job etc. + * + * The organisation of the scheduler is the following: + * + * 1. Each hw run queue has one scheduler + * 2. Each scheduler has multiple run queues with different priorities + * (e.g., HIGH_HW,HIGH_SW, KERNEL, NORMAL) + * 3. Each scheduler run queue has a queue of entities to schedule + * 4. Entities themselves maintain a queue of jobs that will be scheduled on + * the hardware. + * + * The jobs in an entity are always scheduled in the order in which they were pushed. + * + * Note that once a job was taken from the entities queue and pushed to the + * hardware, i.e. the pending queue, the entity must not be referenced anymore + * through the jobs entity pointer. + */ + +/** + * DOC: Flow Control + * + * The DRM GPU scheduler provides a flow control mechanism to regulate the rate + * in which the jobs fetched from scheduler entities are executed. + * + * In this context the &drm_gpu_scheduler keeps track of a driver specified + * credit limit representing the capacity of this scheduler and a credit count; + * every &drm_sched_job carries a driver specified number of credits. + * + * Once a job is executed (but not yet finished), the job's credits contribute + * to the scheduler's credit count until the job is finished. If by executing + * one more job the scheduler's credit count would exceed the scheduler's + * credit limit, the job won't be executed. Instead, the scheduler will wait + * until the credit count has decreased enough to not overflow its credit limit. + * This implies waiting for previously executed jobs. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "sched_internal.h" + +#define CREATE_TRACE_POINTS +#include "gpu_scheduler_trace.h" + +int drm_sched_policy = DRM_SCHED_POLICY_FIFO; + +/** + * DOC: sched_policy (int) + * Used to override default entities scheduling policy in a run queue. + */ +MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default), " __stringify(DRM_SCHED_POLICY_FAIR) " = Fair (experimental)."); +module_param_named(sched_policy, drm_sched_policy, int, 0444); + +static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched) +{ + u32 credits; + + WARN_ON(check_sub_overflow(sched->credit_limit, + atomic_read(&sched->credit_count), + &credits)); + + return credits; +} + +/** + * drm_sched_can_queue -- Can we queue more to the hardware? + * @sched: scheduler instance + * @entity: the scheduler entity + * + * Return true if we can push at least one more job from @entity, false + * otherwise. + */ +bool drm_sched_can_queue(struct drm_gpu_scheduler *sched, + struct drm_sched_entity *entity) +{ + struct drm_sched_job *s_job; + + s_job = drm_sched_entity_queue_peek(entity); + if (!s_job) + return false; + + /* If a job exceeds the credit limit, truncate it to the credit limit + * itself to guarantee forward progress. + */ + if (s_job->credits > sched->credit_limit) { + dev_WARN(sched->dev, + "Jobs may not exceed the credit limit, truncate.\n"); + s_job->credits = sched->credit_limit; + } + + return drm_sched_available_credits(sched) >= s_job->credits; +} + +/** + * drm_sched_run_job_queue - enqueue run-job work + * @sched: scheduler instance + */ +static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) +{ + if (!drm_sched_is_stopped(sched)) + queue_work(sched->submit_wq, &sched->work_run_job); +} + +/** + * drm_sched_run_free_queue - enqueue free-job work + * @sched: scheduler instance + */ +static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched) +{ + if (!drm_sched_is_stopped(sched)) + queue_work(sched->submit_wq, &sched->work_free_job); +} + +/** + * drm_sched_job_done - complete a job + * @s_job: pointer to the job which is done + * @result: 0 on success, -ERRNO on error + * + * Finish the job's fence and resubmit the work items. + */ +static void drm_sched_job_done(struct drm_sched_job *s_job, int result) +{ + struct drm_sched_fence *s_fence = s_job->s_fence; + struct drm_gpu_scheduler *sched = s_fence->sched; + + atomic_sub(s_job->credits, &sched->credit_count); + atomic_dec(sched->score); + + trace_drm_sched_job_done(s_fence); + + dma_fence_get(&s_fence->finished); + drm_sched_fence_finished(s_fence, result); + dma_fence_put(&s_fence->finished); + drm_sched_run_free_queue(sched); +} + +/** + * drm_sched_job_done_cb - the callback for a done job + * @f: fence + * @cb: fence callbacks + */ +static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct drm_sched_job *s_job = container_of(cb, struct drm_sched_job, cb); + + drm_sched_job_done(s_job, f->error); +} + +/** + * drm_sched_start_timeout - start timeout for reset worker + * + * @sched: scheduler instance to start the worker for + * + * Start the timeout for the given scheduler. + */ +static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched) +{ + lockdep_assert_held(&sched->job_list_lock); + + if (sched->timeout != MAX_SCHEDULE_TIMEOUT && + !list_empty(&sched->pending_list)) + mod_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout); +} + +static void drm_sched_start_timeout_unlocked(struct drm_gpu_scheduler *sched) +{ + spin_lock(&sched->job_list_lock); + drm_sched_start_timeout(sched); + spin_unlock(&sched->job_list_lock); +} + +/** + * drm_sched_tdr_queue_imm: - immediately start job timeout handler + * + * @sched: scheduler for which the timeout handling should be started. + * + * Start timeout handling immediately for the named scheduler. + */ +void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched) +{ + spin_lock(&sched->job_list_lock); + sched->timeout = 0; + drm_sched_start_timeout(sched); + spin_unlock(&sched->job_list_lock); +} +EXPORT_SYMBOL(drm_sched_tdr_queue_imm); + +/** + * drm_sched_fault - immediately start timeout handler + * + * @sched: scheduler where the timeout handling should be started. + * + * Start timeout handling immediately when the driver detects a hardware fault. + */ +void drm_sched_fault(struct drm_gpu_scheduler *sched) +{ + if (sched->timeout_wq) + mod_delayed_work(sched->timeout_wq, &sched->work_tdr, 0); +} +EXPORT_SYMBOL(drm_sched_fault); + +/** + * drm_sched_suspend_timeout - Suspend scheduler job timeout + * + * @sched: scheduler instance for which to suspend the timeout + * + * Suspend the delayed work timeout for the scheduler. This is done by + * modifying the delayed work timeout to an arbitrary large value, + * MAX_SCHEDULE_TIMEOUT in this case. + * + * Returns the timeout remaining + * + */ +unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched) +{ + unsigned long sched_timeout, now = jiffies; + + sched_timeout = sched->work_tdr.timer.expires; + + /* + * Modify the timeout to an arbitrarily large value. This also prevents + * the timeout to be restarted when new submissions arrive + */ + if (mod_delayed_work(sched->timeout_wq, &sched->work_tdr, MAX_SCHEDULE_TIMEOUT) + && time_after(sched_timeout, now)) + return sched_timeout - now; + else + return sched->timeout; +} +EXPORT_SYMBOL(drm_sched_suspend_timeout); + +/** + * drm_sched_resume_timeout - Resume scheduler job timeout + * + * @sched: scheduler instance for which to resume the timeout + * @remaining: remaining timeout + * + * Resume the delayed work timeout for the scheduler. + */ +void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched, + unsigned long remaining) +{ + spin_lock(&sched->job_list_lock); + + if (list_empty(&sched->pending_list)) + cancel_delayed_work(&sched->work_tdr); + else + mod_delayed_work(sched->timeout_wq, &sched->work_tdr, remaining); + + spin_unlock(&sched->job_list_lock); +} +EXPORT_SYMBOL(drm_sched_resume_timeout); + +static void drm_sched_job_begin(struct drm_sched_job *s_job) +{ + struct drm_gpu_scheduler *sched = s_job->sched; + + spin_lock(&sched->job_list_lock); + list_add_tail(&s_job->list, &sched->pending_list); + drm_sched_start_timeout(sched); + spin_unlock(&sched->job_list_lock); +} + +/** + * drm_sched_job_reinsert_on_false_timeout - reinsert the job on a false timeout + * @sched: scheduler instance + * @job: job to be reinserted on the pending list + * + * In the case of a "false timeout" - when a timeout occurs but the GPU isn't + * hung and is making progress, the scheduler must reinsert the job back into + * @sched->pending_list. Otherwise, the job and its resources won't be freed + * through the &struct drm_sched_backend_ops.free_job callback. + * + * This function must be used in "false timeout" cases only. + */ +static void drm_sched_job_reinsert_on_false_timeout(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job) +{ + spin_lock(&sched->job_list_lock); + list_add(&job->list, &sched->pending_list); + + /* After reinserting the job, the scheduler enqueues the free-job work + * again if ready. Otherwise, a signaled job could be added to the + * pending list, but never freed. + */ + drm_sched_run_free_queue(sched); + spin_unlock(&sched->job_list_lock); +} + +static void drm_sched_job_timedout(struct work_struct *work) +{ + struct drm_gpu_scheduler *sched; + struct drm_sched_job *job; + enum drm_gpu_sched_stat status = DRM_GPU_SCHED_STAT_RESET; + + sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work); + + /* Protects against concurrent deletion in drm_sched_get_finished_job */ + spin_lock(&sched->job_list_lock); + job = list_first_entry_or_null(&sched->pending_list, + struct drm_sched_job, list); + + if (job) { + /* + * Remove the bad job so it cannot be freed by a concurrent + * &struct drm_sched_backend_ops.free_job. It will be + * reinserted after the scheduler's work items have been + * cancelled, at which point it's safe. + */ + list_del_init(&job->list); + spin_unlock(&sched->job_list_lock); + + status = job->sched->ops->timedout_job(job); + + /* + * Guilty job did complete and hence needs to be manually removed + * See drm_sched_stop doc. + */ + if (sched->free_guilty) { + job->sched->ops->free_job(job); + sched->free_guilty = false; + } + + if (status == DRM_GPU_SCHED_STAT_NO_HANG) + drm_sched_job_reinsert_on_false_timeout(sched, job); + } else { + spin_unlock(&sched->job_list_lock); + } + + if (status != DRM_GPU_SCHED_STAT_ENODEV) + drm_sched_start_timeout_unlocked(sched); +} + +/** + * drm_sched_stop - stop the scheduler + * + * @sched: scheduler instance + * @bad: job which caused the time out + * + * Stop the scheduler and also removes and frees all completed jobs. + * Note: bad job will not be freed as it might be used later and so it's + * callers responsibility to release it manually if it's not part of the + * pending list any more. + * + * This function is typically used for reset recovery (see the docu of + * drm_sched_backend_ops.timedout_job() for details). Do not call it for + * scheduler teardown, i.e., before calling drm_sched_fini(). + * + * As it's only used for reset recovery, drivers must not call this function + * in their &struct drm_sched_backend_ops.timedout_job callback when they + * skip a reset using &enum drm_gpu_sched_stat.DRM_GPU_SCHED_STAT_NO_HANG. + */ +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +{ + struct drm_sched_job *s_job, *tmp; + + drm_sched_wqueue_stop(sched); + + /* + * Reinsert back the bad job here - now it's safe as + * drm_sched_get_finished_job() cannot race against us and release the + * bad job at this point - we parked (waited for) any in progress + * (earlier) cleanups and drm_sched_get_finished_job() will not be + * called now until the scheduler's work items are submitted again. + */ + if (bad && bad->sched == sched) + /* + * Add at the head of the queue to reflect it was the earliest + * job extracted. + */ + list_add(&bad->list, &sched->pending_list); + + /* + * Iterate the job list from later to earlier one and either deactive + * their HW callbacks or remove them from pending list if they already + * signaled. + * This iteration is thread safe as the scheduler's work items have been + * cancelled. + */ + list_for_each_entry_safe_reverse(s_job, tmp, &sched->pending_list, + list) { + if (s_job->s_fence->parent && + dma_fence_remove_callback(s_job->s_fence->parent, + &s_job->cb)) { + dma_fence_put(s_job->s_fence->parent); + s_job->s_fence->parent = NULL; + atomic_sub(s_job->credits, &sched->credit_count); + } else { + /* + * remove job from pending_list. + * Locking here is for concurrent resume timeout + */ + spin_lock(&sched->job_list_lock); + list_del_init(&s_job->list); + spin_unlock(&sched->job_list_lock); + + /* + * Wait for job's HW fence callback to finish using s_job + * before releasing it. + * + * Job is still alive so fence refcount at least 1 + */ + dma_fence_wait(&s_job->s_fence->finished, false); + + /* + * We must keep bad job alive for later use during + * recovery by some of the drivers but leave a hint + * that the guilty job must be released. + */ + if (bad != s_job) + sched->ops->free_job(s_job); + else + sched->free_guilty = true; + } + } + + /* + * Stop pending timer in flight as we rearm it in drm_sched_start. This + * avoids the pending timeout work in progress to fire right away after + * this TDR finished and before the newly restarted jobs had a + * chance to complete. + */ + cancel_delayed_work(&sched->work_tdr); +} +EXPORT_SYMBOL(drm_sched_stop); + +/** + * drm_sched_start - recover jobs after a reset + * + * @sched: scheduler instance + * @errno: error to set on the pending fences + * + * This function is typically used for reset recovery (see the docu of + * drm_sched_backend_ops.timedout_job() for details). Do not call it for + * scheduler startup. The scheduler itself is fully operational after + * drm_sched_init() succeeded. + * + * As it's only used for reset recovery, drivers must not call this function + * in their &struct drm_sched_backend_ops.timedout_job callback when they + * skip a reset using &enum drm_gpu_sched_stat.DRM_GPU_SCHED_STAT_NO_HANG. + */ +void drm_sched_start(struct drm_gpu_scheduler *sched, int errno) +{ + struct drm_sched_job *s_job, *tmp; + + /* + * Locking the list is not required here as the scheduler's work items + * are currently not running, so no new jobs are being inserted or + * removed. Also concurrent GPU recovers can't run in parallel. + */ + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { + struct dma_fence *fence = s_job->s_fence->parent; + + atomic_add(s_job->credits, &sched->credit_count); + + if (!fence) { + drm_sched_job_done(s_job, errno ?: -ECANCELED); + continue; + } + + if (dma_fence_add_callback(fence, &s_job->cb, + drm_sched_job_done_cb)) + drm_sched_job_done(s_job, fence->error ?: errno); + } + + drm_sched_start_timeout_unlocked(sched); + drm_sched_wqueue_start(sched); +} +EXPORT_SYMBOL(drm_sched_start); + +/** + * drm_sched_resubmit_jobs - Deprecated, don't use in new code! + * + * @sched: scheduler instance + * + * Re-submitting jobs was a concept AMD came up as cheap way to implement + * recovery after a job timeout. + * + * This turned out to be not working very well. First of all there are many + * problem with the dma_fence implementation and requirements. Either the + * implementation is risking deadlocks with core memory management or violating + * documented implementation details of the dma_fence object. + * + * Drivers can still save and restore their state for recovery operations, but + * we shouldn't make this a general scheduler feature around the dma_fence + * interface. The suggested driver-side replacement is to use + * drm_sched_for_each_pending_job() after stopping the scheduler and implement + * their own recovery operations. + */ +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false; + struct dma_fence *fence; + + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { + struct drm_sched_fence *s_fence = s_job->s_fence; + + if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + } + + if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED); + + fence = sched->ops->run_job(s_job); + + if (IS_ERR_OR_NULL(fence)) { + if (IS_ERR(fence)) + dma_fence_set_error(&s_fence->finished, PTR_ERR(fence)); + + s_job->s_fence->parent = NULL; + } else { + + s_job->s_fence->parent = dma_fence_get(fence); + + /* Drop for orignal kref_init */ + dma_fence_put(fence); + } + } +} +EXPORT_SYMBOL(drm_sched_resubmit_jobs); + +/** + * drm_sched_job_init - init a scheduler job + * @job: scheduler job to init + * @entity: scheduler entity to use + * @credits: the number of credits this job contributes to the schedulers + * credit limit + * @owner: job owner for debugging + * @drm_client_id: &struct drm_file.client_id of the owner (used by trace + * events) + * + * Refer to drm_sched_entity_push_job() documentation + * for locking considerations. + * + * Drivers must make sure drm_sched_job_cleanup() if this function returns + * successfully, even when @job is aborted before drm_sched_job_arm() is called. + * + * Note that this function does not assign a valid value to each struct member + * of struct drm_sched_job. Take a look at that struct's documentation to see + * who sets which struct member with what lifetime. + * + * WARNING: amdgpu abuses &drm_sched.ready to signal when the hardware + * has died, which can mean that there's no valid runqueue for a @entity. + * This function returns -ENOENT in this case (which probably should be -EIO as + * a more meanigful return value). + * + * Returns 0 for success, negative error code otherwise. + */ +int drm_sched_job_init(struct drm_sched_job *job, + struct drm_sched_entity *entity, + u32 credits, void *owner, + uint64_t drm_client_id) +{ + if (unlikely(!credits)) { + pr_err("*ERROR* %s: credits cannot be 0!\n", __func__); + return -EINVAL; + } + + /* + * We don't know for sure how the user has allocated. Thus, zero the + * struct so that unallowed (i.e., too early) usage of pointers that + * this function does not set is guaranteed to lead to a NULL pointer + * exception instead of UB. + */ + memset(job, 0, sizeof(*job)); + + job->entity = entity; + job->credits = credits; + job->s_fence = drm_sched_fence_alloc(entity, owner, drm_client_id); + if (!job->s_fence) + return -ENOMEM; + + INIT_LIST_HEAD(&job->list); + + xa_init_flags(&job->dependencies, XA_FLAGS_ALLOC); + + return 0; +} +EXPORT_SYMBOL(drm_sched_job_init); + +/** + * drm_sched_job_arm - arm a scheduler job for execution + * @job: scheduler job to arm + * + * This arms a scheduler job for execution. Specifically it initializes the + * &drm_sched_job.s_fence of @job, so that it can be attached to struct dma_resv + * or other places that need to track the completion of this job. It also + * initializes sequence numbers, which are fundamental for fence ordering. + * + * Refer to drm_sched_entity_push_job() documentation for locking + * considerations. + * + * Once this function was called, you *must* submit @job with + * drm_sched_entity_push_job(). + * + * This can only be called if drm_sched_job_init() succeeded. + */ +void drm_sched_job_arm(struct drm_sched_job *job) +{ + struct drm_gpu_scheduler *sched; + struct drm_sched_entity *entity = job->entity; + + BUG_ON(!entity); + drm_sched_entity_select_rq(entity); + sched = entity->rq->sched; + + job->sched = sched; + job->s_priority = entity->priority; + job->entity_stats = drm_sched_entity_stats_get(entity->stats); + + drm_sched_fence_init(job->s_fence, job->entity); +} +EXPORT_SYMBOL(drm_sched_job_arm); + +/** + * drm_sched_job_add_dependency - adds the fence as a job dependency + * @job: scheduler job to add the dependencies to + * @fence: the dma_fence to add to the list of dependencies. + * + * Note that @fence is consumed in both the success and error cases. + * + * Returns: + * 0 on success, or an error on failing to expand the array. + */ +int drm_sched_job_add_dependency(struct drm_sched_job *job, + struct dma_fence *fence) +{ + struct dma_fence *entry; + unsigned long index; + u32 id = 0; + int ret; + + if (!fence) + return 0; + + /* Deduplicate if we already depend on a fence from the same context. + * This lets the size of the array of deps scale with the number of + * engines involved, rather than the number of BOs. + */ + xa_for_each(&job->dependencies, index, entry) { + if (entry->context != fence->context) + continue; + + if (dma_fence_is_later(fence, entry)) { + dma_fence_put(entry); + xa_store(&job->dependencies, index, fence, GFP_KERNEL); + } else { + dma_fence_put(fence); + } + return 0; + } + + ret = xa_alloc(&job->dependencies, &id, fence, xa_limit_32b, GFP_KERNEL); + if (ret != 0) + dma_fence_put(fence); + + return ret; +} +EXPORT_SYMBOL(drm_sched_job_add_dependency); + +/** + * drm_sched_job_add_syncobj_dependency - adds a syncobj's fence as a job dependency + * @job: scheduler job to add the dependencies to + * @file: drm file private pointer + * @handle: syncobj handle to lookup + * @point: timeline point + * + * This adds the fence matching the given syncobj to @job. + * + * Returns: + * 0 on success, or an error on failing to expand the array. + */ +int drm_sched_job_add_syncobj_dependency(struct drm_sched_job *job, + struct drm_file *file, + u32 handle, + u32 point) +{ + struct dma_fence *fence; + int ret; + + ret = drm_syncobj_find_fence(file, handle, point, 0, &fence); + if (ret) + return ret; + + return drm_sched_job_add_dependency(job, fence); +} +EXPORT_SYMBOL(drm_sched_job_add_syncobj_dependency); + +/** + * drm_sched_job_add_resv_dependencies - add all fences from the resv to the job + * @job: scheduler job to add the dependencies to + * @resv: the dma_resv object to get the fences from + * @usage: the dma_resv_usage to use to filter the fences + * + * This adds all fences matching the given usage from @resv to @job. + * Must be called with the @resv lock held. + * + * Returns: + * 0 on success, or an error on failing to expand the array. + */ +int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job, + struct dma_resv *resv, + enum dma_resv_usage usage) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + int ret; + + dma_resv_assert_held(resv); + + dma_resv_for_each_fence(&cursor, resv, usage, fence) { + /* + * As drm_sched_job_add_dependency always consumes the fence + * reference (even when it fails), and dma_resv_for_each_fence + * is not obtaining one, we need to grab one before calling. + */ + ret = drm_sched_job_add_dependency(job, dma_fence_get(fence)); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL(drm_sched_job_add_resv_dependencies); + +/** + * drm_sched_job_add_implicit_dependencies - adds implicit dependencies as job + * dependencies + * @job: scheduler job to add the dependencies to + * @obj: the gem object to add new dependencies from. + * @write: whether the job might write the object (so we need to depend on + * shared fences in the reservation object). + * + * This should be called after drm_gem_lock_reservations() on your array of + * GEM objects used in the job but before updating the reservations with your + * own fences. + * + * Returns: + * 0 on success, or an error on failing to expand the array. + */ +int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job, + struct drm_gem_object *obj, + bool write) +{ + return drm_sched_job_add_resv_dependencies(job, obj->resv, + dma_resv_usage_rw(write)); +} +EXPORT_SYMBOL(drm_sched_job_add_implicit_dependencies); + +/** + * drm_sched_job_has_dependency - check whether fence is the job's dependency + * @job: scheduler job to check + * @fence: fence to look for + * + * Returns: + * True if @fence is found within the job's dependencies, or otherwise false. + */ +bool drm_sched_job_has_dependency(struct drm_sched_job *job, + struct dma_fence *fence) +{ + struct dma_fence *f; + unsigned long index; + + xa_for_each(&job->dependencies, index, f) { + if (f == fence) + return true; + } + + return false; +} +EXPORT_SYMBOL(drm_sched_job_has_dependency); + +/** + * drm_sched_job_cleanup - clean up scheduler job resources + * @job: scheduler job to clean up + * + * Cleans up the resources allocated with drm_sched_job_init(). + * + * Drivers should call this from their error unwind code if @job is aborted + * before drm_sched_job_arm() is called. + * + * drm_sched_job_arm() is a point of no return since it initializes the fences + * and their sequence number etc. Once that function has been called, you *must* + * submit it with drm_sched_entity_push_job() and cannot simply abort it by + * calling drm_sched_job_cleanup(). + * + * This function should be called in the &drm_sched_backend_ops.free_job callback. + */ +void drm_sched_job_cleanup(struct drm_sched_job *job) +{ + struct dma_fence *fence; + unsigned long index; + + if (kref_read(&job->s_fence->finished.refcount)) { + /* The job has been processed by the scheduler, i.e., + * drm_sched_job_arm() and drm_sched_entity_push_job() have + * been called. + */ + dma_fence_put(&job->s_fence->finished); + drm_sched_entity_stats_put(job->entity_stats); + } else { + /* The job was aborted before it has been committed to be run; + * notably, drm_sched_job_arm() has not been called. + */ + drm_sched_fence_free(job->s_fence); + } + + job->s_fence = NULL; + + xa_for_each(&job->dependencies, index, fence) { + dma_fence_put(fence); + } + xa_destroy(&job->dependencies); + +} +EXPORT_SYMBOL(drm_sched_job_cleanup); + +/** + * drm_sched_wakeup - Wake up the scheduler if it is ready to queue + * @sched: scheduler instance + * + * Wake up the scheduler if we can queue jobs. + */ +void drm_sched_wakeup(struct drm_gpu_scheduler *sched) +{ + drm_sched_run_job_queue(sched); +} + +/** + * drm_sched_select_entity - Select next entity to process + * + * @sched: scheduler instance + * + * Return an entity to process or NULL if none are found. + * + * Note, that we break out of the for-loop when "entity" is non-null, which can + * also be an error-pointer--this assures we don't process lower priority + * run-queues. See comments in the respectively called functions. + */ +static struct drm_sched_entity * +drm_sched_select_entity(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_entity *entity = NULL; + int i; + + /* Start with the highest priority. + */ + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + entity = drm_sched_rq_select_entity(sched, sched->sched_rq[i]); + if (entity) + break; + } + + return IS_ERR(entity) ? NULL : entity; +} + +/** + * drm_sched_get_finished_job - fetch the next finished job to be destroyed + * + * @sched: scheduler instance + * + * Informs the caller through @have_more whether there are more finished jobs + * besides the returned one. + * + * Returns the next finished job from the pending list (if there is one) + * ready for it to be destroyed. + */ +static struct drm_sched_job * +drm_sched_get_finished_job(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *job, *next; + + spin_lock(&sched->job_list_lock); + + job = list_first_entry_or_null(&sched->pending_list, + struct drm_sched_job, list); + if (job && dma_fence_is_signaled(&job->s_fence->finished)) { + /* remove job from pending_list */ + list_del_init(&job->list); + + /* cancel this job's TO timer */ + cancel_delayed_work(&sched->work_tdr); + + next = list_first_entry_or_null(&sched->pending_list, + typeof(*next), list); + if (next) { + /* make the scheduled timestamp more accurate */ + if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, + &next->s_fence->scheduled.flags)) + next->s_fence->scheduled.timestamp = + dma_fence_timestamp(&job->s_fence->finished); + + /* start TO timer for next job */ + drm_sched_start_timeout(sched); + } + } else { + job = NULL; + } + + spin_unlock(&sched->job_list_lock); + + return job; +} + +/** + * drm_sched_pick_best - Get a drm sched from a sched_list with the least load + * @sched_list: list of drm_gpu_schedulers + * @num_sched_list: number of drm_gpu_schedulers in the sched_list + * + * Returns pointer of the sched with the least load or NULL if none of the + * drm_gpu_schedulers are ready + */ +struct drm_gpu_scheduler * +drm_sched_pick_best(struct drm_gpu_scheduler **sched_list, + unsigned int num_sched_list) +{ + struct drm_gpu_scheduler *sched, *picked_sched = NULL; + int i; + unsigned int min_score = UINT_MAX, num_score; + + for (i = 0; i < num_sched_list; ++i) { + sched = sched_list[i]; + + if (!sched->ready) { + DRM_WARN("scheduler %s is not ready, skipping", + sched->name); + continue; + } + + num_score = atomic_read(sched->score); + if (num_score < min_score) { + min_score = num_score; + picked_sched = sched; + } + } + + return picked_sched; +} +EXPORT_SYMBOL(drm_sched_pick_best); + +/** + * drm_sched_free_job_work - worker to call free_job + * + * @w: free job work + */ +static void drm_sched_free_job_work(struct work_struct *w) +{ + struct drm_gpu_scheduler *sched = + container_of(w, struct drm_gpu_scheduler, work_free_job); + struct drm_sched_job *job; + + while ((job = drm_sched_get_finished_job(sched))) { + ktime_t duration = drm_sched_entity_stats_job_add_gpu_time(job); + + /* Serialized by the worker. */ + ewma_drm_sched_avgtime_add(&sched->avg_job_us, + ktime_to_us(duration)); + + sched->ops->free_job(job); + } + + drm_sched_run_job_queue(sched); +} + +/** + * drm_sched_run_job_work - worker to call run_job + * + * @w: run job work + */ +static void drm_sched_run_job_work(struct work_struct *w) +{ + struct drm_gpu_scheduler *sched = + container_of(w, struct drm_gpu_scheduler, work_run_job); + struct drm_sched_entity *entity; + struct dma_fence *fence; + struct drm_sched_fence *s_fence; + struct drm_sched_job *sched_job; + int r; + + /* Find entity with a ready job */ + entity = drm_sched_select_entity(sched); + if (!entity) { + /* + * Either no more work to do, or the next ready job needs more + * credits than the scheduler has currently available. + */ + return; + } + + sched_job = drm_sched_entity_pop_job(entity); + if (!sched_job) { + complete_all(&entity->entity_idle); + drm_sched_run_job_queue(sched); + return; + } + + s_fence = sched_job->s_fence; + + atomic_add(sched_job->credits, &sched->credit_count); + drm_sched_job_begin(sched_job); + + trace_drm_sched_job_run(sched_job, entity); + /* + * The run_job() callback must by definition return a fence whose + * refcount has been incremented for the scheduler already. + */ + fence = sched->ops->run_job(sched_job); + complete_all(&entity->entity_idle); + drm_sched_fence_scheduled(s_fence, fence); + + if (!IS_ERR_OR_NULL(fence)) { + r = dma_fence_add_callback(fence, &sched_job->cb, + drm_sched_job_done_cb); + if (r == -ENOENT) + drm_sched_job_done(sched_job, fence->error); + else if (r) + DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", r); + + dma_fence_put(fence); + } else { + drm_sched_job_done(sched_job, IS_ERR(fence) ? + PTR_ERR(fence) : 0); + } + + wake_up(&sched->job_scheduled); + drm_sched_run_job_queue(sched); +} + +static struct workqueue_struct *drm_sched_alloc_wq(const char *name) +{ +#if (IS_ENABLED(CONFIG_LOCKDEP)) + static struct lockdep_map map = { + .name = "drm_sched_lockdep_map" + }; + + /* + * Avoid leaking a lockdep map on each drm sched creation and + * destruction by using a single lockdep map for all drm sched + * allocated submit_wq. + */ + + return alloc_ordered_workqueue_lockdep_map(name, WQ_MEM_RECLAIM, &map); +#else + return alloc_ordered_workqueue(name, WQ_MEM_RECLAIM); +#endif +} + +/** + * drm_sched_init - Init a gpu scheduler instance + * + * @sched: scheduler instance + * @args: scheduler initialization arguments + * + * Return 0 on success, otherwise error code. + */ +int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_args *args) +{ + int i; + + sched->ops = args->ops; + sched->credit_limit = args->credit_limit; + sched->name = args->name; + sched->timeout = args->timeout; + sched->hang_limit = args->hang_limit; + sched->timeout_wq = args->timeout_wq ? args->timeout_wq : system_percpu_wq; + sched->score = args->score ? args->score : &sched->_score; + sched->dev = args->dev; + + if (args->num_rqs > DRM_SCHED_PRIORITY_COUNT) { + /* This is a gross violation--tell drivers what the problem is. + */ + dev_err(sched->dev, "%s: num_rqs cannot be greater than DRM_SCHED_PRIORITY_COUNT\n", + __func__); + return -EINVAL; + } else if (sched->sched_rq) { + /* Not an error, but warn anyway so drivers can + * fine-tune their DRM calling order, and return all + * is good. + */ + dev_warn(sched->dev, "%s: scheduler already initialized!\n", __func__); + return 0; + } + + if (args->submit_wq) { + sched->submit_wq = args->submit_wq; + sched->own_submit_wq = false; + } else { + sched->submit_wq = drm_sched_alloc_wq(args->name); + if (!sched->submit_wq) + return -ENOMEM; + + sched->own_submit_wq = true; + } + + sched->num_user_rqs = args->num_rqs; + sched->num_rqs = drm_sched_policy != DRM_SCHED_POLICY_FAIR ? + args->num_rqs : 1; + sched->sched_rq = kzalloc_objs(*sched->sched_rq, args->num_rqs); + if (!sched->sched_rq) + goto Out_check_own; + + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + sched->sched_rq[i] = kzalloc_obj(*sched->sched_rq[i]); + if (!sched->sched_rq[i]) + goto Out_unroll; + drm_sched_rq_init(sched, sched->sched_rq[i]); + } + + init_waitqueue_head(&sched->job_scheduled); + INIT_LIST_HEAD(&sched->pending_list); + spin_lock_init(&sched->job_list_lock); + atomic_set(&sched->credit_count, 0); + INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout); + INIT_WORK(&sched->work_run_job, drm_sched_run_job_work); + INIT_WORK(&sched->work_free_job, drm_sched_free_job_work); + atomic_set(&sched->_score, 0); + atomic64_set(&sched->job_id_count, 0); + sched->pause_submit = false; + ewma_drm_sched_avgtime_init(&sched->avg_job_us); + + sched->ready = true; + return 0; +Out_unroll: + for (--i ; i >= DRM_SCHED_PRIORITY_KERNEL; i--) + kfree(sched->sched_rq[i]); + + kfree(sched->sched_rq); + sched->sched_rq = NULL; +Out_check_own: + if (sched->own_submit_wq) + destroy_workqueue(sched->submit_wq); + dev_err(sched->dev, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); + return -ENOMEM; +} +EXPORT_SYMBOL(drm_sched_init); + +static void drm_sched_cancel_remaining_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *job, *tmp; + + /* All other accessors are stopped. No locking necessary. */ + list_for_each_entry_safe_reverse(job, tmp, &sched->pending_list, list) { + sched->ops->cancel_job(job); + list_del(&job->list); + sched->ops->free_job(job); + } +} + +/** + * drm_sched_fini - Destroy a gpu scheduler + * + * @sched: scheduler instance + * + * Tears down and cleans up the scheduler. + * + * This stops submission of new jobs to the hardware through &struct + * drm_sched_backend_ops.run_job. If &struct drm_sched_backend_ops.cancel_job + * is implemented, all jobs will be canceled through it and afterwards cleaned + * up through &struct drm_sched_backend_ops.free_job. If cancel_job is not + * implemented, memory could leak. + */ +void drm_sched_fini(struct drm_gpu_scheduler *sched) +{ + int i; + + drm_sched_wqueue_stop(sched); + + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) + kfree(sched->sched_rq[i]); + + /* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */ + wake_up_all(&sched->job_scheduled); + + /* Confirm no work left behind accessing device structures */ + cancel_delayed_work_sync(&sched->work_tdr); + + /* Avoid memory leaks if supported by the driver. */ + if (sched->ops->cancel_job) + drm_sched_cancel_remaining_jobs(sched); + + if (sched->own_submit_wq) + destroy_workqueue(sched->submit_wq); + sched->ready = false; + kfree(sched->sched_rq); + sched->sched_rq = NULL; + + if (!list_empty(&sched->pending_list)) + dev_warn(sched->dev, "Tearing down scheduler while jobs are pending!\n"); +} +EXPORT_SYMBOL(drm_sched_fini); + +/** + * drm_sched_increase_karma - Update sched_entity guilty flag + * + * @bad: The job guilty of time out + * + * Increment on every hang caused by the 'bad' job. If this exceeds the hang + * limit of the scheduler then the respective sched entity is marked guilty and + * jobs from it will not be scheduled further + */ +void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched; + + /* don't change @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma); + +/** + * drm_sched_wqueue_ready - Is the scheduler ready for submission + * + * @sched: scheduler instance + * + * Returns true if submission is ready + */ +bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched) +{ + return sched->ready; +} +EXPORT_SYMBOL(drm_sched_wqueue_ready); + +/** + * drm_sched_wqueue_stop - stop scheduler submission + * @sched: scheduler instance + * + * Stops the scheduler from pulling new jobs from entities. It also stops + * freeing jobs automatically through drm_sched_backend_ops.free_job(). + */ +void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched) +{ + WRITE_ONCE(sched->pause_submit, true); + cancel_work_sync(&sched->work_run_job); + cancel_work_sync(&sched->work_free_job); +} +EXPORT_SYMBOL(drm_sched_wqueue_stop); + +/** + * drm_sched_wqueue_start - start scheduler submission + * @sched: scheduler instance + * + * Restarts the scheduler after drm_sched_wqueue_stop() has stopped it. + * + * This function is not necessary for 'conventional' startup. The scheduler is + * fully operational after drm_sched_init() succeeded. + */ +void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched) +{ + WRITE_ONCE(sched->pause_submit, false); + queue_work(sched->submit_wq, &sched->work_run_job); + queue_work(sched->submit_wq, &sched->work_free_job); +} +EXPORT_SYMBOL(drm_sched_wqueue_start); + +/** + * drm_sched_is_stopped() - Checks whether drm_sched is stopped + * @sched: DRM scheduler + * + * Return: true if sched is stopped, false otherwise + */ +bool drm_sched_is_stopped(struct drm_gpu_scheduler *sched) +{ + return READ_ONCE(sched->pause_submit); +} +EXPORT_SYMBOL(drm_sched_is_stopped); + +/** + * drm_sched_job_is_signaled() - DRM scheduler job is signaled + * @job: DRM scheduler job + * + * Determine if DRM scheduler job is signaled. DRM scheduler should be stopped + * to obtain a stable snapshot of state. Both parent fence (hardware fence) and + * finished fence (software fence) are checked to determine signaling state. + * + * Return: true if job is signaled, false otherwise + */ +bool drm_sched_job_is_signaled(struct drm_sched_job *job) +{ + struct drm_sched_fence *s_fence = job->s_fence; + + WARN_ON(!drm_sched_is_stopped(job->sched)); + return (s_fence->parent && dma_fence_is_signaled(s_fence->parent)) || + dma_fence_is_signaled(&s_fence->finished); +} +EXPORT_SYMBOL(drm_sched_job_is_signaled); diff --git a/drivers/gpu/drm/scheduler/sched_rq.c b/drivers/gpu/drm/scheduler/sched_rq.c new file mode 100644 index 000000000..0464d324d --- /dev/null +++ b/drivers/gpu/drm/scheduler/sched_rq.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: MIT +/* Copyright 2015 Advanced Micro Devices, Inc. */ +/* Copyright (c) 2025 Valve Corporation */ + +#include + +#include +#include + +#include "sched_internal.h" + +static __always_inline bool +drm_sched_entity_compare_before(struct rb_node *a, const struct rb_node *b) +{ + struct drm_sched_entity *ea = + rb_entry((a), struct drm_sched_entity, rb_tree_node); + struct drm_sched_entity *eb = + rb_entry((b), struct drm_sched_entity, rb_tree_node); + + return ktime_before(ea->oldest_job_waiting, eb->oldest_job_waiting); +} + +static void drm_sched_rq_update_prio(struct drm_sched_rq *rq) +{ + enum drm_sched_priority prio = DRM_SCHED_PRIORITY_INVALID; + struct rb_node *rb; + + lockdep_assert_held(&rq->lock); + + rb = rb_first_cached(&rq->rb_tree_root); + if (rb) { + struct drm_sched_entity *entity = + rb_entry(rb, typeof(*entity), rb_tree_node); + + /* + * The normal locking order is entity then run-queue so taking + * the entity lock here would be a locking inversion for the + * case when the current head of the run-queue is different from + * the one we already have locked. The unlocked read is fine + * though, because if the priority had just changed it is no big + * deal for our algorithm, but just a transient reachable only + * by drivers with userspace dynamic priority changes API. Equal + * in effect to the priority change becoming visible a few + * instructions later. + */ + prio = READ_ONCE(entity->priority); + } + + rq->head_prio = prio; +} + +static void drm_sched_rq_remove_fifo_locked(struct drm_sched_entity *entity, + struct drm_sched_rq *rq) +{ + lockdep_assert_held(&entity->lock); + lockdep_assert_held(&rq->lock); + + if (!RB_EMPTY_NODE(&entity->rb_tree_node)) { + rb_erase_cached(&entity->rb_tree_node, &rq->rb_tree_root); + RB_CLEAR_NODE(&entity->rb_tree_node); + drm_sched_rq_update_prio(rq); + } +} + +static void drm_sched_rq_update_fifo_locked(struct drm_sched_entity *entity, + struct drm_sched_rq *rq, + ktime_t ts) +{ + /* + * Both locks need to be grabbed, one to protect from entity->rq change + * for entity from within concurrent drm_sched_entity_select_rq and the + * other to update the rb tree structure. + */ + lockdep_assert_held(&entity->lock); + lockdep_assert_held(&rq->lock); + + drm_sched_rq_remove_fifo_locked(entity, rq); + + entity->oldest_job_waiting = ts; + + rb_add_cached(&entity->rb_tree_node, &rq->rb_tree_root, + drm_sched_entity_compare_before); + drm_sched_rq_update_prio(rq); +} + +/** + * drm_sched_rq_init - initialize a given run queue struct + * @sched: scheduler instance to associate with this run queue + * @rq: scheduler run queue + * + * Initializes a scheduler runqueue. + */ +void drm_sched_rq_init(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq) +{ + spin_lock_init(&rq->lock); + INIT_LIST_HEAD(&rq->entities); + rq->rb_tree_root = RB_ROOT_CACHED; + rq->sched = sched; + rq->head_prio = DRM_SCHED_PRIORITY_INVALID; +} + +/* + * Core part of the CFS-like algorithm is that the virtual runtime of lower + * priority tasks should grow quicker than the higher priority ones, so that + * when we then schedule entities with the aim of keeping their accumulated + * virtual time balanced, we can approach fair distribution of GPU time. + * + * For converting the real GPU time into virtual we pick some multipliers with + * the idea to achieve the following GPU time distribution: + * + * - Kernel priority gets roughly 2x GPU time compared to high. + * - High gets ~4x relative to normal. + * - Normal gets ~8x relative to low. + */ +static const unsigned int vruntime_shift[] = { + [DRM_SCHED_PRIORITY_KERNEL] = 1, + [DRM_SCHED_PRIORITY_HIGH] = 2, + [DRM_SCHED_PRIORITY_NORMAL] = 4, + [DRM_SCHED_PRIORITY_LOW] = 7, +}; + +static ktime_t +drm_sched_rq_get_min_vruntime(struct drm_sched_rq *rq) +{ + ktime_t vruntime = 0; + struct rb_node *rb; + + lockdep_assert_held(&rq->lock); + + rb = rb_first_cached(&rq->rb_tree_root); + if (rb) { + struct drm_sched_entity *entity = + rb_entry(rb, typeof(*entity), rb_tree_node); + struct drm_sched_entity_stats *stats = entity->stats; + + spin_lock(&stats->lock); + vruntime = stats->vruntime; + spin_unlock(&stats->lock); + } + + return vruntime; +} + +static void +drm_sched_entity_save_vruntime(struct drm_sched_entity *entity, + ktime_t min_vruntime) +{ + struct drm_sched_entity_stats *stats = entity->stats; + ktime_t vruntime; + + spin_lock(&stats->lock); + vruntime = stats->vruntime; + if (min_vruntime && vruntime > min_vruntime) + vruntime = ktime_sub(vruntime, min_vruntime); + else + vruntime = 0; + stats->vruntime = vruntime; + spin_unlock(&stats->lock); +} + +static ktime_t +drm_sched_entity_restore_vruntime(struct drm_sched_entity *entity, + ktime_t min_vruntime, + enum drm_sched_priority rq_prio) +{ + struct drm_sched_entity_stats *stats = entity->stats; + struct drm_gpu_scheduler *sched = entity->rq->sched; + enum drm_sched_priority prio = entity->priority; + unsigned long avg_us, sched_avg_us; + ktime_t vruntime; + + BUILD_BUG_ON(DRM_SCHED_PRIORITY_NORMAL < DRM_SCHED_PRIORITY_HIGH); + + spin_lock(&stats->lock); + vruntime = stats->vruntime; + avg_us = ewma_drm_sched_avgtime_read(&stats->avg_job_us); + /* + * Unlocked read of the scheduler average is fine since it is just + * heuristics and data type is a natural word size. + */ + sched_avg_us = ewma_drm_sched_avgtime_read(&sched->avg_job_us); + + /* + * Special handling for entities which were picked from the top of the + * queue and are now re-joining the top with another one already there. + */ + if (!vruntime && rq_prio != DRM_SCHED_PRIORITY_INVALID) { + if (prio > rq_prio) { + /* + * Lower priority should not overtake higher when re- + * joining at the top of the queue so push it back + * somewhere behind the "middle" of the run-queue, + * proportional to the scheduler and entity average job + * durations. + */ + vruntime = us_to_ktime((1 + avg_us + sched_avg_us) << + vruntime_shift[prio]); + } else if (prio < rq_prio) { + /* + * Higher priority can go first. + */ + vruntime = -ns_to_ktime(rq_prio - prio); + } else { + /* Favour entity with shorter jobs (interactivity). */ + if (avg_us <= sched_avg_us) + vruntime = -ns_to_ktime(1); + else + vruntime = ns_to_ktime(1); + } + } + + /* + * Restore saved relative position in the queue. + */ + vruntime = ktime_add(min_vruntime, vruntime); + + stats->vruntime = vruntime; + spin_unlock(&stats->lock); + + return vruntime; +} + +static ktime_t drm_sched_entity_update_vruntime(struct drm_sched_entity *entity) +{ + struct drm_sched_entity_stats *stats = entity->stats; + ktime_t runtime, prev; + + spin_lock(&stats->lock); + prev = stats->prev_runtime; + runtime = stats->runtime; + stats->prev_runtime = runtime; + runtime = ktime_add_ns(stats->vruntime, + ktime_to_ns(ktime_sub(runtime, prev)) << + vruntime_shift[entity->priority]); + stats->vruntime = runtime; + spin_unlock(&stats->lock); + + return runtime; +} + +static ktime_t drm_sched_entity_get_job_ts(struct drm_sched_entity *entity) +{ + return drm_sched_entity_update_vruntime(entity); +} + +/** + * drm_sched_rq_add_entity - add an entity + * @entity: scheduler entity + * @ts: submission timestamp + * + * Adds a scheduler entity to the run queue. + * + * Return: DRM scheduler selected to handle this entity or NULL if entity has + * been stopped and cannot be submitted to. + */ +struct drm_gpu_scheduler * +drm_sched_rq_add_entity(struct drm_sched_entity *entity, ktime_t ts) +{ + struct drm_gpu_scheduler *sched; + struct drm_sched_rq *rq; + + /* Add the entity to the run queue */ + spin_lock(&entity->lock); + if (entity->stopped) { + spin_unlock(&entity->lock); + + DRM_ERROR("Trying to push to a killed entity\n"); + return NULL; + } + + rq = entity->rq; + spin_lock(&rq->lock); + sched = rq->sched; + + if (list_empty(&entity->list)) { + atomic_inc(sched->score); + list_add_tail(&entity->list, &rq->entities); + } + + if (drm_sched_policy == DRM_SCHED_POLICY_FAIR) { + ts = drm_sched_rq_get_min_vruntime(rq); + ts = drm_sched_entity_restore_vruntime(entity, ts, + rq->head_prio); + } else if (drm_sched_policy == DRM_SCHED_POLICY_RR) { + ts = entity->rr_ts; + } + + drm_sched_rq_update_fifo_locked(entity, rq, ts); + + spin_unlock(&rq->lock); + spin_unlock(&entity->lock); + + return sched; +} + +/** + * drm_sched_rq_remove_entity - remove an entity + * @rq: scheduler run queue + * @entity: scheduler entity + * + * Removes a scheduler entity from the run queue. + */ +void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, + struct drm_sched_entity *entity) +{ + lockdep_assert_held(&entity->lock); + + if (list_empty(&entity->list)) + return; + + spin_lock(&rq->lock); + + atomic_dec(rq->sched->score); + list_del_init(&entity->list); + + drm_sched_rq_remove_fifo_locked(entity, rq); + + spin_unlock(&rq->lock); +} + +static ktime_t +drm_sched_rq_next_rr_ts(struct drm_sched_rq *rq, + struct drm_sched_entity *entity) +{ + ktime_t ts; + + lockdep_assert_held(&entity->lock); + lockdep_assert_held(&rq->lock); + + ts = ktime_add_ns(rq->rr_ts, 1); + entity->rr_ts = ts; + rq->rr_ts = ts; + + return ts; +} + +/** + * drm_sched_rq_pop_entity - pops an entity + * @entity: scheduler entity + * + * To be called every time after a job is popped from the entity. + */ +void drm_sched_rq_pop_entity(struct drm_sched_entity *entity) +{ + struct drm_sched_job *next_job; + struct drm_sched_rq *rq; + + /* + * Update the entity's location in the min heap according to + * the timestamp of the next job, if any. + */ + spin_lock(&entity->lock); + rq = entity->rq; + spin_lock(&rq->lock); + next_job = drm_sched_entity_queue_peek(entity); + if (next_job) { + ktime_t ts; + + if (drm_sched_policy == DRM_SCHED_POLICY_FAIR) + ts = drm_sched_entity_get_job_ts(entity); + else if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) + ts = next_job->submit_ts; + else + ts = drm_sched_rq_next_rr_ts(rq, entity); + + drm_sched_rq_update_fifo_locked(entity, rq, ts); + } else { + drm_sched_rq_remove_fifo_locked(entity, rq); + + if (drm_sched_policy == DRM_SCHED_POLICY_FAIR) { + ktime_t min_vruntime; + + min_vruntime = drm_sched_rq_get_min_vruntime(rq); + drm_sched_entity_save_vruntime(entity, min_vruntime); + } + } + spin_unlock(&rq->lock); + spin_unlock(&entity->lock); +} + +/** + * drm_sched_rq_select_entity - Select an entity which provides a job to run + * @sched: the gpu scheduler + * @rq: scheduler run queue to check. + * + * Find oldest waiting ready entity. + * + * Return an entity if one is found; return an error-pointer (!NULL) if an + * entity was ready, but the scheduler had insufficient credits to accommodate + * its job; return NULL, if no ready entity was found. + */ +struct drm_sched_entity * +drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq) +{ + struct rb_node *rb; + + spin_lock(&rq->lock); + for (rb = rb_first_cached(&rq->rb_tree_root); rb; rb = rb_next(rb)) { + struct drm_sched_entity *entity; + + entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node); + if (drm_sched_entity_is_ready(entity)) { + /* If we can't queue yet, preserve the current entity in + * terms of fairness. + */ + if (!drm_sched_can_queue(sched, entity)) { + spin_unlock(&rq->lock); + return ERR_PTR(-ENOSPC); + } + + reinit_completion(&entity->entity_idle); + break; + } + } + spin_unlock(&rq->lock); + + return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL; +} diff --git a/drivers/gpu/drm/scheduler/tests/Makefile b/drivers/gpu/drm/scheduler/tests/Makefile new file mode 100644 index 000000000..9ec185fbb --- /dev/null +++ b/drivers/gpu/drm/scheduler/tests/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 + +drm-sched-tests-y := \ + mock_scheduler.o \ + tests_basic.o \ + tests_scheduler.o + +obj-$(CONFIG_DRM_SCHED_KUNIT_TEST) += drm-sched-tests.o diff --git a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c new file mode 100644 index 000000000..2dfa3efef --- /dev/null +++ b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Valve Corporation */ + +#include + +#include "sched_tests.h" + +/* + * Here we implement the mock "GPU" (or the scheduler backend) which is used by + * the DRM scheduler unit tests in order to exercise the core functionality. + * + * Test cases are implemented in a separate file. + */ + +/** + * drm_mock_sched_entity_new - Create a new mock scheduler entity + * + * @test: KUnit test owning the entity + * @priority: Scheduling priority + * @sched: Mock scheduler on which the entity can be scheduled + * + * Returns: New mock scheduler entity with allocation managed by the test + */ +struct drm_mock_sched_entity * +drm_mock_sched_entity_new(struct kunit *test, + enum drm_sched_priority priority, + struct drm_mock_scheduler *sched) +{ + struct drm_mock_sched_entity *entity; + struct drm_gpu_scheduler *drm_sched; + int ret; + + entity = kunit_kzalloc(test, sizeof(*entity), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, entity); + + drm_sched = &sched->base; + ret = drm_sched_entity_init(&entity->base, + priority, + &drm_sched, 1, + NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + entity->test = test; + + return entity; +} + +/** + * drm_mock_sched_entity_free - Destroys a mock scheduler entity + * + * @entity: Entity to destroy + * + * To be used from the test cases once done with the entity. + */ +void drm_mock_sched_entity_free(struct drm_mock_sched_entity *entity) +{ + drm_sched_entity_destroy(&entity->base); +} + +static void drm_mock_sched_job_complete(struct drm_mock_sched_job *job) +{ + struct drm_mock_scheduler *sched = + drm_sched_to_mock_sched(job->base.sched); + + lockdep_assert_held(&sched->lock); + + job->flags |= DRM_MOCK_SCHED_JOB_DONE; + list_del(&job->link); + dma_fence_signal_locked(&job->hw_fence); + complete(&job->done); +} + +static enum hrtimer_restart +drm_mock_sched_job_signal_timer(struct hrtimer *hrtimer) +{ + struct drm_mock_sched_job *job = + container_of(hrtimer, typeof(*job), timer); + struct drm_mock_scheduler *sched = + drm_sched_to_mock_sched(job->base.sched); + struct drm_mock_sched_job *next; + ktime_t now = ktime_get(); + unsigned long flags; + LIST_HEAD(signal); + + spin_lock_irqsave(&sched->lock, flags); + list_for_each_entry_safe(job, next, &sched->job_list, link) { + if (!job->duration_us) + break; + + if (ktime_before(now, job->finish_at)) + break; + + sched->hw_timeline.cur_seqno = job->hw_fence.seqno; + drm_mock_sched_job_complete(job); + } + spin_unlock_irqrestore(&sched->lock, flags); + + return HRTIMER_NORESTART; +} + +/** + * drm_mock_sched_job_new - Create a new mock scheduler job + * + * @test: KUnit test owning the job + * @entity: Scheduler entity of the job + * + * Returns: New mock scheduler job with allocation managed by the test + */ +struct drm_mock_sched_job * +drm_mock_sched_job_new(struct kunit *test, + struct drm_mock_sched_entity *entity) +{ + struct drm_mock_sched_job *job; + int ret; + + job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, job); + + ret = drm_sched_job_init(&job->base, + &entity->base, + 1, + NULL, + 1); + KUNIT_ASSERT_EQ(test, ret, 0); + + job->test = test; + + init_completion(&job->done); + INIT_LIST_HEAD(&job->link); + hrtimer_setup(&job->timer, drm_mock_sched_job_signal_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + + return job; +} + +static const char *drm_mock_sched_hw_fence_driver_name(struct dma_fence *fence) +{ + return "drm_mock_sched"; +} + +static const char * +drm_mock_sched_hw_fence_timeline_name(struct dma_fence *fence) +{ + struct drm_mock_sched_job *job = + container_of(fence, typeof(*job), hw_fence); + + return (const char *)job->base.sched->name; +} + +static void drm_mock_sched_hw_fence_release(struct dma_fence *fence) +{ + struct drm_mock_sched_job *job = + container_of(fence, typeof(*job), hw_fence); + + hrtimer_cancel(&job->timer); + + /* Containing job is freed by the kunit framework */ +} + +static const struct dma_fence_ops drm_mock_sched_hw_fence_ops = { + .get_driver_name = drm_mock_sched_hw_fence_driver_name, + .get_timeline_name = drm_mock_sched_hw_fence_timeline_name, + .release = drm_mock_sched_hw_fence_release, +}; + +static struct dma_fence *mock_sched_run_job(struct drm_sched_job *sched_job) +{ + struct drm_mock_scheduler *sched = + drm_sched_to_mock_sched(sched_job->sched); + struct drm_mock_sched_job *job = drm_sched_job_to_mock_job(sched_job); + + dma_fence_init(&job->hw_fence, + &drm_mock_sched_hw_fence_ops, + &sched->lock, + sched->hw_timeline.context, + atomic_inc_return(&sched->hw_timeline.next_seqno)); + + dma_fence_get(&job->hw_fence); /* Reference for the job_list */ + + spin_lock_irq(&sched->lock); + if (job->duration_us) { + ktime_t prev_finish_at = 0; + + if (!list_empty(&sched->job_list)) { + struct drm_mock_sched_job *prev = + list_last_entry(&sched->job_list, typeof(*prev), + link); + + prev_finish_at = prev->finish_at; + } + + if (!prev_finish_at) + prev_finish_at = ktime_get(); + + job->finish_at = ktime_add_us(prev_finish_at, job->duration_us); + } + list_add_tail(&job->link, &sched->job_list); + if (job->finish_at) + hrtimer_start(&job->timer, job->finish_at, HRTIMER_MODE_ABS); + spin_unlock_irq(&sched->lock); + + return &job->hw_fence; +} + +/* + * Normally, drivers would take appropriate measures in this callback, such as + * killing the entity the faulty job is associated with, resetting the hardware + * and / or resubmitting non-faulty jobs. + * + * For the mock scheduler, there are no hardware rings to be resetted nor jobs + * to be resubmitted. Thus, this function merely ensures that + * a) timedout fences get signaled properly and removed from the pending list + * b) the mock scheduler framework gets informed about the timeout via a flag + * c) The drm_sched_job, not longer needed, gets freed + */ +static enum drm_gpu_sched_stat +mock_sched_timedout_job(struct drm_sched_job *sched_job) +{ + struct drm_mock_scheduler *sched = drm_sched_to_mock_sched(sched_job->sched); + struct drm_mock_sched_job *job = drm_sched_job_to_mock_job(sched_job); + unsigned long flags; + + if (job->flags & DRM_MOCK_SCHED_JOB_DONT_RESET) { + job->flags |= DRM_MOCK_SCHED_JOB_RESET_SKIPPED; + return DRM_GPU_SCHED_STAT_NO_HANG; + } + + spin_lock_irqsave(&sched->lock, flags); + if (!dma_fence_is_signaled_locked(&job->hw_fence)) { + list_del(&job->link); + job->flags |= DRM_MOCK_SCHED_JOB_TIMEDOUT; + dma_fence_set_error(&job->hw_fence, -ETIMEDOUT); + dma_fence_signal_locked(&job->hw_fence); + } + spin_unlock_irqrestore(&sched->lock, flags); + + dma_fence_put(&job->hw_fence); + drm_sched_job_cleanup(sched_job); + /* Mock job itself is freed by the kunit framework. */ + + return DRM_GPU_SCHED_STAT_RESET; +} + +static void mock_sched_free_job(struct drm_sched_job *sched_job) +{ + struct drm_mock_sched_job *job = drm_sched_job_to_mock_job(sched_job); + + dma_fence_put(&job->hw_fence); + drm_sched_job_cleanup(sched_job); + + /* Mock job itself is freed by the kunit framework. */ +} + +static void mock_sched_cancel_job(struct drm_sched_job *sched_job) +{ + struct drm_mock_scheduler *sched = drm_sched_to_mock_sched(sched_job->sched); + struct drm_mock_sched_job *job = drm_sched_job_to_mock_job(sched_job); + unsigned long flags; + + hrtimer_cancel(&job->timer); + + spin_lock_irqsave(&sched->lock, flags); + if (!dma_fence_is_signaled_locked(&job->hw_fence)) { + list_del(&job->link); + dma_fence_set_error(&job->hw_fence, -ECANCELED); + dma_fence_signal_locked(&job->hw_fence); + } + spin_unlock_irqrestore(&sched->lock, flags); + + /* + * The GPU Scheduler will call drm_sched_backend_ops.free_job(), still. + * Mock job itself is freed by the kunit framework. + */ +} + +static const struct drm_sched_backend_ops drm_mock_scheduler_ops = { + .run_job = mock_sched_run_job, + .timedout_job = mock_sched_timedout_job, + .free_job = mock_sched_free_job, + .cancel_job = mock_sched_cancel_job, +}; + +/** + * drm_mock_sched_new - Create a new mock scheduler + * + * @test: KUnit test owning the job + * @timeout: Job timeout to set + * + * Returns: New mock scheduler with allocation managed by the test + */ +struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout) +{ + static unsigned int instance; + struct drm_sched_init_args args = { + .ops = &drm_mock_scheduler_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, + .credit_limit = U32_MAX, + .hang_limit = 1, + .timeout = timeout, + .name = "drm-mock-scheduler", + }; + struct drm_mock_scheduler *sched; + struct device *dev; + char name[64]; + int ret; + + sched = kunit_kzalloc(test, sizeof(*sched), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sched); + + snprintf(name, sizeof(name), "%s-%u", args.name, instance++); + dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + args.dev = dev; + + ret = drm_sched_init(&sched->base, &args); + KUNIT_ASSERT_EQ(test, ret, 0); + + sched->test = test; + sched->hw_timeline.context = dma_fence_context_alloc(1); + atomic_set(&sched->hw_timeline.next_seqno, 0); + INIT_LIST_HEAD(&sched->job_list); + spin_lock_init(&sched->lock); + + return sched; +} + +/** + * drm_mock_sched_fini - Destroys a mock scheduler + * + * @sched: Scheduler to destroy + * + * To be used from the test cases once done with the scheduler. + */ +void drm_mock_sched_fini(struct drm_mock_scheduler *sched) +{ + drm_sched_fini(&sched->base); +} + +/** + * drm_mock_sched_advance - Advances the mock scheduler timeline + * + * @sched: Scheduler timeline to advance + * @num: By how many jobs to advance + * + * Advancing the scheduler timeline by a number of seqnos will trigger + * signalling of the hardware fences and unlinking the jobs from the internal + * scheduler tracking. + * + * This can be used from test cases which want complete control of the simulated + * job execution timing. For example submitting one job with no set duration + * would never complete it before test cases advances the timeline by one. + */ +unsigned int drm_mock_sched_advance(struct drm_mock_scheduler *sched, + unsigned int num) +{ + struct drm_mock_sched_job *job, *next; + unsigned int found = 0; + unsigned long flags; + LIST_HEAD(signal); + + spin_lock_irqsave(&sched->lock, flags); + if (WARN_ON_ONCE(sched->hw_timeline.cur_seqno + num < + sched->hw_timeline.cur_seqno)) + goto unlock; + sched->hw_timeline.cur_seqno += num; + list_for_each_entry_safe(job, next, &sched->job_list, link) { + if (sched->hw_timeline.cur_seqno < job->hw_fence.seqno) + break; + + drm_mock_sched_job_complete(job); + found++; + } +unlock: + spin_unlock_irqrestore(&sched->lock, flags); + + return found; +} + +MODULE_DESCRIPTION("DRM mock scheduler and tests"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/scheduler/tests/sched_tests.h b/drivers/gpu/drm/scheduler/tests/sched_tests.h new file mode 100644 index 000000000..553d45abd --- /dev/null +++ b/drivers/gpu/drm/scheduler/tests/sched_tests.h @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2025 Valve Corporation */ + +#ifndef _SCHED_TESTS_H_ +#define _SCHED_TESTS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * DOC: Mock DRM scheduler data structures + * + * drm_mock_* data structures are used to implement a mock "GPU". + * + * They subclass the core DRM scheduler objects and add their data on top, which + * enables tracking the submitted jobs and simulating their execution with the + * attributes as specified by the test case. + */ + +/** + * struct drm_mock_scheduler - implements a trivial mock GPU execution engine + * + * @base: DRM scheduler base class + * @test: Backpointer to owning the kunit test case + * @lock: Lock to protect the simulated @hw_timeline and @job_list + * @job_list: List of jobs submitted to the mock GPU + * @hw_timeline: Simulated hardware timeline has a @context, @next_seqno and + * @cur_seqno for implementing a struct dma_fence signaling the + * simulated job completion. + * + * Trivial mock GPU execution engine tracks submitted jobs and enables + * completing them strictly in submission order. + */ +struct drm_mock_scheduler { + struct drm_gpu_scheduler base; + + struct kunit *test; + + spinlock_t lock; + struct list_head job_list; + + struct { + u64 context; + atomic_t next_seqno; + unsigned int cur_seqno; + } hw_timeline; +}; + +/** + * struct drm_mock_sched_entity - implements a mock GPU sched entity + * + * @base: DRM scheduler entity base class + * @test: Backpointer to owning the kunit test case + * + * Mock GPU sched entity is used by the test cases to submit jobs to the mock + * scheduler. + */ +struct drm_mock_sched_entity { + struct drm_sched_entity base; + + struct kunit *test; +}; + +/** + * struct drm_mock_sched_job - implements a mock GPU job + * + * @base: DRM sched job base class + * @done: Completion signaling job completion. + * @flags: Flags designating job state. + * @link: List head element used by job tracking by the drm_mock_scheduler + * @timer: Timer used for simulating job execution duration + * @duration_us: Simulated job duration in micro seconds, or zero if in manual + * timeline advance mode + * @finish_at: Absolute time when the jobs with set duration will complete + * @lock: Lock used for @hw_fence + * @hw_fence: Fence returned to DRM scheduler as the hardware fence + * @test: Backpointer to owning the kunit test case + * + * Mock GPU sched job is used by the test cases to submit jobs to the mock + * scheduler. + */ +struct drm_mock_sched_job { + struct drm_sched_job base; + + struct completion done; + +#define DRM_MOCK_SCHED_JOB_DONE 0x1 +#define DRM_MOCK_SCHED_JOB_TIMEDOUT 0x2 +#define DRM_MOCK_SCHED_JOB_DONT_RESET 0x4 +#define DRM_MOCK_SCHED_JOB_RESET_SKIPPED 0x8 + unsigned long flags; + + struct list_head link; + struct hrtimer timer; + + unsigned int duration_us; + ktime_t finish_at; + + struct dma_fence hw_fence; + + struct kunit *test; +}; + +static inline struct drm_mock_scheduler * +drm_sched_to_mock_sched(struct drm_gpu_scheduler *sched) +{ + return container_of(sched, struct drm_mock_scheduler, base); +}; + +static inline struct drm_mock_sched_entity * +drm_sched_entity_to_mock_entity(struct drm_sched_entity *sched_entity) +{ + return container_of(sched_entity, struct drm_mock_sched_entity, base); +}; + +static inline struct drm_mock_sched_job * +drm_sched_job_to_mock_job(struct drm_sched_job *sched_job) +{ + return container_of(sched_job, struct drm_mock_sched_job, base); +}; + +struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, + long timeout); +void drm_mock_sched_fini(struct drm_mock_scheduler *sched); +unsigned int drm_mock_sched_advance(struct drm_mock_scheduler *sched, + unsigned int num); + +struct drm_mock_sched_entity * +drm_mock_sched_entity_new(struct kunit *test, + enum drm_sched_priority priority, + struct drm_mock_scheduler *sched); +void drm_mock_sched_entity_free(struct drm_mock_sched_entity *entity); + +struct drm_mock_sched_job * +drm_mock_sched_job_new(struct kunit *test, + struct drm_mock_sched_entity *entity); + +/** + * drm_mock_sched_job_submit - Arm and submit a job in one go + * + * @job: Job to arm and submit + */ +static inline void drm_mock_sched_job_submit(struct drm_mock_sched_job *job) +{ + drm_sched_job_arm(&job->base); + drm_sched_entity_push_job(&job->base); +} + +/** + * drm_mock_sched_job_set_duration_us - Set a job duration + * + * @job: Job to set the duration for + * @duration_us: Duration in micro seconds + * + * Jobs with duration set will be automatically completed by the mock scheduler + * as the timeline progresses, unless a job without a set duration is + * encountered in the timelime in which case calling drm_mock_sched_advance() + * will be required to bump the timeline. + */ +static inline void +drm_mock_sched_job_set_duration_us(struct drm_mock_sched_job *job, + unsigned int duration_us) +{ + job->duration_us = duration_us; +} + +/** + * drm_mock_sched_job_is_finished - Check if a job is finished + * + * @job: Job to check + * + * Returns: true if finished + */ +static inline bool +drm_mock_sched_job_is_finished(struct drm_mock_sched_job *job) +{ + return job->flags & DRM_MOCK_SCHED_JOB_DONE; +} + +/** + * drm_mock_sched_job_wait_finished - Wait until a job is finished + * + * @job: Job to wait for + * @timeout: Wait time in jiffies + * + * Returns: true if finished within the timeout provided, otherwise false + */ +static inline bool +drm_mock_sched_job_wait_finished(struct drm_mock_sched_job *job, long timeout) +{ + if (job->flags & DRM_MOCK_SCHED_JOB_DONE) + return true; + + return wait_for_completion_timeout(&job->done, timeout) != 0; +} + +/** + * drm_mock_sched_job_wait_scheduled - Wait until a job is scheduled + * + * @job: Job to wait for + * @timeout: Wait time in jiffies + * + * Returns: true if scheduled within the timeout provided, otherwise false + */ +static inline bool +drm_mock_sched_job_wait_scheduled(struct drm_mock_sched_job *job, long timeout) +{ + KUNIT_ASSERT_EQ(job->test, job->flags & DRM_MOCK_SCHED_JOB_DONE, 0); + + return dma_fence_wait_timeout(&job->base.s_fence->scheduled, + false, + timeout) != 0; +} + +#endif diff --git a/drivers/gpu/drm/scheduler/tests/tests_basic.c b/drivers/gpu/drm/scheduler/tests/tests_basic.c new file mode 100644 index 000000000..da5c49527 --- /dev/null +++ b/drivers/gpu/drm/scheduler/tests/tests_basic.c @@ -0,0 +1,922 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Valve Corporation */ + +#include +#include +#include +#include +#include + +#include "sched_tests.h" + +#define MOCK_TIMEOUT (HZ / 5) + +/* + * DRM scheduler basic tests should check the basic functional correctness of + * the scheduler, including some very light smoke testing. More targeted tests, + * for example focusing on testing specific bugs and other more complicated test + * scenarios, should be implemented in separate source units. + */ + +static int drm_sched_basic_init(struct kunit *test) +{ + test->priv = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + + return 0; +} + +static void drm_sched_basic_exit(struct kunit *test) +{ + struct drm_mock_scheduler *sched = test->priv; + + drm_mock_sched_fini(sched); +} + +static int drm_sched_timeout_init(struct kunit *test) +{ + test->priv = drm_mock_sched_new(test, MOCK_TIMEOUT); + + return 0; +} + +static void drm_sched_basic_submit(struct kunit *test) +{ + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_entity *entity; + struct drm_mock_sched_job *job; + unsigned int i; + bool done; + + /* + * Submit one job to the scheduler and verify that it gets scheduled + * and completed only when the mock hw backend processes it. + */ + + entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + job = drm_mock_sched_job_new(test, entity); + + drm_mock_sched_job_submit(job); + + done = drm_mock_sched_job_wait_scheduled(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + done = drm_mock_sched_job_wait_finished(job, HZ / 2); + KUNIT_ASSERT_FALSE(test, done); + + i = drm_mock_sched_advance(sched, 1); + KUNIT_ASSERT_EQ(test, i, 1); + + done = drm_mock_sched_job_wait_finished(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + drm_mock_sched_entity_free(entity); +} + +struct drm_sched_basic_params { + const char *description; + unsigned int queue_depth; + unsigned int num_entities; + unsigned int job_us; + bool dep_chain; +}; + +static const struct drm_sched_basic_params drm_sched_basic_cases[] = { + { + .description = "A queue of jobs in a single entity", + .queue_depth = 100, + .job_us = 1000, + .num_entities = 1, + }, + { + .description = "A chain of dependent jobs across multiple entities", + .queue_depth = 100, + .job_us = 1000, + .num_entities = 1, + .dep_chain = true, + }, + { + .description = "Multiple independent job queues", + .queue_depth = 100, + .job_us = 1000, + .num_entities = 4, + }, + { + .description = "Multiple inter-dependent job queues", + .queue_depth = 100, + .job_us = 1000, + .num_entities = 4, + .dep_chain = true, + }, +}; + +static void +drm_sched_basic_desc(const struct drm_sched_basic_params *params, char *desc) +{ + strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(drm_sched_basic, drm_sched_basic_cases, drm_sched_basic_desc); + +static void drm_sched_basic_test(struct kunit *test) +{ + const struct drm_sched_basic_params *params = test->param_value; + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_job *job, *prev = NULL; + struct drm_mock_sched_entity **entity; + unsigned int i, cur_ent = 0; + bool done; + + entity = kunit_kcalloc(test, params->num_entities, sizeof(*entity), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, entity); + + for (i = 0; i < params->num_entities; i++) + entity[i] = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + + for (i = 0; i < params->queue_depth; i++) { + job = drm_mock_sched_job_new(test, entity[cur_ent++]); + cur_ent %= params->num_entities; + drm_mock_sched_job_set_duration_us(job, params->job_us); + if (params->dep_chain && prev) + drm_sched_job_add_dependency(&job->base, + dma_fence_get(&prev->base.s_fence->finished)); + drm_mock_sched_job_submit(job); + prev = job; + } + + done = drm_mock_sched_job_wait_finished(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + for (i = 0; i < params->num_entities; i++) + drm_mock_sched_entity_free(entity[i]); +} + +static void drm_sched_basic_entity_cleanup(struct kunit *test) +{ + struct drm_mock_sched_job *job, *mid, *prev = NULL; + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_entity *entity[4]; + const unsigned int qd = 100; + unsigned int i, cur_ent = 0; + bool done; + + /* + * Submit a queue of jobs across different entities with an explicit + * chain of dependencies between them and trigger entity cleanup while + * the queue is still being processed. + */ + + for (i = 0; i < ARRAY_SIZE(entity); i++) + entity[i] = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + + for (i = 0; i < qd; i++) { + job = drm_mock_sched_job_new(test, entity[cur_ent++]); + cur_ent %= ARRAY_SIZE(entity); + drm_mock_sched_job_set_duration_us(job, 1000); + if (prev) + drm_sched_job_add_dependency(&job->base, + dma_fence_get(&prev->base.s_fence->finished)); + drm_mock_sched_job_submit(job); + if (i == qd / 2) + mid = job; + prev = job; + } + + done = drm_mock_sched_job_wait_finished(mid, HZ); + KUNIT_ASSERT_TRUE(test, done); + + /* Exit with half of the queue still pending to be executed. */ + for (i = 0; i < ARRAY_SIZE(entity); i++) + drm_mock_sched_entity_free(entity[i]); +} + +static struct kunit_case drm_sched_basic_tests[] = { + KUNIT_CASE(drm_sched_basic_submit), + KUNIT_CASE_PARAM(drm_sched_basic_test, drm_sched_basic_gen_params), + KUNIT_CASE(drm_sched_basic_entity_cleanup), + {} +}; + +static struct kunit_suite drm_sched_basic = { + .name = "drm_sched_basic_tests", + .init = drm_sched_basic_init, + .exit = drm_sched_basic_exit, + .test_cases = drm_sched_basic_tests, +}; + +static void drm_sched_basic_cancel(struct kunit *test) +{ + struct drm_mock_sched_entity *entity; + struct drm_mock_scheduler *sched; + struct drm_mock_sched_job *job; + bool done; + + /* + * Check that drm_sched_fini() uses the cancel_job() callback to cancel + * jobs that are still pending. + */ + + sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + entity = drm_mock_sched_entity_new(test, DRM_SCHED_PRIORITY_NORMAL, + sched); + + job = drm_mock_sched_job_new(test, entity); + + drm_mock_sched_job_submit(job); + + done = drm_mock_sched_job_wait_scheduled(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + drm_mock_sched_entity_free(entity); + drm_mock_sched_fini(sched); + + KUNIT_ASSERT_EQ(test, job->hw_fence.error, -ECANCELED); +} + +struct sched_concurrent_context { + struct drm_mock_scheduler *sched; + struct workqueue_struct *sub_wq; + struct kunit *test; + struct completion wait_go; +}; + +KUNIT_DEFINE_ACTION_WRAPPER(drm_mock_sched_fini_wrap, drm_mock_sched_fini, + struct drm_mock_scheduler *); + +KUNIT_DEFINE_ACTION_WRAPPER(drm_mock_sched_entity_free_wrap, drm_mock_sched_entity_free, + struct drm_mock_sched_entity *); + +static void complete_destroy_workqueue(void *context) +{ + struct sched_concurrent_context *ctx = context; + + complete_all(&ctx->wait_go); + + destroy_workqueue(ctx->sub_wq); +} + +static int drm_sched_concurrent_init(struct kunit *test) +{ + struct sched_concurrent_context *ctx; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + + init_completion(&ctx->wait_go); + + ctx->sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + + ret = kunit_add_action_or_reset(test, drm_mock_sched_fini_wrap, ctx->sched); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* Use an unbounded workqueue to maximize job submission concurrency */ + ctx->sub_wq = alloc_workqueue("drm-sched-submitters-wq", WQ_UNBOUND, + WQ_UNBOUND_MAX_ACTIVE); + KUNIT_ASSERT_NOT_NULL(test, ctx->sub_wq); + + ret = kunit_add_action_or_reset(test, complete_destroy_workqueue, ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + + ctx->test = test; + test->priv = ctx; + + return 0; +} + +struct drm_sched_parallel_params { + const char *description; + unsigned int num_jobs; + unsigned int num_workers; +}; + +static const struct drm_sched_parallel_params drm_sched_parallel_cases[] = { + { + .description = "Parallel submission of multiple jobs per worker", + .num_jobs = 8, + .num_workers = 16, + }, +}; + +static void +drm_sched_parallel_desc(const struct drm_sched_parallel_params *params, char *desc) +{ + strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(drm_sched_parallel, drm_sched_parallel_cases, drm_sched_parallel_desc); + +struct parallel_worker { + struct work_struct work; + struct sched_concurrent_context *ctx; + struct drm_mock_sched_entity *entity; + struct drm_mock_sched_job **jobs; + unsigned int id; +}; + +static void drm_sched_parallel_worker(struct work_struct *work) +{ + const struct drm_sched_parallel_params *params; + struct sched_concurrent_context *test_ctx; + struct parallel_worker *worker; + unsigned int i; + + worker = container_of(work, struct parallel_worker, work); + test_ctx = worker->ctx; + params = test_ctx->test->param_value; + + wait_for_completion(&test_ctx->wait_go); + + kunit_info(test_ctx->test, "Parallel worker %u submitting %u jobs started\n", + worker->id, params->num_jobs); + + for (i = 0; i < params->num_jobs; i++) + drm_mock_sched_job_submit(worker->jobs[i]); +} + +/* + * Spawns workers that submit a sequence of jobs to the mock scheduler. + * Once all jobs are submitted, the timeline is manually advanced. + */ +static void drm_sched_parallel_submit_test(struct kunit *test) +{ + struct sched_concurrent_context *ctx = test->priv; + const struct drm_sched_parallel_params *params = test->param_value; + struct parallel_worker *workers, *worker; + struct drm_mock_sched_job *job; + unsigned int i, j, completed_jobs, total_jobs; + bool done; + int ret; + + KUNIT_ASSERT_GT(test, params->num_workers, 0); + KUNIT_ASSERT_GT(test, params->num_jobs, 0); + + workers = kunit_kcalloc(test, params->num_workers, sizeof(*workers), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + + /* + * Init workers only after all jobs and entities have been successfully + * allocated. In this way, the cleanup logic for when an assertion fail + * can be simplified. + */ + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + worker->id = i; + worker->ctx = ctx; + worker->entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + ctx->sched); + + worker->jobs = kunit_kcalloc(test, params->num_jobs, + sizeof(*worker->jobs), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, worker->jobs); + + for (j = 0; j < params->num_jobs; j++) { + job = drm_mock_sched_job_new(test, worker->entity); + worker->jobs[j] = job; + } + + ret = kunit_add_action_or_reset(test, drm_mock_sched_entity_free_wrap, + worker->entity); + KUNIT_ASSERT_EQ(test, ret, 0); + } + + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + INIT_WORK(&worker->work, drm_sched_parallel_worker); + queue_work(ctx->sub_wq, &worker->work); + } + + complete_all(&ctx->wait_go); + flush_workqueue(ctx->sub_wq); + + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + for (j = 0; j < params->num_jobs; j++) { + job = worker->jobs[j]; + done = drm_mock_sched_job_wait_scheduled(job, HZ); + KUNIT_EXPECT_TRUE(test, done); + } + } + + total_jobs = params->num_workers * params->num_jobs; + completed_jobs = drm_mock_sched_advance(ctx->sched, total_jobs); + KUNIT_EXPECT_EQ(test, completed_jobs, total_jobs); + + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + for (j = 0; j < params->num_jobs; j++) { + job = worker->jobs[j]; + done = drm_mock_sched_job_wait_finished(job, HZ); + KUNIT_EXPECT_TRUE(test, done); + } + } +} + +struct drm_sched_interleaved_params { + const char *description; + unsigned int test_duration_ms; + unsigned int job_base_duration_us; + unsigned int num_workers; + unsigned int num_in_flight_jobs; +}; + +static const struct drm_sched_interleaved_params drm_sched_interleaved_cases[] = { + { + .description = "Interleaved submission of multiple jobs per worker", + .test_duration_ms = 1000, + .job_base_duration_us = 100, + .num_workers = 16, + .num_in_flight_jobs = 8, + }, +}; + +static void +drm_sched_interleaved_desc(const struct drm_sched_interleaved_params *params, char *desc) +{ + strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(drm_sched_interleaved, drm_sched_interleaved_cases, + drm_sched_interleaved_desc); + +struct interleaved_worker { + struct work_struct work; + struct sched_concurrent_context *ctx; + struct drm_mock_sched_entity *entity; + struct drm_mock_sched_job **jobs; + unsigned int id; + unsigned int job_count; + unsigned int job_duration_us; +}; + +static void drm_sched_interleaved_worker(struct work_struct *work) +{ + struct sched_concurrent_context *test_ctx; + const struct drm_sched_interleaved_params *params; + struct interleaved_worker *worker; + unsigned int i, j, max_in_flight_job; + unsigned long timeout; + bool done; + + worker = container_of(work, struct interleaved_worker, work); + test_ctx = worker->ctx; + params = test_ctx->test->param_value; + + wait_for_completion(&test_ctx->wait_go); + + kunit_info(test_ctx->test, "Worker %u submitting %u jobs of %u us started\n", + worker->id, worker->job_count, worker->job_duration_us); + + timeout = msecs_to_jiffies(params->test_duration_ms * 2); + + /* Fill the submission window */ + max_in_flight_job = min(worker->job_count, params->num_in_flight_jobs); + for (i = 0; i < max_in_flight_job; i++) + drm_mock_sched_job_submit(worker->jobs[i]); + + /* Keep the window full by submitting a new job at once until done */ + for (i = 0; i < worker->job_count; i++) { + done = drm_mock_sched_job_wait_finished(worker->jobs[i], timeout); + if (!done) + kunit_info(test_ctx->test, "Job %u of worker %u timed out\n", + i, worker->id); + + j = i + max_in_flight_job; + if (j < worker->job_count) + drm_mock_sched_job_submit(worker->jobs[j]); + } +} + +/* + * Spawns workers that submit a sequence of jobs to the mock scheduler. Job + * durations are chosen as multiples of a base duration value specified as + * a test parameter. Since the scheduler serializes jobs from all workers, + * the total test duration budget is divided into equal shares among workers. + * These shares are then used to compute the number of jobs that each worker + * can submit. + */ +static void drm_sched_interleaved_submit_test(struct kunit *test) +{ + const struct drm_sched_interleaved_params *params = test->param_value; + struct sched_concurrent_context *ctx = test->priv; + struct interleaved_worker *workers, *worker; + struct drm_mock_sched_job *job; + unsigned int worker_share_us; + unsigned int i, j; + bool done; + int ret; + + KUNIT_ASSERT_GT(test, params->num_workers, 0); + KUNIT_ASSERT_GT(test, params->job_base_duration_us, 0); + + workers = kunit_kcalloc(test, params->num_workers, sizeof(*workers), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + + /* Divide the available test time into equal shares among the workers */ + worker_share_us = (params->test_duration_ms * USEC_PER_MSEC) / + params->num_workers; + + /* + * Init workers only after all jobs and entities have been successfully + * allocated. In this way, the cleanup logic for when an assertion fails + * can be simplified. + */ + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + worker->id = i; + worker->ctx = ctx; + + worker->job_duration_us = params->job_base_duration_us * (i + 1); + worker->job_count = worker_share_us / worker->job_duration_us; + worker->job_count = max(1U, worker->job_count); + + worker->entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + ctx->sched); + + worker->jobs = kunit_kcalloc(test, worker->job_count, + sizeof(*worker->jobs), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, worker->jobs); + + for (j = 0; j < worker->job_count; j++) { + job = drm_mock_sched_job_new(test, worker->entity); + drm_mock_sched_job_set_duration_us(job, worker->job_duration_us); + + worker->jobs[j] = job; + } + + ret = kunit_add_action_or_reset(test, drm_mock_sched_entity_free_wrap, + worker->entity); + KUNIT_ASSERT_EQ(test, ret, 0); + } + + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + INIT_WORK(&worker->work, drm_sched_interleaved_worker); + queue_work(ctx->sub_wq, &worker->work); + } + + complete_all(&ctx->wait_go); + flush_workqueue(ctx->sub_wq); + + for (i = 0; i < params->num_workers; i++) { + worker = &workers[i]; + for (j = 0; j < worker->job_count; j++) { + job = worker->jobs[j]; + done = drm_mock_sched_job_is_finished(job); + KUNIT_EXPECT_TRUE(test, done); + } + } +} + +static struct kunit_case drm_sched_concurrent_tests[] = { + KUNIT_CASE_PARAM(drm_sched_parallel_submit_test, drm_sched_parallel_gen_params), + KUNIT_CASE_PARAM(drm_sched_interleaved_submit_test, drm_sched_interleaved_gen_params), + {} +}; + +static struct kunit_suite drm_sched_concurrent = { + .name = "drm_sched_concurrent_tests", + .init = drm_sched_concurrent_init, + .test_cases = drm_sched_concurrent_tests, + .attr = { + .speed = KUNIT_SPEED_SLOW, + }, +}; + +static struct kunit_case drm_sched_cancel_tests[] = { + KUNIT_CASE(drm_sched_basic_cancel), + {} +}; + +static struct kunit_suite drm_sched_cancel = { + .name = "drm_sched_basic_cancel_tests", + .init = drm_sched_basic_init, + .exit = drm_sched_basic_exit, + .test_cases = drm_sched_cancel_tests, +}; + +static void drm_sched_basic_timeout(struct kunit *test) +{ + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_entity *entity; + struct drm_mock_sched_job *job; + bool done; + + /* + * Submit a single job against a scheduler with the timeout configured + * and verify that the timeout handling will run if the backend fails + * to complete it in time. + */ + + entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + job = drm_mock_sched_job_new(test, entity); + + drm_mock_sched_job_submit(job); + + done = drm_mock_sched_job_wait_scheduled(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + done = drm_mock_sched_job_wait_finished(job, MOCK_TIMEOUT / 2); + KUNIT_ASSERT_FALSE(test, done); + + KUNIT_ASSERT_EQ(test, + job->flags & DRM_MOCK_SCHED_JOB_TIMEDOUT, + 0); + + done = drm_mock_sched_job_wait_finished(job, MOCK_TIMEOUT); + KUNIT_ASSERT_FALSE(test, done); + + KUNIT_ASSERT_EQ(test, + job->flags & DRM_MOCK_SCHED_JOB_TIMEDOUT, + DRM_MOCK_SCHED_JOB_TIMEDOUT); + + drm_mock_sched_entity_free(entity); +} + +static void drm_sched_skip_reset(struct kunit *test) +{ + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_entity *entity; + struct drm_mock_sched_job *job; + unsigned int i; + bool done; + + /* + * Submit a single job against a scheduler with the timeout configured + * and verify that if the job is still running, the timeout handler + * will skip the reset and allow the job to complete. + */ + + entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + job = drm_mock_sched_job_new(test, entity); + + job->flags = DRM_MOCK_SCHED_JOB_DONT_RESET; + + drm_mock_sched_job_submit(job); + + done = drm_mock_sched_job_wait_scheduled(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + done = drm_mock_sched_job_wait_finished(job, 2 * MOCK_TIMEOUT); + KUNIT_ASSERT_FALSE(test, done); + + KUNIT_ASSERT_EQ(test, + job->flags & DRM_MOCK_SCHED_JOB_RESET_SKIPPED, + DRM_MOCK_SCHED_JOB_RESET_SKIPPED); + + i = drm_mock_sched_advance(sched, 1); + KUNIT_ASSERT_EQ(test, i, 1); + + done = drm_mock_sched_job_wait_finished(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + drm_mock_sched_entity_free(entity); +} + +static struct kunit_case drm_sched_timeout_tests[] = { + KUNIT_CASE(drm_sched_basic_timeout), + KUNIT_CASE(drm_sched_skip_reset), + {} +}; + +static struct kunit_suite drm_sched_timeout = { + .name = "drm_sched_basic_timeout_tests", + .init = drm_sched_timeout_init, + .exit = drm_sched_basic_exit, + .test_cases = drm_sched_timeout_tests, +}; + +static void drm_sched_priorities(struct kunit *test) +{ + struct drm_mock_sched_entity *entity[DRM_SCHED_PRIORITY_COUNT]; + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_job *job; + const unsigned int qd = 100; + unsigned int i, cur_ent = 0; + enum drm_sched_priority p; + bool done; + + /* + * Submit a bunch of jobs against entities configured with different + * priorities. + */ + + BUILD_BUG_ON(DRM_SCHED_PRIORITY_KERNEL > DRM_SCHED_PRIORITY_LOW); + BUILD_BUG_ON(ARRAY_SIZE(entity) != DRM_SCHED_PRIORITY_COUNT); + + for (p = DRM_SCHED_PRIORITY_KERNEL; p <= DRM_SCHED_PRIORITY_LOW; p++) + entity[p] = drm_mock_sched_entity_new(test, p, sched); + + for (i = 0; i < qd; i++) { + job = drm_mock_sched_job_new(test, entity[cur_ent++]); + cur_ent %= ARRAY_SIZE(entity); + drm_mock_sched_job_set_duration_us(job, 1000); + drm_mock_sched_job_submit(job); + } + + done = drm_mock_sched_job_wait_finished(job, HZ); + KUNIT_ASSERT_TRUE(test, done); + + for (i = 0; i < ARRAY_SIZE(entity); i++) + drm_mock_sched_entity_free(entity[i]); +} + +static void drm_sched_change_priority(struct kunit *test) +{ + struct drm_mock_sched_entity *entity[DRM_SCHED_PRIORITY_COUNT]; + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_job *job; + const unsigned int qd = 1000; + unsigned int i, cur_ent = 0; + enum drm_sched_priority p; + + /* + * Submit a bunch of jobs against entities configured with different + * priorities and while waiting for them to complete, periodically keep + * changing their priorities. + * + * We set up the queue-depth (qd) and job duration so the priority + * changing loop has some time to interact with submissions to the + * backend and job completions as they progress. + */ + + for (p = DRM_SCHED_PRIORITY_KERNEL; p <= DRM_SCHED_PRIORITY_LOW; p++) + entity[p] = drm_mock_sched_entity_new(test, p, sched); + + for (i = 0; i < qd; i++) { + job = drm_mock_sched_job_new(test, entity[cur_ent++]); + cur_ent %= ARRAY_SIZE(entity); + drm_mock_sched_job_set_duration_us(job, 1000); + drm_mock_sched_job_submit(job); + } + + do { + drm_sched_entity_set_priority(&entity[cur_ent]->base, + (entity[cur_ent]->base.priority + 1) % + DRM_SCHED_PRIORITY_COUNT); + cur_ent++; + cur_ent %= ARRAY_SIZE(entity); + usleep_range(200, 500); + } while (!drm_mock_sched_job_is_finished(job)); + + for (i = 0; i < ARRAY_SIZE(entity); i++) + drm_mock_sched_entity_free(entity[i]); +} + +static struct kunit_case drm_sched_priority_tests[] = { + KUNIT_CASE(drm_sched_priorities), + KUNIT_CASE_SLOW(drm_sched_change_priority), + {} +}; + +static struct kunit_suite drm_sched_priority = { + .name = "drm_sched_basic_priority_tests", + .init = drm_sched_basic_init, + .exit = drm_sched_basic_exit, + .test_cases = drm_sched_priority_tests, +}; + +static void drm_sched_test_modify_sched(struct kunit *test) +{ + unsigned int i, cur_ent = 0, cur_sched = 0; + struct drm_mock_sched_entity *entity[13]; + struct drm_mock_scheduler *sched[3]; + struct drm_mock_sched_job *job; + const unsigned int qd = 1000; + + /* + * Submit a bunch of jobs against entities configured with different + * schedulers and while waiting for them to complete, periodically keep + * changing schedulers associated with each entity. + * + * We set up the queue-depth (qd) and job duration so the sched modify + * loop has some time to interact with submissions to the backend and + * job completions as they progress. + * + * For the number of schedulers and entities we use primes in order to + * perturb the entity->sched assignments with less of a regular pattern. + */ + + for (i = 0; i < ARRAY_SIZE(sched); i++) + sched[i] = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + + for (i = 0; i < ARRAY_SIZE(entity); i++) + entity[i] = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched[i % ARRAY_SIZE(sched)]); + + for (i = 0; i < qd; i++) { + job = drm_mock_sched_job_new(test, entity[cur_ent++]); + cur_ent %= ARRAY_SIZE(entity); + drm_mock_sched_job_set_duration_us(job, 1000); + drm_mock_sched_job_submit(job); + } + + do { + struct drm_gpu_scheduler *modify; + + usleep_range(200, 500); + cur_ent++; + cur_ent %= ARRAY_SIZE(entity); + cur_sched++; + cur_sched %= ARRAY_SIZE(sched); + modify = &sched[cur_sched]->base; + drm_sched_entity_modify_sched(&entity[cur_ent]->base, &modify, + 1); + } while (!drm_mock_sched_job_is_finished(job)); + + for (i = 0; i < ARRAY_SIZE(entity); i++) + drm_mock_sched_entity_free(entity[i]); + + for (i = 0; i < ARRAY_SIZE(sched); i++) + drm_mock_sched_fini(sched[i]); +} + +static struct kunit_case drm_sched_modify_sched_tests[] = { + KUNIT_CASE(drm_sched_test_modify_sched), + {} +}; + +static struct kunit_suite drm_sched_modify_sched = { + .name = "drm_sched_basic_modify_sched_tests", + .test_cases = drm_sched_modify_sched_tests, +}; + +static void drm_sched_test_credits(struct kunit *test) +{ + struct drm_mock_sched_entity *entity; + struct drm_mock_scheduler *sched; + struct drm_mock_sched_job *job[2]; + bool done; + int i; + + /* + * Check that the configured credit limit is respected. + */ + + sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + sched->base.credit_limit = 1; + + entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + + job[0] = drm_mock_sched_job_new(test, entity); + job[1] = drm_mock_sched_job_new(test, entity); + + drm_mock_sched_job_submit(job[0]); + drm_mock_sched_job_submit(job[1]); + + done = drm_mock_sched_job_wait_scheduled(job[0], HZ); + KUNIT_ASSERT_TRUE(test, done); + + done = drm_mock_sched_job_wait_scheduled(job[1], HZ); + KUNIT_ASSERT_FALSE(test, done); + + i = drm_mock_sched_advance(sched, 1); + KUNIT_ASSERT_EQ(test, i, 1); + + done = drm_mock_sched_job_wait_scheduled(job[1], HZ); + KUNIT_ASSERT_TRUE(test, done); + + i = drm_mock_sched_advance(sched, 1); + KUNIT_ASSERT_EQ(test, i, 1); + + done = drm_mock_sched_job_wait_finished(job[1], HZ); + KUNIT_ASSERT_TRUE(test, done); + + drm_mock_sched_entity_free(entity); + drm_mock_sched_fini(sched); +} + +static struct kunit_case drm_sched_credits_tests[] = { + KUNIT_CASE_SLOW(drm_sched_test_credits), + {} +}; + +static struct kunit_suite drm_sched_credits = { + .name = "drm_sched_basic_credits_tests", + .test_cases = drm_sched_credits_tests, +}; + +kunit_test_suites(&drm_sched_basic, + &drm_sched_concurrent, + &drm_sched_timeout, + &drm_sched_cancel, + &drm_sched_priority, + &drm_sched_modify_sched, + &drm_sched_credits); diff --git a/drivers/gpu/drm/scheduler/tests/tests_scheduler.c b/drivers/gpu/drm/scheduler/tests/tests_scheduler.c new file mode 100644 index 000000000..285546a22 --- /dev/null +++ b/drivers/gpu/drm/scheduler/tests/tests_scheduler.c @@ -0,0 +1,874 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Valve Corporation */ + +#include +#include +#include +#include + +#include "sched_tests.h" + +/* + * DRM scheduler tests exercise load balancing decisions ie. entity selection + * logic. + */ + +static int drm_sched_scheduler_init(struct kunit *test) +{ + struct drm_mock_scheduler *sched; + + sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + sched->base.credit_limit = 1; + + test->priv = sched; + + return 0; +} + +static int drm_sched_scheduler_init2(struct kunit *test) +{ + struct drm_mock_scheduler *sched; + + sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT); + sched->base.credit_limit = 2; + + test->priv = sched; + + return 0; +} + +static void drm_sched_scheduler_exit(struct kunit *test) +{ + struct drm_mock_scheduler *sched = test->priv; + + drm_mock_sched_fini(sched); +} + +static void drm_sched_scheduler_queue_overhead(struct kunit *test) +{ + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_entity *entity; + const unsigned int job_us = 1000; + const unsigned int jobs = 1000; + const unsigned int total_us = jobs * job_us; + struct drm_mock_sched_job *job, *first; + ktime_t start, end; + bool done; + int i; + + /* + * Deep queue job at a time processing (single credit). + * + * This measures the overhead of picking and processing a job at a time + * by comparing the ideal total "GPU" time of all submitted jobs versus + * the time actually taken. + */ + + KUNIT_ASSERT_EQ(test, sched->base.credit_limit, 1); + + entity = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + + for (i = 0; i <= jobs; i++) { + job = drm_mock_sched_job_new(test, entity); + if (i == 0) + first = job; /* Extra first job blocks the queue */ + else + drm_mock_sched_job_set_duration_us(job, job_us); + drm_mock_sched_job_submit(job); + } + + done = drm_mock_sched_job_wait_scheduled(first, HZ); + KUNIT_ASSERT_TRUE(test, done); + + start = ktime_get(); + i = drm_mock_sched_advance(sched, 1); /* Release the queue */ + KUNIT_ASSERT_EQ(test, i, 1); + + /* Wait with a safe margin to avoid every failing. */ + done = drm_mock_sched_job_wait_finished(job, + usecs_to_jiffies(total_us) * 5); + end = ktime_get(); + KUNIT_ASSERT_TRUE(test, done); + + pr_info("Expected %uus, actual %lldus\n", + total_us, + ktime_to_us(ktime_sub(end, start))); + + drm_mock_sched_entity_free(entity); +} + +static void drm_sched_scheduler_ping_pong(struct kunit *test) +{ + struct drm_mock_sched_job *job, *first, *prev = NULL; + struct drm_mock_scheduler *sched = test->priv; + struct drm_mock_sched_entity *entity[2]; + const unsigned int job_us = 1000; + const unsigned int jobs = 1000; + const unsigned int total_us = jobs * job_us; + ktime_t start, end; + bool done; + int i; + + /* + * Two entitites in inter-dependency chain. + * + * This measures the overhead of picking and processing a job at a time, + * where each job depends on the previous one from the diffferent + * entity, by comparing the ideal total "GPU" time of all submitted jobs + * versus the time actually taken. + */ + + KUNIT_ASSERT_EQ(test, sched->base.credit_limit, 1); + + for (i = 0; i < ARRAY_SIZE(entity); i++) + entity[i] = drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + + for (i = 0; i <= jobs; i++) { + job = drm_mock_sched_job_new(test, entity[i & 1]); + if (i == 0) + first = job; /* Extra first job blocks the queue */ + else + drm_mock_sched_job_set_duration_us(job, job_us); + if (prev) + drm_sched_job_add_dependency(&job->base, + dma_fence_get(&prev->base.s_fence->finished)); + drm_mock_sched_job_submit(job); + prev = job; + } + + done = drm_mock_sched_job_wait_scheduled(first, HZ); + KUNIT_ASSERT_TRUE(test, done); + + start = ktime_get(); + i = drm_mock_sched_advance(sched, 1); /* Release the queue */ + KUNIT_ASSERT_EQ(test, i, 1); + + /* Wait with a safe margin to avoid every failing. */ + done = drm_mock_sched_job_wait_finished(job, + usecs_to_jiffies(total_us) * 5); + end = ktime_get(); + KUNIT_ASSERT_TRUE(test, done); + + pr_info("Expected %uus, actual %lldus\n", + total_us, + ktime_to_us(ktime_sub(end, start))); + + for (i = 0; i < ARRAY_SIZE(entity); i++) + drm_mock_sched_entity_free(entity[i]); +} + +static struct kunit_case drm_sched_scheduler_overhead_tests[] = { + KUNIT_CASE_SLOW(drm_sched_scheduler_queue_overhead), + KUNIT_CASE_SLOW(drm_sched_scheduler_ping_pong), + {} +}; + +static struct kunit_suite drm_sched_scheduler_overhead = { + .name = "drm_sched_scheduler_overhead_tests", + .init = drm_sched_scheduler_init, + .exit = drm_sched_scheduler_exit, + .test_cases = drm_sched_scheduler_overhead_tests, +}; + +/* + * struct drm_sched_client_params - describe a workload emitted from a client + * + * A simulated client will create an entity with a scheduling @priority and emit + * jobs in a loop where each iteration will consist of: + * + * 1. Submit @job_cnt jobs, each with a set duration of @job_us. + * 2. If @sync is true wait for last submitted job to finish. + * 3. Sleep for @wait_us micro-seconds. + * 4. Repeat. + */ +struct drm_sched_client_params { + enum drm_sched_priority priority; + unsigned int job_cnt; + unsigned int job_us; + bool sync; + unsigned int wait_us; +}; + +struct drm_sched_test_params { + const char *description; + unsigned int num_clients; + struct drm_sched_client_params client[2]; +}; + +static const struct drm_sched_test_params drm_sched_cases[] = { + { + .description = "Normal priority and normal priority", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + }, + { + .description = "Normal priority and low priority", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_LOW, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + }, + { + .description = "High priority and normal priority", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_HIGH, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + }, + { + .description = "High priority and low priority", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_HIGH, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_LOW, + .job_cnt = 1, + .job_us = 8000, + .wait_us = 0, + .sync = false, + }, + }, + { + .description = "50% and 50%", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 1500, + .wait_us = 1500, + .sync = true, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 2500, + .wait_us = 2500, + .sync = true, + }, + }, + { + .description = "50% and 50% low priority", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 1500, + .wait_us = 1500, + .sync = true, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_LOW, + .job_cnt = 1, + .job_us = 2500, + .wait_us = 2500, + .sync = true, + }, + }, + { + .description = "50% high priority and 50%", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_HIGH, + .job_cnt = 1, + .job_us = 1500, + .wait_us = 1500, + .sync = true, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 2500, + .wait_us = 2500, + .sync = true, + }, + }, + { + .description = "Low priority hog and interactive client", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_LOW, + .job_cnt = 3, + .job_us = 2500, + .wait_us = 500, + .sync = false, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 500, + .wait_us = 10000, + .sync = true, + }, + }, + { + .description = "Heavy rendering and interactive client", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 3, + .job_us = 2500, + .wait_us = 2500, + .sync = true, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 1000, + .wait_us = 9000, + .sync = true, + }, + }, + { + .description = "Very heavy rendering and interactive client", + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 4, + .job_us = 50000, + .wait_us = 1, + .sync = true, + }, + .client[1] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 1, + .job_us = 1000, + .wait_us = 9000, + .sync = true, + }, + }, +}; + +static void +drm_sched_desc(const struct drm_sched_test_params *params, char *desc) +{ + strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(drm_sched_scheduler_two_clients, + drm_sched_cases, + drm_sched_desc); + +/* + * struct test_client_stats - track client stats + * + * For each client executing a simulated workload we track some timings for + * which we are interested in the minimum of all iterations (@min_us), maximum + * (@max_us) and the overall total for all iterations (@tot_us). + */ +struct test_client_stats { + unsigned int min_us; + unsigned int max_us; + unsigned long tot_us; +}; + +/* + * struct test_client - a simulated userspace client submitting scheduler work + * + * Each client executing a simulated workload is represented by one of these. + * + * Each of them instantiates a scheduling @entity and executes a workloads as + * defined in @params. Based on those @params the theoretical execution time of + * the client is calculated as @ideal_duration, while the actual wall time is + * tracked in @duration (calculated based on the @start and @end client time- + * stamps). + * + * Numerical @id is assigned to each for logging purposes. + * + * @worker and @work are used to provide an independent execution context from + * which scheduler jobs are submitted. + * + * During execution statistics on how long it took to submit and execute one + * iteration (whether or not synchronous) is kept in @cycle_time, while + * @latency_time tracks the @cycle_time minus the ideal duration of the one + * cycle. + * + * Once the client has completed the set number of iterations it will write the + * completion status into @done. + */ +struct test_client { + struct kunit *test; /* Backpointer to the kunit test. */ + + struct drm_mock_sched_entity *entity; + struct kthread_worker *worker; + struct kthread_work work; + + struct drm_sched_client_params params; + + unsigned int id; + ktime_t duration; + ktime_t ideal_duration; + unsigned int cycles; + unsigned int cycle; + ktime_t start; + ktime_t end; + bool done; + + struct test_client_stats cycle_time; + struct test_client_stats latency_time; +}; + +static void +update_stats(struct test_client_stats *stats, unsigned int us) +{ + if (us > stats->max_us) + stats->max_us = us; + if (us < stats->min_us) + stats->min_us = us; + stats->tot_us += us; +} + +static unsigned int +get_stats_avg(struct test_client_stats *stats, unsigned int cycles) +{ + return div_u64(stats->tot_us, cycles); +} + +static void drm_sched_client_work(struct kthread_work *work) +{ + struct test_client *client = container_of(work, typeof(*client), work); + const long sync_wait = MAX_SCHEDULE_TIMEOUT; + unsigned int cycle, work_us, period_us; + struct drm_mock_sched_job *job = NULL; + + work_us = client->params.job_cnt * client->params.job_us; + period_us = work_us + client->params.wait_us; + client->cycles = + DIV_ROUND_UP((unsigned int)ktime_to_us(client->duration), + period_us); + client->ideal_duration = us_to_ktime(client->cycles * period_us); + + client->start = ktime_get(); + + for (cycle = 0; cycle < client->cycles; cycle++) { + ktime_t cycle_time; + unsigned int batch; + unsigned long us; + + if (READ_ONCE(client->done)) + break; + + cycle_time = ktime_get(); + for (batch = 0; batch < client->params.job_cnt; batch++) { + job = drm_mock_sched_job_new(client->test, + client->entity); + drm_mock_sched_job_set_duration_us(job, + client->params.job_us); + drm_mock_sched_job_submit(job); + } + + if (client->params.sync) + drm_mock_sched_job_wait_finished(job, sync_wait); + + cycle_time = ktime_sub(ktime_get(), cycle_time); + us = ktime_to_us(cycle_time); + update_stats(&client->cycle_time, us); + if (ktime_to_us(cycle_time) >= (long)work_us) + us = ktime_to_us(cycle_time) - work_us; + else if (WARN_ON_ONCE(client->params.sync)) /* GPU job took less than expected. */ + us = 0; + update_stats(&client->latency_time, us); + WRITE_ONCE(client->cycle, cycle); + + if (READ_ONCE(client->done)) + break; + + if (client->params.wait_us) + fsleep(client->params.wait_us); + else if (!client->params.sync) + cond_resched(); /* Do not hog the CPU if fully async. */ + } + + client->done = drm_mock_sched_job_wait_finished(job, sync_wait); + client->end = ktime_get(); +} + +static const char *prio_str(enum drm_sched_priority prio) +{ + switch (prio) { + case DRM_SCHED_PRIORITY_KERNEL: + return "kernel"; + case DRM_SCHED_PRIORITY_LOW: + return "low"; + case DRM_SCHED_PRIORITY_NORMAL: + return "normal"; + case DRM_SCHED_PRIORITY_HIGH: + return "high"; + default: + return "???"; + } +} + +static bool client_done(struct test_client *client) +{ + return READ_ONCE(client->done); /* READ_ONCE to document lockless read from a loop. */ +} + +static void drm_sched_scheduler_two_clients_test(struct kunit *test) +{ + const struct drm_sched_test_params *params = test->param_value; + struct drm_mock_scheduler *sched = test->priv; + struct test_client client[2] = { }; + unsigned int prev_cycle[2] = { }; + unsigned int i, j; + ktime_t start; + + /* + * Same job stream from two clients. + */ + + for (i = 0; i < ARRAY_SIZE(client); i++) + client[i].entity = + drm_mock_sched_entity_new(test, + params->client[i].priority, + sched); + + for (i = 0; i < ARRAY_SIZE(client); i++) { + client[i].test = test; + client[i].id = i; + client[i].duration = ms_to_ktime(1000); + client[i].params = params->client[i]; + client[i].cycle_time.min_us = ~0U; + client[i].latency_time.min_us = ~0U; + client[i].worker = + kthread_create_worker(0, "%s-%u", __func__, i); + if (IS_ERR(client[i].worker)) { + for (j = 0; j < i; j++) + kthread_destroy_worker(client[j].worker); + KUNIT_FAIL(test, "Failed to create worker!\n"); + } + + kthread_init_work(&client[i].work, drm_sched_client_work); + } + + for (i = 0; i < ARRAY_SIZE(client); i++) + kthread_queue_work(client[i].worker, &client[i].work); + + /* + * The clients (workers) can be a mix of async (deep submission queue), + * sync (one job at a time), or something in between. Therefore it is + * difficult to display a single metric representing their progress. + * + * Each struct drm_sched_client_params describes the actual submission + * pattern which happens in the following steps: + * 1. Submit N jobs + * 2. Wait for last submitted job to finish + * 3. Sleep for U micro-seconds + * 4. Goto 1. for C cycles + * + * Where number of cycles is calculated to match the target client + * duration from the respective struct drm_sched_test_params. + * + * To asses scheduling behaviour what we output for both clients is: + * - pct: Percentage progress of the jobs submitted + * - cps: "Cycles" per second (where one cycle is one complete + * iteration from the above) + * - qd: Number of outstanding jobs in the client/entity + */ + + pr_info(" [pct] - Job submission progress\n" +" [cps] - Cycles per second\n" +" [qd] - Number of outstanding jobs in the client/entity\n"); + pr_info("%s:\n\t pct1 cps1 qd1; pct2 cps2 qd2\n", + params->description); + start = ktime_get(); + while (!client_done(&client[0]) || !client_done(&client[1])) { + const unsigned int period_ms = 100; + const unsigned int frequency = 1000 / period_ms; + unsigned int pct[2], qd[2], cycle[2], cps[2]; + + for (i = 0; i < ARRAY_SIZE(client); i++) { + qd[i] = spsc_queue_count(&client[i].entity->base.job_queue); + cycle[i] = READ_ONCE(client[i].cycle); + cps[i] = DIV_ROUND_UP(100 * frequency * + (cycle[i] - prev_cycle[i]), + 100); + if (client[i].cycles) + pct[i] = DIV_ROUND_UP(100 * (1 + cycle[i]), + client[i].cycles); + else + pct[i] = 0; + prev_cycle[i] = cycle[i]; + } + + if (client_done(&client[0])) + pr_info("\t+%6lldms: ; %3u %5u %4u\n", + ktime_to_ms(ktime_sub(ktime_get(), start)), + pct[1], cps[1], qd[1]); + else if (client_done(&client[1])) + pr_info("\t+%6lldms: %3u %5u %4u;\n", + ktime_to_ms(ktime_sub(ktime_get(), start)), + pct[0], cps[0], qd[0]); + else + pr_info("\t+%6lldms: %3u %5u %4u; %3u %5u %4u\n", + ktime_to_ms(ktime_sub(ktime_get(), start)), + pct[0], cps[0], qd[0], + pct[1], cps[1], qd[1]); + + msleep(period_ms); + } + + for (i = 0; i < ARRAY_SIZE(client); i++) { + kthread_flush_work(&client[i].work); + kthread_destroy_worker(client[i].worker); + } + + for (i = 0; i < ARRAY_SIZE(client); i++) + KUNIT_ASSERT_TRUE(test, client[i].done); + + for (i = 0; i < ARRAY_SIZE(client); i++) { + pr_info(" %u: prio=%s sync=%u elapsed_ms=%lldms (ideal_ms=%lldms) cycle_time(min,avg,max)=%u,%u,%u us latency_time(min,avg,max)=%u,%u,%u us", + i, + prio_str(params->client[i].priority), + params->client[i].sync, + ktime_to_ms(ktime_sub(client[i].end, client[i].start)), + ktime_to_ms(client[i].ideal_duration), + client[i].cycle_time.min_us, + get_stats_avg(&client[i].cycle_time, client[i].cycles), + client[i].cycle_time.max_us, + client[i].latency_time.min_us, + get_stats_avg(&client[i].latency_time, client[i].cycles), + client[i].latency_time.max_us); + drm_mock_sched_entity_free(client[i].entity); + } +} + +static struct kunit_case drm_sched_scheduler_two_clients_tests[] = { + KUNIT_CASE_PARAM_ATTR(drm_sched_scheduler_two_clients_test, + drm_sched_scheduler_two_clients_gen_params, + { .speed = KUNIT_SPEED_SLOW }), + {} +}; + +static struct kunit_suite drm_sched_scheduler_two_clients1 = { + .name = "drm_sched_scheduler_two_clients_one_credit_tests", + .init = drm_sched_scheduler_init, + .exit = drm_sched_scheduler_exit, + .test_cases = drm_sched_scheduler_two_clients_tests, +}; + +static struct kunit_suite drm_sched_scheduler_two_clients2 = { + .name = "drm_sched_scheduler_two_clients_two_credits_tests", + .init = drm_sched_scheduler_init2, + .exit = drm_sched_scheduler_exit, + .test_cases = drm_sched_scheduler_two_clients_tests, +}; + +static const struct drm_sched_test_params drm_sched_many_cases[] = { + { + .description = "2 clients", + .num_clients = 2, + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 4, + .job_us = 1000, + .wait_us = 0, + .sync = true, + }, + }, + { + .description = "3 clients", + .num_clients = 3, + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 4, + .job_us = 1000, + .wait_us = 0, + .sync = true, + }, + }, + { + .description = "7 clients", + .num_clients = 7, + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 4, + .job_us = 1000, + .wait_us = 0, + .sync = true, + }, + }, + { + .description = "13 clients", + .num_clients = 13, + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 4, + .job_us = 1000, + .wait_us = 0, + .sync = true, + }, + }, + { + .description = "31 clients", + .num_clients = 31, + .client[0] = { + .priority = DRM_SCHED_PRIORITY_NORMAL, + .job_cnt = 2, + .job_us = 1000, + .wait_us = 0, + .sync = true, + }, + }, +}; + +KUNIT_ARRAY_PARAM(drm_sched_scheduler_many_clients, + drm_sched_many_cases, + drm_sched_desc); + +static void drm_sched_scheduler_many_clients_test(struct kunit *test) +{ + const struct drm_sched_test_params *params = test->param_value; + struct drm_mock_scheduler *sched = test->priv; + const unsigned int clients = params->num_clients; + unsigned int i, j, delta_total = 0, loops = 0; + struct test_client *client; + unsigned int *prev_cycle; + ktime_t start; + char *buf; + + /* + * Many clients with deep-ish async queues. + */ + + buf = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buf); + client = kunit_kcalloc(test, clients, sizeof(*client), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, client); + prev_cycle = kunit_kcalloc(test, clients, sizeof(*prev_cycle), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prev_cycle); + + for (i = 0; i < clients; i++) + client[i].entity = + drm_mock_sched_entity_new(test, + DRM_SCHED_PRIORITY_NORMAL, + sched); + + for (i = 0; i < clients; i++) { + client[i].test = test; + client[i].id = i; + client[i].params = params->client[0]; + client[i].duration = ms_to_ktime(1000 / clients); + client[i].cycle_time.min_us = ~0U; + client[i].latency_time.min_us = ~0U; + client[i].worker = + kthread_create_worker(0, "%s-%u", __func__, i); + if (IS_ERR(client[i].worker)) { + for (j = 0; j < i; j++) + kthread_destroy_worker(client[j].worker); + KUNIT_FAIL(test, "Failed to create worker!\n"); + } + + kthread_init_work(&client[i].work, drm_sched_client_work); + } + + for (i = 0; i < clients; i++) + kthread_queue_work(client[i].worker, &client[i].work); + + start = ktime_get(); + pr_info("%u clients:\n\tt\t\tcycle:\t min avg max : ...\n", clients); + for (;;) { + unsigned int min = ~0; + unsigned int max = 0; + unsigned int total = 0; + bool done = true; + char pbuf[16]; + + memset(buf, 0, PAGE_SIZE); + for (i = 0; i < clients; i++) { + unsigned int cycle, cycles; + + /* Read current progress from the threaded worker. */ + cycle = READ_ONCE(client[i].cycle); + cycles = READ_ONCE(client[i].cycles); + + snprintf(pbuf, sizeof(pbuf), " %3d", cycle); + strncat(buf, pbuf, PAGE_SIZE); + + total += cycle; + if (cycle < min) + min = cycle; + if (cycle > max) + max = cycle; + + if (!min || (cycle + 1) < cycles) + done = false; + } + + loops++; + delta_total += max - min; + + pr_info("\t+%6lldms\t\t %3u %3u %3u :%s\n", + ktime_to_ms(ktime_sub(ktime_get(), start)), + min, DIV_ROUND_UP(total, clients), max, buf); + + if (done) + break; + + msleep(100); + } + + pr_info(" avg_max_min_delta(x100)=%u\n", + loops ? DIV_ROUND_UP(delta_total * 100, loops) : 0); + + for (i = 0; i < clients; i++) { + kthread_flush_work(&client[i].work); + kthread_destroy_worker(client[i].worker); + } + + for (i = 0; i < clients; i++) + drm_mock_sched_entity_free(client[i].entity); +} + +static struct kunit_case drm_sched_scheduler_many_clients_tests[] = { + KUNIT_CASE_PARAM_ATTR(drm_sched_scheduler_many_clients_test, + drm_sched_scheduler_many_clients_gen_params, + { .speed = KUNIT_SPEED_SLOW }), + {} +}; + +static struct kunit_suite drm_sched_scheduler_many_clients = { + .name = "drm_sched_scheduler_many_clients_tests", + .init = drm_sched_scheduler_init2, + .exit = drm_sched_scheduler_exit, + .test_cases = drm_sched_scheduler_many_clients_tests, +}; + +kunit_test_suites(&drm_sched_scheduler_overhead, + &drm_sched_scheduler_two_clients1, + &drm_sched_scheduler_two_clients2, + &drm_sched_scheduler_many_clients); -- cgit v1.3.1