From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- drivers/gpu/drm/ttm/Makefile | 11 + drivers/gpu/drm/ttm/tests/.kunitconfig | 3 + drivers/gpu/drm/ttm/tests/Makefile | 11 + drivers/gpu/drm/ttm/tests/TODO | 27 + drivers/gpu/drm/ttm/tests/ttm_bo_test.c | 637 +++++++++ drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c | 1217 +++++++++++++++++ drivers/gpu/drm/ttm/tests/ttm_device_test.c | 206 +++ drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c | 304 +++++ drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h | 52 + drivers/gpu/drm/ttm/tests/ttm_mock_manager.c | 238 ++++ drivers/gpu/drm/ttm/tests/ttm_mock_manager.h | 30 + drivers/gpu/drm/ttm/tests/ttm_pool_test.c | 431 ++++++ drivers/gpu/drm/ttm/tests/ttm_resource_test.c | 337 +++++ drivers/gpu/drm/ttm/tests/ttm_tt_test.c | 402 ++++++ drivers/gpu/drm/ttm/ttm_agp_backend.c | 145 ++ drivers/gpu/drm/ttm/ttm_backup.c | 226 ++++ drivers/gpu/drm/ttm/ttm_bo.c | 1556 +++++++++++++++++++++ drivers/gpu/drm/ttm/ttm_bo_internal.h | 60 + drivers/gpu/drm/ttm/ttm_bo_util.c | 1167 ++++++++++++++++ drivers/gpu/drm/ttm/ttm_bo_vm.c | 514 +++++++ drivers/gpu/drm/ttm/ttm_device.c | 319 +++++ drivers/gpu/drm/ttm/ttm_execbuf_util.c | 163 +++ drivers/gpu/drm/ttm/ttm_module.c | 92 ++ drivers/gpu/drm/ttm/ttm_module.h | 43 + drivers/gpu/drm/ttm/ttm_pool.c | 1564 ++++++++++++++++++++++ drivers/gpu/drm/ttm/ttm_pool_internal.h | 33 + drivers/gpu/drm/ttm/ttm_range_manager.c | 242 ++++ drivers/gpu/drm/ttm/ttm_resource.c | 1026 ++++++++++++++ drivers/gpu/drm/ttm/ttm_sys_manager.c | 49 + drivers/gpu/drm/ttm/ttm_tt.c | 565 ++++++++ 30 files changed, 11670 insertions(+) create mode 100644 drivers/gpu/drm/ttm/Makefile create mode 100644 drivers/gpu/drm/ttm/tests/.kunitconfig create mode 100644 drivers/gpu/drm/ttm/tests/Makefile create mode 100644 drivers/gpu/drm/ttm/tests/TODO create mode 100644 drivers/gpu/drm/ttm/tests/ttm_bo_test.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_device_test.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h create mode 100644 drivers/gpu/drm/ttm/tests/ttm_mock_manager.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_mock_manager.h create mode 100644 drivers/gpu/drm/ttm/tests/ttm_pool_test.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_resource_test.c create mode 100644 drivers/gpu/drm/ttm/tests/ttm_tt_test.c create mode 100644 drivers/gpu/drm/ttm/ttm_agp_backend.c create mode 100644 drivers/gpu/drm/ttm/ttm_backup.c create mode 100644 drivers/gpu/drm/ttm/ttm_bo.c create mode 100644 drivers/gpu/drm/ttm/ttm_bo_internal.h create mode 100644 drivers/gpu/drm/ttm/ttm_bo_util.c create mode 100644 drivers/gpu/drm/ttm/ttm_bo_vm.c create mode 100644 drivers/gpu/drm/ttm/ttm_device.c create mode 100644 drivers/gpu/drm/ttm/ttm_execbuf_util.c create mode 100644 drivers/gpu/drm/ttm/ttm_module.c create mode 100644 drivers/gpu/drm/ttm/ttm_module.h create mode 100644 drivers/gpu/drm/ttm/ttm_pool.c create mode 100644 drivers/gpu/drm/ttm/ttm_pool_internal.h create mode 100644 drivers/gpu/drm/ttm/ttm_range_manager.c create mode 100644 drivers/gpu/drm/ttm/ttm_resource.c create mode 100644 drivers/gpu/drm/ttm/ttm_sys_manager.c create mode 100644 drivers/gpu/drm/ttm/ttm_tt.c (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile new file mode 100644 index 000000000..40d07a352 --- /dev/null +++ b/drivers/gpu/drm/ttm/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the drm device driver. This driver provides support for the + +ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ + ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \ + ttm_device.o ttm_sys_manager.o ttm_backup.o +ttm-$(CONFIG_AGP) += ttm_agp_backend.o + +obj-$(CONFIG_DRM_TTM) += ttm.o +obj-$(CONFIG_DRM_TTM_KUNIT_TEST) += tests/ diff --git a/drivers/gpu/drm/ttm/tests/.kunitconfig b/drivers/gpu/drm/ttm/tests/.kunitconfig new file mode 100644 index 000000000..1ae1ffabd --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_DRM=y +CONFIG_DRM_TTM_KUNIT_TEST=y diff --git a/drivers/gpu/drm/ttm/tests/Makefile b/drivers/gpu/drm/ttm/tests/Makefile new file mode 100644 index 000000000..f3149de77 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 AND MIT + +obj-$(CONFIG_DRM_TTM_KUNIT_TEST) += \ + ttm_device_test.o \ + ttm_pool_test.o \ + ttm_resource_test.o \ + ttm_tt_test.o \ + ttm_bo_test.o \ + ttm_bo_validate_test.o \ + ttm_mock_manager.o \ + ttm_kunit_helpers.o diff --git a/drivers/gpu/drm/ttm/tests/TODO b/drivers/gpu/drm/ttm/tests/TODO new file mode 100644 index 000000000..45b03d184 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/TODO @@ -0,0 +1,27 @@ +TODO +===== + +- Add a test case where the only evictable BO is busy +- Update eviction tests so they use parametrized "from" memory type +- Improve mock manager's implementation, e.g. allocate a block of + dummy memory that can be used when testing page mapping functions +- Suggestion: Add test cases with external BOs +- Suggestion: randomize the number and size of tested buffers in + ttm_bo_validate() +- Agree on the naming convention +- Rewrite the mock manager: drop use_tt and manage mock memory using + drm_mm manager + +Notes and gotchas +================= + +- These tests are built and run with a UML kernel, because + 1) We are interested in hardware-independent testing + 2) We don't want to have actual DRM devices interacting with TTM + at the same time as the test one. Getting these to work in + parallel would require some time (...and that's a "todo" in itself!) +- Triggering ttm_bo_vm_ops callbacks from KUnit (i.e. kernel) might be + a challenge, but is worth trying. Look at selftests like + i915/gem/selftests/i915_gem_mman.c for inspiration +- The test suite uses UML where ioremap() call returns NULL, meaning that + ttm_bo_ioremap() can't be tested, unless we find a way to stub it diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c new file mode 100644 index 000000000..f3103307b --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -0,0 +1,637 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ttm_kunit_helpers.h" + +#define BO_SIZE SZ_8K + +#ifdef CONFIG_PREEMPT_RT +#define ww_mutex_base_lock(b) rt_mutex_lock(b) +#else +#define ww_mutex_base_lock(b) mutex_lock(b) +#endif + +struct ttm_bo_test_case { + const char *description; + bool interruptible; + bool no_wait; +}; + +static const struct ttm_bo_test_case ttm_bo_reserved_cases[] = { + { + .description = "Cannot be interrupted and sleeps", + .interruptible = false, + .no_wait = false, + }, + { + .description = "Cannot be interrupted, locks straight away", + .interruptible = false, + .no_wait = true, + }, + { + .description = "Can be interrupted, sleeps", + .interruptible = true, + .no_wait = false, + }, +}; + +static void ttm_bo_init_case_desc(const struct ttm_bo_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_bo_reserve, ttm_bo_reserved_cases, ttm_bo_init_case_desc); + +static void ttm_bo_reserve_optimistic_no_ticket(struct kunit *test) +{ + const struct ttm_bo_test_case *params = test->param_value; + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_bo_reserve(bo, params->interruptible, params->no_wait, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + + dma_resv_unlock(bo->base.resv); +} + +static void ttm_bo_reserve_locked_no_sleep(struct kunit *test) +{ + struct ttm_buffer_object *bo; + bool interruptible = false; + bool no_wait = true; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + /* Let's lock it beforehand */ + dma_resv_lock(bo->base.resv, NULL); + + err = ttm_bo_reserve(bo, interruptible, no_wait, NULL); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, -EBUSY); +} + +static void ttm_bo_reserve_no_wait_ticket(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ww_acquire_ctx ctx; + bool interruptible = false; + bool no_wait = true; + int err; + + ww_acquire_init(&ctx, &reservation_ww_class); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + KUNIT_ASSERT_EQ(test, err, -EBUSY); + + ww_acquire_fini(&ctx); +} + +static void ttm_bo_reserve_double_resv(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ww_acquire_ctx ctx; + bool interruptible = false; + bool no_wait = false; + int err; + + ww_acquire_init(&ctx, &reservation_ww_class); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + + dma_resv_unlock(bo->base.resv); + ww_acquire_fini(&ctx); + + KUNIT_ASSERT_EQ(test, err, -EALREADY); +} + +/* + * A test case heavily inspired by ww_test_edeadlk_normal(). It injects + * a deadlock by manipulating the sequence number of the context that holds + * dma_resv lock of bo2 so the other context is "wounded" and has to back off + * (indicated by -EDEADLK). The subtest checks if ttm_bo_reserve() properly + * propagates that error. + */ +static void ttm_bo_reserve_deadlock(struct kunit *test) +{ + struct ttm_buffer_object *bo1, *bo2; + struct ww_acquire_ctx ctx1, ctx2; + bool interruptible = false; + bool no_wait = false; + int err; + + bo1 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo2 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + ww_acquire_init(&ctx1, &reservation_ww_class); + ww_mutex_base_lock(&bo2->base.resv->lock.base); + + /* The deadlock will be caught by WW mutex, don't warn about it */ + lock_release(&bo2->base.resv->lock.base.dep_map, 1); + + bo2->base.resv->lock.ctx = &ctx2; + ctx2 = ctx1; + ctx2.stamp--; /* Make the context holding the lock younger */ + + err = ttm_bo_reserve(bo1, interruptible, no_wait, &ctx1); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_bo_reserve(bo2, interruptible, no_wait, &ctx1); + KUNIT_ASSERT_EQ(test, err, -EDEADLK); + + dma_resv_unlock(bo1->base.resv); + ww_acquire_fini(&ctx1); +} + +#if IS_BUILTIN(CONFIG_DRM_TTM_KUNIT_TEST) +struct signal_timer { + struct timer_list timer; + struct ww_acquire_ctx *ctx; +}; + +static void signal_for_ttm_bo_reserve(struct timer_list *t) +{ + struct signal_timer *s_timer = timer_container_of(s_timer, t, timer); + struct task_struct *task = s_timer->ctx->task; + + do_send_sig_info(SIGTERM, SEND_SIG_PRIV, task, PIDTYPE_PID); +} + +static int threaded_ttm_bo_reserve(void *arg) +{ + struct ttm_buffer_object *bo = arg; + struct signal_timer s_timer; + struct ww_acquire_ctx ctx; + bool interruptible = true; + bool no_wait = false; + int err; + + ww_acquire_init(&ctx, &reservation_ww_class); + + /* Prepare a signal that will interrupt the reservation attempt */ + timer_setup_on_stack(&s_timer.timer, &signal_for_ttm_bo_reserve, 0); + s_timer.ctx = &ctx; + + mod_timer(&s_timer.timer, msecs_to_jiffies(100)); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + + timer_delete_sync(&s_timer.timer); + timer_destroy_on_stack(&s_timer.timer); + + ww_acquire_fini(&ctx); + + return err; +} + +static void ttm_bo_reserve_interrupted(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct task_struct *task; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + task = kthread_create(threaded_ttm_bo_reserve, bo, "ttm-bo-reserve"); + + if (IS_ERR(task)) + KUNIT_FAIL(test, "Couldn't create ttm bo reserve task\n"); + + /* Take a lock so the threaded reserve has to wait */ + dma_resv_lock(bo->base.resv, NULL); + + wake_up_process(task); + msleep(20); + err = kthread_stop(task); + + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, -ERESTARTSYS); +} +#endif /* IS_BUILTIN(CONFIG_DRM_TTM_KUNIT_TEST) */ + +static void ttm_bo_unreserve_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_device *ttm_dev; + struct ttm_resource *res1, *res2; + struct ttm_place *place; + struct ttm_resource_manager *man; + unsigned int bo_prio = TTM_MAX_BO_PRIORITY - 1; + u32 mem_type = TTM_PL_SYSTEM; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->priority = bo_prio; + + err = ttm_resource_alloc(bo, place, &res1, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + + bo->resource = res1; + + /* Add a dummy resource to populate LRU */ + ttm_resource_alloc(bo, place, &res2, NULL); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unreserve(bo); + + man = ttm_manager_type(priv->ttm_dev, mem_type); + KUNIT_ASSERT_EQ(test, + list_is_last(&res1->lru.link, &man->lru[bo->priority]), 1); + + ttm_resource_free(bo, &res2); + ttm_resource_free(bo, &res1); +} + +static void ttm_bo_unreserve_pinned(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_device *ttm_dev; + struct ttm_resource *res1, *res2; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + place = ttm_place_kunit_init(test, mem_type, 0); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_pin(bo); + + err = ttm_resource_alloc(bo, place, &res1, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res1; + + /* Add a dummy resource to the pinned list */ + err = ttm_resource_alloc(bo, place, &res2, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, + list_is_last(&res2->lru.link, &priv->ttm_dev->unevictable), 1); + + ttm_bo_unreserve(bo); + KUNIT_ASSERT_EQ(test, + list_is_last(&res1->lru.link, &priv->ttm_dev->unevictable), 1); + + ttm_resource_free(bo, &res1); + ttm_resource_free(bo, &res2); +} + +static void ttm_bo_unreserve_bulk(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_lru_bulk_move lru_bulk_move; + struct ttm_lru_bulk_move_pos *pos; + struct ttm_buffer_object *bo1, *bo2; + struct ttm_resource *res1, *res2; + struct ttm_device *ttm_dev; + struct ttm_place *place; + struct dma_resv *resv; + u32 mem_type = TTM_PL_SYSTEM; + unsigned int bo_priority = 0; + int err; + + ttm_lru_bulk_move_init(&lru_bulk_move); + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + resv = kunit_kzalloc(test, sizeof(*resv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, resv); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + dma_resv_init(resv); + + bo1 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, resv); + bo2 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, resv); + + dma_resv_lock(bo1->base.resv, NULL); + ttm_bo_set_bulk_move(bo1, &lru_bulk_move); + dma_resv_unlock(bo1->base.resv); + + err = ttm_resource_alloc(bo1, place, &res1, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo1->resource = res1; + + dma_resv_lock(bo2->base.resv, NULL); + ttm_bo_set_bulk_move(bo2, &lru_bulk_move); + dma_resv_unlock(bo2->base.resv); + + err = ttm_resource_alloc(bo2, place, &res2, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo2->resource = res2; + + ttm_bo_reserve(bo1, false, false, NULL); + ttm_bo_unreserve(bo1); + + pos = &lru_bulk_move.pos[mem_type][bo_priority]; + KUNIT_ASSERT_PTR_EQ(test, res1, pos->last); + + ttm_resource_free(bo1, &res1); + ttm_resource_free(bo2, &res2); + + dma_resv_fini(resv); +} + +static void ttm_bo_fini_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + struct ttm_device *ttm_dev; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_device; + + err = ttm_resource_alloc(bo, place, &res, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_bo_fini(bo); +} + +static const char *mock_name(struct dma_fence *f) +{ + return "kunit-ttm-bo-put"; +} + +static const struct dma_fence_ops mock_fence_ops = { + .get_driver_name = mock_name, + .get_timeline_name = mock_name, +}; + +static void ttm_bo_fini_shared_resv(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct dma_resv *external_resv; + struct dma_fence *fence; + /* A dummy DMA fence lock */ + spinlock_t fence_lock; + struct ttm_device *ttm_dev; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + external_resv = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, external_resv); + + dma_resv_init(external_resv); + + fence = kunit_kzalloc(test, sizeof(*fence), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fence); + + spin_lock_init(&fence_lock); + dma_fence_init(fence, &mock_fence_ops, &fence_lock, 0, 0); + + dma_resv_lock(external_resv, NULL); + dma_resv_reserve_fences(external_resv, 1); + dma_resv_add_fence(external_resv, fence, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_unlock(external_resv); + + dma_fence_signal(fence); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_device; + bo->base.resv = external_resv; + + ttm_bo_fini(bo); +} + +static void ttm_bo_pin_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_device *ttm_dev; + unsigned int no_pins = 3; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + for (int i = 0; i < no_pins; i++) { + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_pin(bo); + dma_resv_unlock(bo->base.resv); + } + + KUNIT_ASSERT_EQ(test, bo->pin_count, no_pins); +} + +static void ttm_bo_pin_unpin_resource(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_lru_bulk_move lru_bulk_move; + struct ttm_lru_bulk_move_pos *pos; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + struct ttm_device *ttm_dev; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + unsigned int bo_priority = 0; + int err; + + ttm_lru_bulk_move_init(&lru_bulk_move); + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_resource_alloc(bo, place, &res, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res; + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_set_bulk_move(bo, &lru_bulk_move); + ttm_bo_pin(bo); + dma_resv_unlock(bo->base.resv); + + pos = &lru_bulk_move.pos[mem_type][bo_priority]; + + KUNIT_ASSERT_EQ(test, bo->pin_count, 1); + KUNIT_ASSERT_NULL(test, pos->first); + KUNIT_ASSERT_NULL(test, pos->last); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_PTR_EQ(test, res, pos->last); + KUNIT_ASSERT_EQ(test, bo->pin_count, 0); + + ttm_resource_free(bo, &res); +} + +static void ttm_bo_multiple_pin_one_unpin(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_lru_bulk_move lru_bulk_move; + struct ttm_lru_bulk_move_pos *pos; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + struct ttm_device *ttm_dev; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + unsigned int bo_priority = 0; + int err; + + ttm_lru_bulk_move_init(&lru_bulk_move); + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_resource_alloc(bo, place, &res, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res; + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_set_bulk_move(bo, &lru_bulk_move); + + /* Multiple pins */ + ttm_bo_pin(bo); + ttm_bo_pin(bo); + + dma_resv_unlock(bo->base.resv); + + pos = &lru_bulk_move.pos[mem_type][bo_priority]; + + KUNIT_ASSERT_EQ(test, bo->pin_count, 2); + KUNIT_ASSERT_NULL(test, pos->first); + KUNIT_ASSERT_NULL(test, pos->last); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, bo->pin_count, 1); + KUNIT_ASSERT_NULL(test, pos->first); + KUNIT_ASSERT_NULL(test, pos->last); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + ttm_resource_free(bo, &res); +} + +static struct kunit_case ttm_bo_test_cases[] = { + KUNIT_CASE_PARAM(ttm_bo_reserve_optimistic_no_ticket, + ttm_bo_reserve_gen_params), + KUNIT_CASE(ttm_bo_reserve_locked_no_sleep), + KUNIT_CASE(ttm_bo_reserve_no_wait_ticket), + KUNIT_CASE(ttm_bo_reserve_double_resv), +#if IS_BUILTIN(CONFIG_DRM_TTM_KUNIT_TEST) + KUNIT_CASE(ttm_bo_reserve_interrupted), +#endif + KUNIT_CASE(ttm_bo_reserve_deadlock), + KUNIT_CASE(ttm_bo_unreserve_basic), + KUNIT_CASE(ttm_bo_unreserve_pinned), + KUNIT_CASE(ttm_bo_unreserve_bulk), + KUNIT_CASE(ttm_bo_fini_basic), + KUNIT_CASE(ttm_bo_fini_shared_resv), + KUNIT_CASE(ttm_bo_pin_basic), + KUNIT_CASE(ttm_bo_pin_unpin_resource), + KUNIT_CASE(ttm_bo_multiple_pin_one_unpin), + {} +}; + +static struct kunit_suite ttm_bo_test_suite = { + .name = "ttm_bo", + .init = ttm_test_devices_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_bo_test_cases, +}; + +kunit_test_suites(&ttm_bo_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_bo APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c new file mode 100644 index 000000000..56ad8ef32 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c @@ -0,0 +1,1217 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include +#include + +#include +#include +#include + +#include "ttm_kunit_helpers.h" +#include "ttm_mock_manager.h" + +#define BO_SIZE SZ_4K +#define MANAGER_SIZE SZ_1M + +static struct spinlock fence_lock; + +struct ttm_bo_validate_test_case { + const char *description; + enum ttm_bo_type bo_type; + u32 mem_type; + bool with_ttm; + bool no_gpu_wait; +}; + +static struct ttm_placement *ttm_placement_kunit_init(struct kunit *test, + struct ttm_place *places, + unsigned int num_places) +{ + struct ttm_placement *placement; + + placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, placement); + + placement->num_placement = num_places; + placement->placement = places; + + return placement; +} + +static const char *fence_name(struct dma_fence *f) +{ + return "ttm-bo-validate-fence"; +} + +static const struct dma_fence_ops fence_ops = { + .get_driver_name = fence_name, + .get_timeline_name = fence_name, +}; + +static struct dma_fence *alloc_mock_fence(struct kunit *test) +{ + struct dma_fence *fence; + + fence = kunit_kzalloc(test, sizeof(*fence), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fence); + + dma_fence_init(fence, &fence_ops, &fence_lock, 0, 0); + + return fence; +} + +static void dma_resv_kunit_active_fence_init(struct kunit *test, + struct dma_resv *resv, + enum dma_resv_usage usage) +{ + struct dma_fence *fence; + + fence = alloc_mock_fence(test); + dma_fence_enable_signaling(fence); + + dma_resv_lock(resv, NULL); + dma_resv_reserve_fences(resv, 1); + dma_resv_add_fence(resv, fence, usage); + dma_resv_unlock(resv); +} + +static void ttm_bo_validate_case_desc(const struct ttm_bo_validate_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +static const struct ttm_bo_validate_test_case ttm_bo_type_cases[] = { + { + .description = "Buffer object for userspace", + .bo_type = ttm_bo_type_device, + }, + { + .description = "Kernel buffer object", + .bo_type = ttm_bo_type_kernel, + }, + { + .description = "Shared buffer object", + .bo_type = ttm_bo_type_sg, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_types, ttm_bo_type_cases, + ttm_bo_validate_case_desc); + +static void ttm_bo_init_reserved_sys_man(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_test_devices *priv = test->priv; + enum ttm_bo_type bo_type = params->bo_type; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx, NULL, NULL, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, kref_read(&bo->kref), 1); + KUNIT_EXPECT_PTR_EQ(test, bo->bdev, priv->ttm_dev); + KUNIT_EXPECT_EQ(test, bo->type, bo_type); + KUNIT_EXPECT_EQ(test, bo->page_alignment, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, bo->destroy, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, bo->pin_count, 0); + KUNIT_EXPECT_NULL(test, bo->bulk_move); + KUNIT_EXPECT_NOT_NULL(test, bo->ttm); + KUNIT_EXPECT_FALSE(test, ttm_tt_is_populated(bo->ttm)); + KUNIT_EXPECT_NOT_NULL(test, (void *)bo->base.resv->fences); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, size); + + if (bo_type != ttm_bo_type_kernel) + KUNIT_EXPECT_TRUE(test, + drm_mm_node_allocated(&bo->base.vma_node.vm_node)); + + ttm_resource_free(bo, &bo->resource); + ttm_bo_fini(bo); +} + +static void ttm_bo_init_reserved_mock_man(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + enum ttm_bo_type bo_type = params->bo_type; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + u32 mem_type = TTM_PL_VRAM; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx, NULL, NULL, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, kref_read(&bo->kref), 1); + KUNIT_EXPECT_PTR_EQ(test, bo->bdev, priv->ttm_dev); + KUNIT_EXPECT_EQ(test, bo->type, bo_type); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, size); + + if (bo_type != ttm_bo_type_kernel) + KUNIT_EXPECT_TRUE(test, + drm_mm_node_allocated(&bo->base.vma_node.vm_node)); + + ttm_resource_free(bo, &bo->resource); + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_init_reserved_resv(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct dma_resv resv; + int err; + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + dma_resv_init(&resv); + dma_resv_lock(&resv, NULL); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx, NULL, &resv, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_PTR_EQ(test, bo->base.resv, &resv); + + ttm_resource_free(bo, &bo->resource); + ttm_bo_fini(bo); +} + +static void ttm_bo_validate_basic(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + u32 fst_mem = TTM_PL_SYSTEM, snd_mem = TTM_PL_VRAM; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *fst_placement, *snd_placement; + struct ttm_test_devices *priv = test->priv; + struct ttm_place *fst_place, *snd_place; + u32 size = ALIGN(SZ_8K, PAGE_SIZE); + struct ttm_buffer_object *bo; + int err; + + ttm_mock_manager_init(priv->ttm_dev, snd_mem, MANAGER_SIZE); + + fst_place = ttm_place_kunit_init(test, fst_mem, 0); + fst_placement = ttm_placement_kunit_init(test, fst_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, params->bo_type, + fst_placement, PAGE_SIZE, &ctx_init, NULL, + NULL, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + snd_place = ttm_place_kunit_init(test, snd_mem, GPU_BUDDY_TOPDOWN_ALLOCATION); + snd_placement = ttm_placement_kunit_init(test, snd_place, 1); + + err = ttm_bo_validate(bo, snd_placement, &ctx_val); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, bo->base.size); + KUNIT_EXPECT_NOT_NULL(test, bo->ttm); + KUNIT_EXPECT_TRUE(test, ttm_tt_is_populated(bo->ttm)); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); + KUNIT_EXPECT_EQ(test, bo->resource->placement, + GPU_BUDDY_TOPDOWN_ALLOCATION); + + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, snd_mem); +} + +static void ttm_bo_validate_invalid_placement(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + u32 unknown_mem_type = TTM_PL_PRIV + 1; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, unknown_mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + + ttm_bo_fini(bo); +} + +static void ttm_bo_validate_failed_alloc(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + u32 mem_type = TTM_PL_VRAM; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + ttm_bad_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + + ttm_bo_fini(bo); + ttm_bad_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_pinned(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + ttm_bo_pin(bo); + err = ttm_bo_validate(bo, placement, &ctx); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -EINVAL); + + ttm_bo_reserve(bo, false, false, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + ttm_bo_fini(bo); +} + +static const struct ttm_bo_validate_test_case ttm_mem_type_cases[] = { + { + .description = "System manager", + .mem_type = TTM_PL_SYSTEM, + }, + { + .description = "VRAM manager", + .mem_type = TTM_PL_VRAM, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_validate_mem, ttm_mem_type_cases, + ttm_bo_validate_case_desc); + +static void ttm_bo_validate_same_placement(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, params->mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + if (params->mem_type != TTM_PL_SYSTEM) + ttm_mock_manager_init(priv->ttm_dev, params->mem_type, MANAGER_SIZE); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, params->bo_type, + placement, PAGE_SIZE, &ctx_init, NULL, + NULL, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + err = ttm_bo_validate(bo, placement, &ctx_val); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, 0); + + ttm_bo_fini(bo); + + if (params->mem_type != TTM_PL_SYSTEM) + ttm_mock_manager_fini(priv->ttm_dev, params->mem_type); +} + +static void ttm_bo_validate_busy_placement(struct kunit *test) +{ + u32 fst_mem = TTM_PL_VRAM, snd_mem = TTM_PL_VRAM + 1; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *placement_init, *placement_val; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_place *init_place, places[2]; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + int err; + + ttm_bad_manager_init(priv->ttm_dev, fst_mem, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, snd_mem, MANAGER_SIZE); + + init_place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement_init = ttm_placement_kunit_init(test, init_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement_init, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + places[0] = (struct ttm_place){ .mem_type = fst_mem, .flags = TTM_PL_FLAG_DESIRED }; + places[1] = (struct ttm_place){ .mem_type = snd_mem, .flags = TTM_PL_FLAG_FALLBACK }; + placement_val = ttm_placement_kunit_init(test, places, 2); + + err = ttm_bo_validate(bo, placement_val, &ctx_val); + dma_resv_unlock(bo->base.resv); + + man = ttm_manager_type(priv->ttm_dev, snd_mem); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, bo->base.size); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); + KUNIT_ASSERT_TRUE(test, list_is_singular(&man->lru[bo->priority])); + + ttm_bo_fini(bo); + ttm_bad_manager_fini(priv->ttm_dev, fst_mem); + ttm_mock_manager_fini(priv->ttm_dev, snd_mem); +} + +static void ttm_bo_validate_multihop(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *placement_init, *placement_val; + u32 fst_mem = TTM_PL_VRAM, tmp_mem = TTM_PL_TT, final_mem = TTM_PL_SYSTEM; + struct ttm_test_devices *priv = test->priv; + struct ttm_place *fst_place, *final_place; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_buffer_object *bo; + int err; + + ttm_mock_manager_init(priv->ttm_dev, fst_mem, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, tmp_mem, MANAGER_SIZE); + + fst_place = ttm_place_kunit_init(test, fst_mem, 0); + placement_init = ttm_placement_kunit_init(test, fst_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, params->bo_type, + placement_init, PAGE_SIZE, &ctx_init, NULL, + NULL, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + final_place = ttm_place_kunit_init(test, final_mem, 0); + placement_val = ttm_placement_kunit_init(test, final_place, 1); + + err = ttm_bo_validate(bo, placement_val, &ctx_val); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, size * 2); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, final_mem); + + ttm_bo_fini(bo); + + ttm_mock_manager_fini(priv->ttm_dev, fst_mem); + ttm_mock_manager_fini(priv->ttm_dev, tmp_mem); +} + +static const struct ttm_bo_validate_test_case ttm_bo_no_placement_cases[] = { + { + .description = "Buffer object in system domain, no page vector", + }, + { + .description = "Buffer object in system domain with an existing page vector", + .with_ttm = true, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_no_placement, ttm_bo_no_placement_cases, + ttm_bo_validate_case_desc); + +static void ttm_bo_validate_no_placement_signaled(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_resource_manager *man; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_tt *old_tt; + u32 flags; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + man = ttm_manager_type(priv->ttm_dev, mem_type); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + if (params->with_ttm) { + old_tt = priv->ttm_dev->funcs->ttm_tt_create(bo, 0); + ttm_pool_alloc(&priv->ttm_dev->pool, old_tt, &ctx); + bo->ttm = old_tt; + } + + placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, placement); + + ttm_bo_reserve(bo, false, false, NULL); + + err = ttm_resource_alloc(bo, place, &bo->resource, NULL); + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, man->usage, size); + + err = ttm_bo_validate(bo, placement, &ctx); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, man->usage, 0); + KUNIT_ASSERT_NOT_NULL(test, bo->ttm); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, 0); + + if (params->with_ttm) { + flags = bo->ttm->page_flags; + + KUNIT_ASSERT_PTR_EQ(test, bo->ttm, old_tt); + KUNIT_ASSERT_FALSE(test, flags & TTM_TT_FLAG_PRIV_POPULATED); + KUNIT_ASSERT_TRUE(test, flags & TTM_TT_FLAG_ZERO_ALLOC); + } + + ttm_bo_fini(bo); +} + +static int threaded_dma_resv_signal(void *arg) +{ + struct ttm_buffer_object *bo = arg; + struct dma_resv *resv = bo->base.resv; + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) { + dma_fence_signal(fence); + } + dma_resv_iter_end(&cursor); + + return 0; +} + +static void ttm_bo_validate_no_placement_not_signaled(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct task_struct *task; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = params->bo_type; + + err = ttm_resource_alloc(bo, place, &bo->resource, NULL); + KUNIT_EXPECT_EQ(test, err, 0); + + placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, placement); + + /* Create an active fence to simulate a non-idle resv object */ + spin_lock_init(&fence_lock); + dma_resv_kunit_active_fence_init(test, bo->base.resv, usage); + + task = kthread_create(threaded_dma_resv_signal, bo, "dma-resv-signal"); + if (IS_ERR(task)) + KUNIT_FAIL(test, "Couldn't create dma resv signal task\n"); + + wake_up_process(task); + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, bo->ttm); + KUNIT_ASSERT_NULL(test, bo->resource); + KUNIT_ASSERT_NULL(test, bo->bulk_move); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, 0); + + if (bo->type != ttm_bo_type_sg) + KUNIT_ASSERT_PTR_EQ(test, bo->base.resv, &bo->base._resv); + + /* Make sure we have an idle object at this point */ + dma_resv_wait_timeout(bo->base.resv, usage, false, MAX_SCHEDULE_TIMEOUT); + + ttm_bo_fini(bo); +} + +static void ttm_bo_validate_move_fence_signaled(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_resource_manager *man; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + man = ttm_manager_type(priv->ttm_dev, mem_type); + man->eviction_fences[0] = dma_fence_get_stub(); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, size); + + ttm_bo_fini(bo); + dma_fence_put(man->eviction_fences[0]); +} + +static const struct ttm_bo_validate_test_case ttm_bo_validate_wait_cases[] = { + { + .description = "Waits for GPU", + .no_gpu_wait = false, + }, + { + .description = "Tries to lock straight away", + .no_gpu_wait = true, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_validate_wait, ttm_bo_validate_wait_cases, + ttm_bo_validate_case_desc); + +static int threaded_fence_signal(void *arg) +{ + struct dma_fence *fence = arg; + + msleep(20); + + return dma_fence_check_and_signal(fence) ? -EINVAL : 0; +} + +static void ttm_bo_validate_move_fence_not_signaled(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_operation_ctx ctx_init = { }, + ctx_val = { .no_wait_gpu = params->no_gpu_wait }; + u32 fst_mem = TTM_PL_VRAM, snd_mem = TTM_PL_VRAM + 1; + struct ttm_placement *placement_init, *placement_val; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_place *init_place, places[2]; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + struct task_struct *task; + int err; + + init_place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement_init = ttm_placement_kunit_init(test, init_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement_init, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_mock_manager_init(priv->ttm_dev, fst_mem, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, snd_mem, MANAGER_SIZE); + + places[0] = (struct ttm_place){ .mem_type = fst_mem, .flags = TTM_PL_FLAG_DESIRED }; + places[1] = (struct ttm_place){ .mem_type = snd_mem, .flags = TTM_PL_FLAG_FALLBACK }; + placement_val = ttm_placement_kunit_init(test, places, 2); + + spin_lock_init(&fence_lock); + man = ttm_manager_type(priv->ttm_dev, fst_mem); + man->eviction_fences[0] = alloc_mock_fence(test); + + task = kthread_create(threaded_fence_signal, man->eviction_fences[0], "move-fence-signal"); + if (IS_ERR(task)) + KUNIT_FAIL(test, "Couldn't create move fence signal task\n"); + + wake_up_process(task); + err = ttm_bo_validate(bo, placement_val, &ctx_val); + dma_resv_unlock(bo->base.resv); + + dma_fence_wait_timeout(man->eviction_fences[0], false, MAX_SCHEDULE_TIMEOUT); + man->eviction_fences[0] = NULL; + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, size); + + if (params->no_gpu_wait) + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); + else + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, fst_mem); + + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, fst_mem); + ttm_mock_manager_fini(priv->ttm_dev, snd_mem); +} + +static void ttm_bo_validate_swapout(struct kunit *test) +{ + u32 mem_type = TTM_PL_TT; + struct ttm_test_devices *priv = test->priv; + struct ttm_operation_ctx ctx_init = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_resource_manager *man; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + s64 swapped; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + man = ttm_manager_type(priv->ttm_dev, mem_type); + KUNIT_ASSERT_NOT_NULL(test, man); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo->base.resv); + + swapped = ttm_bo_swapout(priv->ttm_dev, &ctx_init, man, GFP_KERNEL, + MANAGER_SIZE / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, swapped, MANAGER_SIZE / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, TTM_PL_SYSTEM); + KUNIT_EXPECT_TRUE(test, bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED); + + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_happy_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT, + mem_type_evict = TTM_PL_SYSTEM; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + u32 small = SZ_8K, medium = SZ_512K, + big = MANAGER_SIZE - (small + medium); + u32 bo_sizes[] = { small, medium, big }; + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bos, *bo_val; + struct ttm_placement *placement; + struct ttm_place *place; + u32 bo_no = 3; + int i, err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_multihop, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bos = kunit_kmalloc_array(test, bo_no, sizeof(*bos), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bos); + + memset(bos, 0, sizeof(*bos) * bo_no); + for (i = 0; i < bo_no; i++) { + drm_gem_private_object_init(priv->drm, &bos[i].base, bo_sizes[i]); + err = ttm_bo_init_reserved(priv->ttm_dev, &bos[i], bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bos[i].base.resv); + } + + bo_val = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_val->type = bo_type; + + ttm_bo_reserve(bo_val, false, false, NULL); + err = ttm_bo_validate(bo_val, placement, &ctx_val); + ttm_bo_unreserve(bo_val); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bos[0].resource->mem_type, mem_type_evict); + KUNIT_EXPECT_TRUE(test, bos[0].ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC); + KUNIT_EXPECT_TRUE(test, bos[0].ttm->page_flags & TTM_TT_FLAG_PRIV_POPULATED); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, small * 2 + BO_SIZE); + KUNIT_EXPECT_EQ(test, bos[1].resource->mem_type, mem_type); + + for (i = 0; i < bo_no; i++) + ttm_bo_fini(&bos[i]); + ttm_bo_fini(bo_val); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_multihop); +} + +static void ttm_bo_validate_all_pinned_evict(struct kunit *test) +{ + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_buffer_object *bo_big, *bo_small; + struct ttm_test_devices *priv = test->priv; + struct ttm_placement *placement; + u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_multihop, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_big = kunit_kzalloc(test, sizeof(*bo_big), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_big); + + drm_gem_private_object_init(priv->drm, &bo_big->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_big, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_bo_pin(bo_big); + dma_resv_unlock(bo_big->base.resv); + + bo_small = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_small->type = bo_type; + + ttm_bo_reserve(bo_small, false, false, NULL); + err = ttm_bo_validate(bo_small, placement, &ctx_val); + ttm_bo_unreserve(bo_small); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + + ttm_bo_fini(bo_small); + + ttm_bo_reserve(bo_big, false, false, NULL); + ttm_bo_unpin(bo_big); + dma_resv_unlock(bo_big->base.resv); + ttm_bo_fini(bo_big); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_multihop); +} + +static void ttm_bo_validate_allowed_only_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT, + mem_type_evict = TTM_PL_SYSTEM; + struct ttm_buffer_object *bo, *bo_evictable, *bo_pinned; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_placement *placement; + struct ttm_place *place; + u32 size = SZ_512K; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_multihop, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_pinned = kunit_kzalloc(test, sizeof(*bo_pinned), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_pinned); + + drm_gem_private_object_init(priv->drm, &bo_pinned->base, size); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_pinned, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + ttm_bo_pin(bo_pinned); + dma_resv_unlock(bo_pinned->base.resv); + + bo_evictable = kunit_kzalloc(test, sizeof(*bo_evictable), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_evictable); + + drm_gem_private_object_init(priv->drm, &bo_evictable->base, size); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_evictable, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_evictable->base.resv); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx_val); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, bo_pinned->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, bo_evictable->resource->mem_type, mem_type_evict); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, size * 2 + BO_SIZE); + + ttm_bo_fini(bo); + ttm_bo_fini(bo_evictable); + + ttm_bo_reserve(bo_pinned, false, false, NULL); + ttm_bo_unpin(bo_pinned); + dma_resv_unlock(bo_pinned->base.resv); + ttm_bo_fini(bo_pinned); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_multihop); +} + +static void ttm_bo_validate_deleted_evict(struct kunit *test) +{ + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + u32 small = SZ_8K, big = MANAGER_SIZE - BO_SIZE; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_buffer_object *bo_big, *bo_small; + struct ttm_test_devices *priv = test->priv; + struct ttm_resource_manager *man; + u32 mem_type = TTM_PL_VRAM; + struct ttm_placement *placement; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + man = ttm_manager_type(priv->ttm_dev, mem_type); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_big = kunit_kzalloc(test, sizeof(*bo_big), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_big); + + drm_gem_private_object_init(priv->drm, &bo_big->base, big); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_big, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ttm_resource_manager_usage(man), big); + + dma_resv_unlock(bo_big->base.resv); + bo_big->deleted = true; + + bo_small = ttm_bo_kunit_init(test, test->priv, small, NULL); + bo_small->type = bo_type; + + ttm_bo_reserve(bo_small, false, false, NULL); + err = ttm_bo_validate(bo_small, placement, &ctx_val); + ttm_bo_unreserve(bo_small); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo_small->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, ttm_resource_manager_usage(man), small); + KUNIT_EXPECT_NULL(test, bo_big->ttm); + KUNIT_EXPECT_NULL(test, bo_big->resource); + + ttm_bo_fini(bo_small); + ttm_bo_fini(bo_big); + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_busy_domain_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_VRAM, mem_type_evict = TTM_PL_MOCK1; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo_init, *bo_val; + struct ttm_placement *placement; + struct ttm_place *place; + int err; + + /* + * Drop the default device and setup a new one that points to busy + * thus unsuitable eviction domain + */ + ttm_device_fini(priv->ttm_dev); + + err = ttm_device_kunit_init_bad_evict(test->priv, priv->ttm_dev); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_busy_manager_init(priv->ttm_dev, mem_type_evict, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_init = kunit_kzalloc(test, sizeof(*bo_init), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_init); + + drm_gem_private_object_init(priv->drm, &bo_init->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_init, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_init->base.resv); + + bo_val = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_val->type = bo_type; + + ttm_bo_reserve(bo_val, false, false, NULL); + err = ttm_bo_validate(bo_val, placement, &ctx_val); + ttm_bo_unreserve(bo_val); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + KUNIT_EXPECT_EQ(test, bo_init->resource->mem_type, mem_type); + KUNIT_EXPECT_NULL(test, bo_val->resource); + + ttm_bo_fini(bo_init); + ttm_bo_fini(bo_val); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_bad_manager_fini(priv->ttm_dev, mem_type_evict); +} + +static void ttm_bo_validate_evict_gutting(struct kunit *test) +{ + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo, *bo_evict; + u32 mem_type = TTM_PL_MOCK1; + struct ttm_placement *placement; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_evict = kunit_kzalloc(test, sizeof(*bo_evict), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_evict); + + drm_gem_private_object_init(priv->drm, &bo_evict->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_evict, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_evict->base.resv); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx_val); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, mem_type); + KUNIT_ASSERT_NULL(test, bo_evict->resource); + KUNIT_ASSERT_TRUE(test, bo_evict->ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC); + + ttm_bo_fini(bo_evict); + ttm_bo_fini(bo); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_recrusive_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_TT, mem_type_evict = TTM_PL_MOCK2; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *placement_tt, *placement_mock; + struct ttm_buffer_object *bo_tt, *bo_mock, *bo_val; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_place *place_tt, *place_mock; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_type_evict, MANAGER_SIZE); + + place_tt = ttm_place_kunit_init(test, mem_type, 0); + place_mock = ttm_place_kunit_init(test, mem_type_evict, 0); + + placement_tt = ttm_placement_kunit_init(test, place_tt, 1); + placement_mock = ttm_placement_kunit_init(test, place_mock, 1); + + bo_tt = kunit_kzalloc(test, sizeof(*bo_tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_tt); + + bo_mock = kunit_kzalloc(test, sizeof(*bo_mock), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_mock); + + drm_gem_private_object_init(priv->drm, &bo_tt->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_tt, bo_type, placement_tt, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_tt->base.resv); + + drm_gem_private_object_init(priv->drm, &bo_mock->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_mock, bo_type, placement_mock, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_mock->base.resv); + + bo_val = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_val->type = bo_type; + + ttm_bo_reserve(bo_val, false, false, NULL); + err = ttm_bo_validate(bo_val, placement_tt, &ctx_val); + ttm_bo_unreserve(bo_val); + + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_type_evict); + + ttm_bo_fini(bo_val); + ttm_bo_fini(bo_tt); + ttm_bo_fini(bo_mock); +} + +static struct kunit_case ttm_bo_validate_test_cases[] = { + KUNIT_CASE_PARAM(ttm_bo_init_reserved_sys_man, ttm_bo_types_gen_params), + KUNIT_CASE_PARAM(ttm_bo_init_reserved_mock_man, ttm_bo_types_gen_params), + KUNIT_CASE(ttm_bo_init_reserved_resv), + KUNIT_CASE_PARAM(ttm_bo_validate_basic, ttm_bo_types_gen_params), + KUNIT_CASE(ttm_bo_validate_invalid_placement), + KUNIT_CASE_PARAM(ttm_bo_validate_same_placement, + ttm_bo_validate_mem_gen_params), + KUNIT_CASE(ttm_bo_validate_failed_alloc), + KUNIT_CASE(ttm_bo_validate_pinned), + KUNIT_CASE(ttm_bo_validate_busy_placement), + KUNIT_CASE_PARAM(ttm_bo_validate_multihop, ttm_bo_types_gen_params), + KUNIT_CASE_PARAM(ttm_bo_validate_no_placement_signaled, + ttm_bo_no_placement_gen_params), + KUNIT_CASE_PARAM(ttm_bo_validate_no_placement_not_signaled, + ttm_bo_types_gen_params), + KUNIT_CASE(ttm_bo_validate_move_fence_signaled), + KUNIT_CASE_PARAM(ttm_bo_validate_move_fence_not_signaled, + ttm_bo_validate_wait_gen_params), + KUNIT_CASE(ttm_bo_validate_swapout), + KUNIT_CASE(ttm_bo_validate_happy_evict), + KUNIT_CASE(ttm_bo_validate_all_pinned_evict), + KUNIT_CASE(ttm_bo_validate_allowed_only_evict), + KUNIT_CASE(ttm_bo_validate_deleted_evict), + KUNIT_CASE(ttm_bo_validate_busy_domain_evict), + KUNIT_CASE(ttm_bo_validate_evict_gutting), + KUNIT_CASE(ttm_bo_validate_recrusive_evict), + {} +}; + +static struct kunit_suite ttm_bo_validate_test_suite = { + .name = "ttm_bo_validate", + .init = ttm_test_devices_all_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_bo_validate_test_cases, +}; + +kunit_test_suites(&ttm_bo_validate_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_bo APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_device_test.c b/drivers/gpu/drm/ttm/tests/ttm_device_test.c new file mode 100644 index 000000000..8bcac84e9 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_device_test.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include +#include +#include + +#include "ttm_kunit_helpers.h" +#include "../ttm_pool_internal.h" + +struct ttm_device_test_case { + const char *description; + unsigned int alloc_flags; + bool pools_init_expected; +}; + +static void ttm_device_init_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_device *ttm_dev; + struct ttm_resource_manager *ttm_sys_man; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_EXPECT_PTR_EQ(test, ttm_dev->funcs, &ttm_dev_funcs); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev->wq); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev->man_drv[TTM_PL_SYSTEM]); + + ttm_sys_man = &ttm_dev->sysman; + KUNIT_ASSERT_NOT_NULL(test, ttm_sys_man); + KUNIT_EXPECT_TRUE(test, ttm_sys_man->use_tt); + KUNIT_EXPECT_TRUE(test, ttm_sys_man->use_type); + KUNIT_ASSERT_NOT_NULL(test, ttm_sys_man->func); + + KUNIT_EXPECT_PTR_EQ(test, ttm_dev->dev_mapping, + priv->drm->anon_inode->i_mapping); + + ttm_device_fini(ttm_dev); +} + +static void ttm_device_init_multiple(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_device *ttm_devs; + unsigned int i, num_dev = 3; + int err; + + ttm_devs = kunit_kcalloc(test, num_dev, sizeof(*ttm_devs), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_devs); + + for (i = 0; i < num_dev; i++) { + err = ttm_device_kunit_init(priv, &ttm_devs[i], 0); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_EXPECT_PTR_EQ(test, ttm_devs[i].dev_mapping, + priv->drm->anon_inode->i_mapping); + KUNIT_ASSERT_NOT_NULL(test, ttm_devs[i].wq); + KUNIT_EXPECT_PTR_EQ(test, ttm_devs[i].funcs, &ttm_dev_funcs); + KUNIT_ASSERT_NOT_NULL(test, ttm_devs[i].man_drv[TTM_PL_SYSTEM]); + } + + KUNIT_ASSERT_EQ(test, list_count_nodes(&ttm_devs[0].device_list), num_dev); + + for (i = 0; i < num_dev; i++) + ttm_device_fini(&ttm_devs[i]); +} + +static void ttm_device_fini_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_device *ttm_dev; + struct ttm_resource_manager *man; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + man = ttm_manager_type(ttm_dev, TTM_PL_SYSTEM); + KUNIT_ASSERT_NOT_NULL(test, man); + + ttm_device_fini(ttm_dev); + + KUNIT_ASSERT_FALSE(test, man->use_type); + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[0])); + KUNIT_ASSERT_NULL(test, ttm_dev->man_drv[TTM_PL_SYSTEM]); +} + +static void ttm_device_init_no_vma_man(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct drm_device *drm = priv->drm; + struct ttm_device *ttm_dev; + struct drm_vma_offset_manager *vma_man; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + /* Let's pretend there's no VMA manager allocated */ + vma_man = drm->vma_offset_manager; + drm->vma_offset_manager = NULL; + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_EXPECT_EQ(test, err, -EINVAL); + + /* Bring the manager back for a graceful cleanup */ + drm->vma_offset_manager = vma_man; +} + +static const struct ttm_device_test_case ttm_device_cases[] = { + { + .description = "No DMA allocations, no DMA32 required", + .pools_init_expected = false, + }, + { + .description = "DMA allocations, DMA32 required", + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC | + TTM_ALLOCATION_POOL_USE_DMA32, + .pools_init_expected = true, + }, + { + .description = "No DMA allocations, DMA32 required", + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA32, + .pools_init_expected = false, + }, + { + .description = "DMA allocations, no DMA32 required", + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC, + .pools_init_expected = true, + }, +}; + +static void ttm_device_case_desc(const struct ttm_device_test_case *t, char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_device, ttm_device_cases, ttm_device_case_desc); + +static void ttm_device_init_pools(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + const struct ttm_device_test_case *params = test->param_value; + struct ttm_device *ttm_dev; + struct ttm_pool *pool; + struct ttm_pool_type pt; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, params->alloc_flags); + KUNIT_ASSERT_EQ(test, err, 0); + + pool = &ttm_dev->pool; + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_EXPECT_PTR_EQ(test, pool->dev, priv->dev); + KUNIT_EXPECT_EQ(test, pool->alloc_flags, params->alloc_flags); + + if (params->pools_init_expected) { + for (int i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + for (int j = 0; j < NR_PAGE_ORDERS; ++j) { + pt = pool->caching[i].orders[j]; + KUNIT_EXPECT_PTR_EQ(test, pt.pool, pool); + KUNIT_EXPECT_EQ(test, pt.caching, i); + KUNIT_EXPECT_EQ(test, pt.order, j); + + if (ttm_pool_uses_dma_alloc(pool)) + KUNIT_ASSERT_FALSE(test, + list_lru_count(&pt.pages)); + } + } + } + + ttm_device_fini(ttm_dev); +} + +static struct kunit_case ttm_device_test_cases[] = { + KUNIT_CASE(ttm_device_init_basic), + KUNIT_CASE(ttm_device_init_multiple), + KUNIT_CASE(ttm_device_fini_basic), + KUNIT_CASE(ttm_device_init_no_vma_man), + KUNIT_CASE_PARAM(ttm_device_init_pools, ttm_device_gen_params), + {} +}; + +static struct kunit_suite ttm_device_test_suite = { + .name = "ttm_device", + .init = ttm_test_devices_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_device_test_cases, +}; + +kunit_test_suites(&ttm_device_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_device APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c new file mode 100644 index 000000000..5cfe8f3f8 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +#include + +#include "ttm_kunit_helpers.h" + +static const struct ttm_place sys_place = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_SYSTEM, + .flags = TTM_PL_FLAG_FALLBACK, +}; + +static const struct ttm_place mock1_place = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_MOCK1, + .flags = TTM_PL_FLAG_FALLBACK, +}; + +static const struct ttm_place mock2_place = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_MOCK2, + .flags = TTM_PL_FLAG_FALLBACK, +}; + +static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_place, +}; + +static struct ttm_placement bad_placement = { + .num_placement = 1, + .placement = &mock1_place, +}; + +static struct ttm_placement mock_placement = { + .num_placement = 1, + .placement = &mock2_place, +}; + +static struct ttm_tt *ttm_tt_simple_create(struct ttm_buffer_object *bo, u32 page_flags) +{ + struct ttm_tt *tt; + + tt = kzalloc_obj(*tt); + ttm_tt_init(tt, bo, page_flags, ttm_cached, 0); + + return tt; +} + +static void ttm_tt_simple_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) +{ + kfree(ttm); +} + +static int mock_move(struct ttm_buffer_object *bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_resource *new_mem, + struct ttm_place *hop) +{ + struct ttm_resource *old_mem = bo->resource; + + if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm)) { + ttm_bo_move_null(bo, new_mem); + return 0; + } + + if (bo->resource->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM) { + hop->mem_type = TTM_PL_TT; + hop->flags = TTM_PL_FLAG_TEMPORARY; + hop->fpfn = 0; + hop->lpfn = 0; + return -EMULTIHOP; + } + + if ((old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_TT) || + (old_mem->mem_type == TTM_PL_TT && + new_mem->mem_type == TTM_PL_SYSTEM)) { + ttm_bo_move_null(bo, new_mem); + return 0; + } + + return ttm_bo_move_memcpy(bo, ctx, new_mem); +} + +static void mock_evict_flags(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + switch (bo->resource->mem_type) { + case TTM_PL_VRAM: + case TTM_PL_SYSTEM: + *placement = sys_placement; + break; + case TTM_PL_TT: + *placement = mock_placement; + break; + case TTM_PL_MOCK1: + /* Purge objects coming from this domain */ + break; + } +} + +static void bad_evict_flags(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + *placement = bad_placement; +} + +static int ttm_device_kunit_init_with_funcs(struct ttm_test_devices *priv, + struct ttm_device *ttm, + unsigned int alloc_flags, + struct ttm_device_funcs *funcs) +{ + struct drm_device *drm = priv->drm; + int err; + + err = ttm_device_init(ttm, funcs, drm->dev, + drm->anon_inode->i_mapping, + drm->vma_offset_manager, + alloc_flags); + + return err; +} + +struct ttm_device_funcs ttm_dev_funcs = { + .ttm_tt_create = ttm_tt_simple_create, + .ttm_tt_destroy = ttm_tt_simple_destroy, + .move = mock_move, + .eviction_valuable = ttm_bo_eviction_valuable, + .evict_flags = mock_evict_flags, +}; +EXPORT_SYMBOL_GPL(ttm_dev_funcs); + +int ttm_device_kunit_init(struct ttm_test_devices *priv, + struct ttm_device *ttm, + unsigned int alloc_flags) +{ + return ttm_device_kunit_init_with_funcs(priv, ttm, alloc_flags, + &ttm_dev_funcs); +} +EXPORT_SYMBOL_GPL(ttm_device_kunit_init); + +struct ttm_device_funcs ttm_dev_funcs_bad_evict = { + .ttm_tt_create = ttm_tt_simple_create, + .ttm_tt_destroy = ttm_tt_simple_destroy, + .move = mock_move, + .eviction_valuable = ttm_bo_eviction_valuable, + .evict_flags = bad_evict_flags, +}; +EXPORT_SYMBOL_GPL(ttm_dev_funcs_bad_evict); + +int ttm_device_kunit_init_bad_evict(struct ttm_test_devices *priv, + struct ttm_device *ttm) +{ + return ttm_device_kunit_init_with_funcs(priv, ttm, 0, + &ttm_dev_funcs_bad_evict); +} +EXPORT_SYMBOL_GPL(ttm_device_kunit_init_bad_evict); + +struct ttm_buffer_object *ttm_bo_kunit_init(struct kunit *test, + struct ttm_test_devices *devs, + size_t size, + struct dma_resv *obj) +{ + struct drm_gem_object gem_obj = { }; + struct ttm_buffer_object *bo; + int err; + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + bo->base = gem_obj; + + if (obj) + bo->base.resv = obj; + + err = drm_gem_object_init(devs->drm, &bo->base, size); + KUNIT_ASSERT_EQ(test, err, 0); + + bo->bdev = devs->ttm_dev; + bo->destroy = dummy_ttm_bo_destroy; + + kref_init(&bo->kref); + + return bo; +} +EXPORT_SYMBOL_GPL(ttm_bo_kunit_init); + +struct ttm_place *ttm_place_kunit_init(struct kunit *test, u32 mem_type, u32 flags) +{ + struct ttm_place *place; + + place = kunit_kzalloc(test, sizeof(*place), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, place); + + place->mem_type = mem_type; + place->flags = flags; + + return place; +} +EXPORT_SYMBOL_GPL(ttm_place_kunit_init); + +void dummy_ttm_bo_destroy(struct ttm_buffer_object *bo) +{ + drm_gem_object_release(&bo->base); +} +EXPORT_SYMBOL_GPL(dummy_ttm_bo_destroy); + +struct ttm_test_devices *ttm_test_devices_basic(struct kunit *test) +{ + struct ttm_test_devices *devs; + + devs = kunit_kzalloc(test, sizeof(*devs), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, devs); + + devs->dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, devs->dev); + + /* Set mask for alloc_coherent mappings to enable ttm_pool_alloc testing */ + devs->dev->coherent_dma_mask = -1; + + devs->drm = __drm_kunit_helper_alloc_drm_device(test, devs->dev, + sizeof(*devs->drm), 0, + DRIVER_GEM); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, devs->drm); + + return devs; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_basic); + +struct ttm_test_devices *ttm_test_devices_all(struct kunit *test) +{ + struct ttm_test_devices *devs; + struct ttm_device *ttm_dev; + int err; + + devs = ttm_test_devices_basic(test); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(devs, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + devs->ttm_dev = ttm_dev; + + return devs; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_all); + +void ttm_test_devices_put(struct kunit *test, struct ttm_test_devices *devs) +{ + if (devs->ttm_dev) + ttm_device_fini(devs->ttm_dev); + + drm_kunit_helper_free_device(test, devs->dev); +} +EXPORT_SYMBOL_GPL(ttm_test_devices_put); + +int ttm_test_devices_init(struct kunit *test) +{ + struct ttm_test_devices *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv = ttm_test_devices_basic(test); + test->priv = priv; + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_init); + +int ttm_test_devices_all_init(struct kunit *test) +{ + struct ttm_test_devices *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv = ttm_test_devices_all(test); + test->priv = priv; + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_all_init); + +void ttm_test_devices_fini(struct kunit *test) +{ + ttm_test_devices_put(test, test->priv); +} +EXPORT_SYMBOL_GPL(ttm_test_devices_fini); + +MODULE_DESCRIPTION("TTM KUnit test helper functions"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h new file mode 100644 index 000000000..f8402b979 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2023 Intel Corporation + */ +#ifndef TTM_KUNIT_HELPERS_H +#define TTM_KUNIT_HELPERS_H + +#include +#include +#include +#include + +#include +#include + +#define TTM_PL_MOCK1 (TTM_PL_PRIV + 1) +#define TTM_PL_MOCK2 (TTM_PL_PRIV + 2) + +extern struct ttm_device_funcs ttm_dev_funcs; +extern struct ttm_device_funcs ttm_dev_funcs_bad_evict; + +struct ttm_test_devices { + struct drm_device *drm; + struct device *dev; + struct ttm_device *ttm_dev; +}; + +/* Building blocks for test-specific init functions */ +int ttm_device_kunit_init(struct ttm_test_devices *priv, + struct ttm_device *ttm, + unsigned int alloc_flags); +int ttm_device_kunit_init_bad_evict(struct ttm_test_devices *priv, + struct ttm_device *ttm); +struct ttm_buffer_object *ttm_bo_kunit_init(struct kunit *test, + struct ttm_test_devices *devs, + size_t size, + struct dma_resv *obj); +struct ttm_place *ttm_place_kunit_init(struct kunit *test, u32 mem_type, + u32 flags); +void dummy_ttm_bo_destroy(struct ttm_buffer_object *bo); + +struct ttm_test_devices *ttm_test_devices_basic(struct kunit *test); +struct ttm_test_devices *ttm_test_devices_all(struct kunit *test); + +void ttm_test_devices_put(struct kunit *test, struct ttm_test_devices *devs); + +/* Generic init/fini for tests that only need DRM/TTM devices */ +int ttm_test_devices_init(struct kunit *test); +int ttm_test_devices_all_init(struct kunit *test); +void ttm_test_devices_fini(struct kunit *test); + +#endif // TTM_KUNIT_HELPERS_H diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c new file mode 100644 index 000000000..0d91bc51f --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include + +#include +#include +#include + +#include "ttm_mock_manager.h" + +static inline struct ttm_mock_manager * +to_mock_mgr(struct ttm_resource_manager *man) +{ + return container_of(man, struct ttm_mock_manager, man); +} + +static inline struct ttm_mock_resource * +to_mock_mgr_resource(struct ttm_resource *res) +{ + return container_of(res, struct ttm_mock_resource, base); +} + +static int ttm_mock_manager_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct ttm_mock_manager *manager = to_mock_mgr(man); + struct ttm_mock_resource *mock_res; + struct gpu_buddy *mm = &manager->mm; + u64 lpfn, fpfn, alloc_size; + int err; + + mock_res = kzalloc_obj(*mock_res); + + if (!mock_res) + return -ENOMEM; + + fpfn = 0; + lpfn = man->size; + + ttm_resource_init(bo, place, &mock_res->base); + INIT_LIST_HEAD(&mock_res->blocks); + + if (place->flags & TTM_PL_FLAG_TOPDOWN) + mock_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) + mock_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; + + alloc_size = (uint64_t)mock_res->base.size; + mutex_lock(&manager->lock); + err = gpu_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size, + manager->default_page_size, + &mock_res->blocks, + mock_res->flags); + + if (err) + goto error_free_blocks; + mutex_unlock(&manager->lock); + + *res = &mock_res->base; + return 0; + +error_free_blocks: + gpu_buddy_free_list(mm, &mock_res->blocks, 0); + ttm_resource_fini(man, &mock_res->base); + mutex_unlock(&manager->lock); + + return err; +} + +static void ttm_mock_manager_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct ttm_mock_manager *manager = to_mock_mgr(man); + struct ttm_mock_resource *mock_res = to_mock_mgr_resource(res); + struct gpu_buddy *mm = &manager->mm; + + mutex_lock(&manager->lock); + gpu_buddy_free_list(mm, &mock_res->blocks, 0); + mutex_unlock(&manager->lock); + + ttm_resource_fini(man, res); + kfree(mock_res); +} + +static const struct ttm_resource_manager_func ttm_mock_manager_funcs = { + .alloc = ttm_mock_manager_alloc, + .free = ttm_mock_manager_free, +}; + +int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) +{ + struct ttm_mock_manager *manager; + struct ttm_resource_manager *base; + int err; + + manager = kzalloc_obj(*manager); + if (!manager) + return -ENOMEM; + + mutex_init(&manager->lock); + + err = gpu_buddy_init(&manager->mm, size, PAGE_SIZE); + + if (err) { + kfree(manager); + return err; + } + + manager->default_page_size = PAGE_SIZE; + base = &manager->man; + base->func = &ttm_mock_manager_funcs; + base->use_tt = true; + + ttm_resource_manager_init(base, bdev, size); + ttm_set_driver_manager(bdev, mem_type, base); + ttm_resource_manager_set_used(base, true); + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_mock_manager_init); + +void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type) +{ + struct ttm_resource_manager *man; + struct ttm_mock_manager *mock_man; + int err; + + man = ttm_manager_type(bdev, mem_type); + mock_man = to_mock_mgr(man); + + err = ttm_resource_manager_evict_all(bdev, man); + if (err) + return; + + ttm_resource_manager_set_used(man, false); + + mutex_lock(&mock_man->lock); + gpu_buddy_fini(&mock_man->mm); + mutex_unlock(&mock_man->lock); + + ttm_set_driver_manager(bdev, mem_type, NULL); +} +EXPORT_SYMBOL_GPL(ttm_mock_manager_fini); + +static int ttm_bad_manager_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + return -ENOSPC; +} + +static int ttm_busy_manager_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + return -EBUSY; +} + +static void ttm_bad_manager_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ +} + +static bool ttm_bad_manager_compatible(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + return true; +} + +static const struct ttm_resource_manager_func ttm_bad_manager_funcs = { + .alloc = ttm_bad_manager_alloc, + .free = ttm_bad_manager_free, + .compatible = ttm_bad_manager_compatible +}; + +static const struct ttm_resource_manager_func ttm_bad_busy_manager_funcs = { + .alloc = ttm_busy_manager_alloc, + .free = ttm_bad_manager_free, + .compatible = ttm_bad_manager_compatible +}; + +int ttm_bad_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) +{ + struct ttm_resource_manager *man; + + man = kzalloc_obj(*man); + if (!man) + return -ENOMEM; + + man->func = &ttm_bad_manager_funcs; + + ttm_resource_manager_init(man, bdev, size); + ttm_set_driver_manager(bdev, mem_type, man); + ttm_resource_manager_set_used(man, true); + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_bad_manager_init); + +int ttm_busy_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) +{ + struct ttm_resource_manager *man; + + ttm_bad_manager_init(bdev, mem_type, size); + man = ttm_manager_type(bdev, mem_type); + + man->func = &ttm_bad_busy_manager_funcs; + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_busy_manager_init); + +void ttm_bad_manager_fini(struct ttm_device *bdev, uint32_t mem_type) +{ + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, mem_type); + + ttm_resource_manager_set_used(man, false); + ttm_set_driver_manager(bdev, mem_type, NULL); + + kfree(man); +} +EXPORT_SYMBOL_GPL(ttm_bad_manager_fini); + +MODULE_DESCRIPTION("KUnit tests for ttm with mock resource managers"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h new file mode 100644 index 000000000..08710756f --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2023 Intel Corporation + */ +#ifndef TTM_MOCK_MANAGER_H +#define TTM_MOCK_MANAGER_H + +#include + +struct ttm_mock_manager { + struct ttm_resource_manager man; + struct gpu_buddy mm; + u64 default_page_size; + /* protects allocations of mock buffer objects */ + struct mutex lock; +}; + +struct ttm_mock_resource { + struct ttm_resource base; + struct list_head blocks; + unsigned long flags; +}; + +int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size); +int ttm_bad_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size); +int ttm_busy_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size); +void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type); +void ttm_bad_manager_fini(struct ttm_device *bdev, u32 mem_type); + +#endif // TTM_MOCK_MANAGER_H diff --git a/drivers/gpu/drm/ttm/tests/ttm_pool_test.c b/drivers/gpu/drm/ttm/tests/ttm_pool_test.c new file mode 100644 index 000000000..be75c8abf --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_pool_test.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include + +#include +#include + +#include "ttm_kunit_helpers.h" +#include "../ttm_pool_internal.h" + +struct ttm_pool_test_case { + const char *description; + unsigned int order; + unsigned int alloc_flags; +}; + +struct ttm_pool_test_priv { + struct ttm_test_devices *devs; + + /* Used to create mock ttm_tts */ + struct ttm_buffer_object *mock_bo; +}; + +static struct ttm_operation_ctx simple_ctx = { + .interruptible = true, + .no_wait_gpu = false, +}; + +static int ttm_pool_test_init(struct kunit *test) +{ + struct ttm_pool_test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv->devs = ttm_test_devices_basic(test); + test->priv = priv; + + return 0; +} + +static void ttm_pool_test_fini(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + + ttm_test_devices_put(test, priv->devs); +} + +static struct ttm_tt *ttm_tt_kunit_init(struct kunit *test, + u32 page_flags, + enum ttm_caching caching, + size_t size) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err; + + bo = ttm_bo_kunit_init(test, priv->devs, size, NULL); + KUNIT_ASSERT_NOT_NULL(test, bo); + priv->mock_bo = bo; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, priv->mock_bo, page_flags, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + return tt; +} + +static struct ttm_pool *ttm_pool_pre_populated(struct kunit *test, + size_t size, + enum ttm_caching caching) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + struct ttm_pool *pool; + struct ttm_tt *tt; + int err; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, TTM_ALLOCATION_POOL_USE_DMA_ALLOC); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + return pool; +} + +static const struct ttm_pool_test_case ttm_pool_basic_cases[] = { + { + .description = "One page", + .order = 0, + }, + { + .description = "More than one page", + .order = 2, + }, + { + .description = "Above the allocation limit", + .order = MAX_PAGE_ORDER + 1, + }, + { + .description = "One page, with coherent DMA mappings enabled", + .order = 0, + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC, + }, + { + .description = "Above the allocation limit, with coherent DMA mappings enabled", + .order = MAX_PAGE_ORDER + 1, + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC, + }, +}; + +static void ttm_pool_alloc_case_desc(const struct ttm_pool_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_pool_alloc_basic, ttm_pool_basic_cases, + ttm_pool_alloc_case_desc); + +static void ttm_pool_alloc_basic(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + const struct ttm_pool_test_case *params = test->param_value; + struct ttm_tt *tt; + struct ttm_pool *pool; + struct page *fst_page, *last_page; + enum ttm_caching caching = ttm_uncached; + unsigned int expected_num_pages = 1 << params->order; + size_t size = expected_num_pages * PAGE_SIZE; + int err; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, params->alloc_flags); + + KUNIT_ASSERT_PTR_EQ(test, pool->dev, devs->dev); + KUNIT_ASSERT_EQ(test, pool->nid, NUMA_NO_NODE); + KUNIT_ASSERT_EQ(test, pool->alloc_flags, params->alloc_flags); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, tt->num_pages, expected_num_pages); + + fst_page = tt->pages[0]; + last_page = tt->pages[tt->num_pages - 1]; + + if (params->order <= MAX_PAGE_ORDER) { + if (ttm_pool_uses_dma_alloc(pool)) { + KUNIT_ASSERT_NOT_NULL(test, (void *)fst_page->private); + KUNIT_ASSERT_NOT_NULL(test, (void *)last_page->private); + } else { + KUNIT_ASSERT_EQ(test, fst_page->private, params->order); + } + } else { + if (ttm_pool_uses_dma_alloc(pool)) { + KUNIT_ASSERT_NOT_NULL(test, (void *)fst_page->private); + KUNIT_ASSERT_NULL(test, (void *)last_page->private); + } else { + /* + * We expect to alloc one big block, followed by + * order 0 blocks + */ + KUNIT_ASSERT_EQ(test, fst_page->private, + min_t(unsigned int, MAX_PAGE_ORDER, + params->order)); + KUNIT_ASSERT_EQ(test, last_page->private, 0); + } + } + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_basic_dma_addr(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + const struct ttm_pool_test_case *params = test->param_value; + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_buffer_object *bo; + dma_addr_t dma1, dma2; + enum ttm_caching caching = ttm_uncached; + unsigned int expected_num_pages = 1 << params->order; + size_t size = expected_num_pages * PAGE_SIZE; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, devs, size, NULL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + err = ttm_sg_tt_init(tt, bo, 0, caching); + KUNIT_ASSERT_EQ(test, err, 0); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, TTM_ALLOCATION_POOL_USE_DMA_ALLOC); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, tt->num_pages, expected_num_pages); + + dma1 = tt->dma_address[0]; + dma2 = tt->dma_address[tt->num_pages - 1]; + + KUNIT_ASSERT_NOT_NULL(test, (void *)(uintptr_t)dma1); + KUNIT_ASSERT_NOT_NULL(test, (void *)(uintptr_t)dma2); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_order_caching_match(struct kunit *test) +{ + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 0; + size_t size = PAGE_SIZE; + int err; + + pool = ttm_pool_pre_populated(test, size, caching); + + pt = &pool->caching[caching].orders[order]; + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt->pages)); + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt->pages)); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_caching_mismatch(struct kunit *test) +{ + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt_pool, *pt_tt; + enum ttm_caching tt_caching = ttm_uncached; + enum ttm_caching pool_caching = ttm_cached; + size_t size = PAGE_SIZE; + unsigned int order = 0; + int err; + + pool = ttm_pool_pre_populated(test, size, pool_caching); + + pt_pool = &pool->caching[pool_caching].orders[order]; + pt_tt = &pool->caching[tt_caching].orders[order]; + + tt = ttm_tt_kunit_init(test, 0, tt_caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt_tt->pages)); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_tt->pages)); + + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_order_mismatch(struct kunit *test) +{ + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt_pool, *pt_tt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 2; + size_t fst_size = (1 << order) * PAGE_SIZE; + size_t snd_size = PAGE_SIZE; + int err; + + pool = ttm_pool_pre_populated(test, fst_size, caching); + + pt_pool = &pool->caching[caching].orders[order]; + pt_tt = &pool->caching[caching].orders[0]; + + tt = ttm_tt_kunit_init(test, 0, caching, snd_size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt_tt->pages)); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_tt->pages)); + + ttm_pool_fini(pool); +} + +static void ttm_pool_free_dma_alloc(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 2; + size_t size = (1 << order) * PAGE_SIZE; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, TTM_ALLOCATION_POOL_USE_DMA_ALLOC); + ttm_pool_alloc(pool, tt, &simple_ctx); + + pt = &pool->caching[caching].orders[order]; + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt->pages)); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt->pages)); + + ttm_pool_fini(pool); +} + +static void ttm_pool_free_no_dma_alloc(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + struct ttm_tt *tt; + struct ttm_pool *pool; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 2; + size_t size = (1 << order) * PAGE_SIZE; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, 0); + ttm_pool_alloc(pool, tt, &simple_ctx); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + ttm_pool_fini(pool); +} + +static void ttm_pool_fini_basic(struct kunit *test) +{ + struct ttm_pool *pool; + struct ttm_pool_type *pt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 0; + size_t size = PAGE_SIZE; + + pool = ttm_pool_pre_populated(test, size, caching); + pt = &pool->caching[caching].orders[order]; + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt->pages)); + + ttm_pool_fini(pool); + + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt->pages)); +} + +static struct kunit_case ttm_pool_test_cases[] = { + KUNIT_CASE_PARAM(ttm_pool_alloc_basic, ttm_pool_alloc_basic_gen_params), + KUNIT_CASE_PARAM(ttm_pool_alloc_basic_dma_addr, + ttm_pool_alloc_basic_gen_params), + KUNIT_CASE(ttm_pool_alloc_order_caching_match), + KUNIT_CASE(ttm_pool_alloc_caching_mismatch), + KUNIT_CASE(ttm_pool_alloc_order_mismatch), + KUNIT_CASE(ttm_pool_free_dma_alloc), + KUNIT_CASE(ttm_pool_free_no_dma_alloc), + KUNIT_CASE(ttm_pool_fini_basic), + {} +}; + +static struct kunit_suite ttm_pool_test_suite = { + .name = "ttm_pool", + .init = ttm_pool_test_init, + .exit = ttm_pool_test_fini, + .test_cases = ttm_pool_test_cases, +}; + +kunit_test_suites(&ttm_pool_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_pool APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_resource_test.c b/drivers/gpu/drm/ttm/tests/ttm_resource_test.c new file mode 100644 index 000000000..c0e4e35e0 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_resource_test.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include + +#include "ttm_kunit_helpers.h" + +#define RES_SIZE SZ_4K +#define TTM_PRIV_DUMMY_REG (TTM_NUM_MEM_TYPES - 1) + +struct ttm_resource_test_case { + const char *description; + u32 mem_type; + u32 flags; +}; + +struct ttm_resource_test_priv { + struct ttm_test_devices *devs; + struct ttm_buffer_object *bo; + struct ttm_place *place; +}; + +static const struct ttm_resource_manager_func ttm_resource_manager_mock_funcs = { }; + +static int ttm_resource_test_init(struct kunit *test) +{ + struct ttm_resource_test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv->devs = ttm_test_devices_all(test); + KUNIT_ASSERT_NOT_NULL(test, priv->devs); + + test->priv = priv; + + return 0; +} + +static void ttm_resource_test_fini(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + + ttm_test_devices_put(test, priv->devs); +} + +static void ttm_init_test_mocks(struct kunit *test, + struct ttm_resource_test_priv *priv, + u32 mem_type, u32 flags) +{ + size_t size = RES_SIZE; + + /* Make sure we have what we need for a good BO mock */ + KUNIT_ASSERT_NOT_NULL(test, priv->devs->ttm_dev); + + priv->bo = ttm_bo_kunit_init(test, priv->devs, size, NULL); + priv->place = ttm_place_kunit_init(test, mem_type, flags); +} + +static void ttm_init_test_manager(struct kunit *test, + struct ttm_resource_test_priv *priv, + u32 mem_type) +{ + struct ttm_device *ttm_dev = priv->devs->ttm_dev; + struct ttm_resource_manager *man; + size_t size = SZ_16K; + + man = kunit_kzalloc(test, sizeof(*man), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, man); + + man->use_tt = false; + man->func = &ttm_resource_manager_mock_funcs; + + ttm_resource_manager_init(man, ttm_dev, size); + ttm_set_driver_manager(ttm_dev, mem_type, man); + ttm_resource_manager_set_used(man, true); +} + +static const struct ttm_resource_test_case ttm_resource_cases[] = { + { + .description = "Init resource in TTM_PL_SYSTEM", + .mem_type = TTM_PL_SYSTEM, + }, + { + .description = "Init resource in TTM_PL_VRAM", + .mem_type = TTM_PL_VRAM, + }, + { + .description = "Init resource in a private placement", + .mem_type = TTM_PRIV_DUMMY_REG, + }, + { + .description = "Init resource in TTM_PL_SYSTEM, set placement flags", + .mem_type = TTM_PL_SYSTEM, + .flags = TTM_PL_FLAG_TOPDOWN, + }, +}; + +static void ttm_resource_case_desc(const struct ttm_resource_test_case *t, char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_resource, ttm_resource_cases, ttm_resource_case_desc); + +static void ttm_resource_init_basic(struct kunit *test) +{ + const struct ttm_resource_test_case *params = test->param_value; + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + u64 expected_usage; + + ttm_init_test_mocks(test, priv, params->mem_type, params->flags); + bo = priv->bo; + place = priv->place; + + if (params->mem_type > TTM_PL_SYSTEM) + ttm_init_test_manager(test, priv, params->mem_type); + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + expected_usage = man->usage + RES_SIZE; + + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[bo->priority])); + + ttm_resource_init(bo, place, res); + + KUNIT_ASSERT_EQ(test, res->start, 0); + KUNIT_ASSERT_EQ(test, res->size, RES_SIZE); + KUNIT_ASSERT_EQ(test, res->mem_type, place->mem_type); + KUNIT_ASSERT_EQ(test, res->placement, place->flags); + KUNIT_ASSERT_PTR_EQ(test, res->bo, bo); + + KUNIT_ASSERT_NULL(test, res->bus.addr); + KUNIT_ASSERT_EQ(test, res->bus.offset, 0); + KUNIT_ASSERT_FALSE(test, res->bus.is_iomem); + KUNIT_ASSERT_EQ(test, res->bus.caching, ttm_cached); + KUNIT_ASSERT_EQ(test, man->usage, expected_usage); + + KUNIT_ASSERT_TRUE(test, list_is_singular(&man->lru[bo->priority])); + + ttm_resource_fini(man, res); +} + +static void ttm_resource_init_pinned(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + + ttm_init_test_mocks(test, priv, TTM_PL_SYSTEM, 0); + bo = priv->bo; + place = priv->place; + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + KUNIT_ASSERT_TRUE(test, list_empty(&bo->bdev->unevictable)); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_pin(bo); + ttm_resource_init(bo, place, res); + KUNIT_ASSERT_TRUE(test, list_is_singular(&bo->bdev->unevictable)); + + ttm_bo_unpin(bo); + ttm_resource_fini(man, res); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_TRUE(test, list_empty(&bo->bdev->unevictable)); +} + +static void ttm_resource_fini_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + + ttm_init_test_mocks(test, priv, TTM_PL_SYSTEM, 0); + bo = priv->bo; + place = priv->place; + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + ttm_resource_init(bo, place, res); + ttm_resource_fini(man, res); + + KUNIT_ASSERT_TRUE(test, list_empty(&res->lru.link)); + KUNIT_ASSERT_EQ(test, man->usage, 0); +} + +static void ttm_resource_manager_init_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + size_t size = SZ_16K; + int i; + + man = kunit_kzalloc(test, sizeof(*man), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, man); + + ttm_resource_manager_init(man, priv->devs->ttm_dev, size); + + KUNIT_ASSERT_PTR_EQ(test, man->bdev, priv->devs->ttm_dev); + KUNIT_ASSERT_EQ(test, man->size, size); + KUNIT_ASSERT_EQ(test, man->usage, 0); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) + KUNIT_ASSERT_NULL(test, man->eviction_fences[i]); + + for (int i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[i])); +} + +static void ttm_resource_manager_usage_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + u64 actual_usage; + + ttm_init_test_mocks(test, priv, TTM_PL_SYSTEM, TTM_PL_FLAG_TOPDOWN); + bo = priv->bo; + place = priv->place; + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + + ttm_resource_init(bo, place, res); + actual_usage = ttm_resource_manager_usage(man); + + KUNIT_ASSERT_EQ(test, actual_usage, RES_SIZE); + + ttm_resource_fini(man, res); +} + +static void ttm_resource_manager_set_used_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + + man = ttm_manager_type(priv->devs->ttm_dev, TTM_PL_SYSTEM); + KUNIT_ASSERT_TRUE(test, man->use_type); + + ttm_resource_manager_set_used(man, false); + KUNIT_ASSERT_FALSE(test, man->use_type); +} + +static void ttm_sys_man_alloc_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource *res; + u32 mem_type = TTM_PL_SYSTEM; + int ret; + + ttm_init_test_mocks(test, priv, mem_type, 0); + bo = priv->bo; + place = priv->place; + + man = ttm_manager_type(priv->devs->ttm_dev, mem_type); + ret = man->func->alloc(man, bo, place, &res); + + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, res->size, RES_SIZE); + KUNIT_ASSERT_EQ(test, res->mem_type, mem_type); + KUNIT_ASSERT_PTR_EQ(test, res->bo, bo); + + ttm_resource_fini(man, res); +} + +static void ttm_sys_man_free_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource *res; + u32 mem_type = TTM_PL_SYSTEM; + + ttm_init_test_mocks(test, priv, mem_type, 0); + bo = priv->bo; + place = priv->place; + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + ttm_resource_alloc(bo, place, &res, NULL); + + man = ttm_manager_type(priv->devs->ttm_dev, mem_type); + man->func->free(man, res); + + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[bo->priority])); + KUNIT_ASSERT_EQ(test, man->usage, 0); +} + +static struct kunit_case ttm_resource_test_cases[] = { + KUNIT_CASE_PARAM(ttm_resource_init_basic, ttm_resource_gen_params), + KUNIT_CASE(ttm_resource_init_pinned), + KUNIT_CASE(ttm_resource_fini_basic), + KUNIT_CASE(ttm_resource_manager_init_basic), + KUNIT_CASE(ttm_resource_manager_usage_basic), + KUNIT_CASE(ttm_resource_manager_set_used_basic), + KUNIT_CASE(ttm_sys_man_alloc_basic), + KUNIT_CASE(ttm_sys_man_free_basic), + {} +}; + +static struct kunit_suite ttm_resource_test_suite = { + .name = "ttm_resource", + .init = ttm_resource_test_init, + .exit = ttm_resource_test_fini, + .test_cases = ttm_resource_test_cases, +}; + +kunit_test_suites(&ttm_resource_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_resource and ttm_sys_man APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_tt_test.c b/drivers/gpu/drm/ttm/tests/ttm_tt_test.c new file mode 100644 index 000000000..bd5f7d0b9 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_tt_test.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include +#include + +#include "ttm_kunit_helpers.h" + +#define BO_SIZE SZ_4K + +struct ttm_tt_test_case { + const char *description; + u32 size; + u32 extra_pages_num; +}; + +static const struct ttm_tt_test_case ttm_tt_init_basic_cases[] = { + { + .description = "Page-aligned size", + .size = SZ_4K, + }, + { + .description = "Extra pages requested", + .size = SZ_4K, + .extra_pages_num = 1, + }, +}; + +static void ttm_tt_init_case_desc(const struct ttm_tt_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_tt_init_basic, ttm_tt_init_basic_cases, + ttm_tt_init_case_desc); + +static void ttm_tt_init_basic(struct kunit *test) +{ + const struct ttm_tt_test_case *params = test->param_value; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + u32 page_flags = TTM_TT_FLAG_ZERO_ALLOC; + enum ttm_caching caching = ttm_cached; + u32 extra_pages = params->extra_pages_num; + int num_pages = params->size >> PAGE_SHIFT; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, params->size, NULL); + + err = ttm_tt_init(tt, bo, page_flags, caching, extra_pages); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_ASSERT_EQ(test, tt->num_pages, num_pages + extra_pages); + + KUNIT_ASSERT_EQ(test, tt->page_flags, page_flags); + KUNIT_ASSERT_EQ(test, tt->caching, caching); + + KUNIT_ASSERT_NULL(test, tt->dma_address); + KUNIT_ASSERT_NULL(test, tt->swap_storage); +} + +static void ttm_tt_init_misaligned(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + u32 size = SZ_8K; + int num_pages = (size + SZ_4K) >> PAGE_SHIFT; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + + /* Make the object size misaligned */ + bo->base.size += 1; + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_ASSERT_EQ(test, tt->num_pages, num_pages); +} + +static void ttm_tt_fini_basic(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, tt->pages); + + ttm_tt_fini(tt); + KUNIT_ASSERT_NULL(test, tt->pages); +} + +static void ttm_tt_fini_sg(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_sg_tt_init(tt, bo, 0, caching); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, tt->dma_address); + + ttm_tt_fini(tt); + KUNIT_ASSERT_NULL(test, tt->dma_address); +} + +static void ttm_tt_fini_shmem(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + struct file *shmem; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + shmem = shmem_file_setup("ttm swap", BO_SIZE, EMPTY_VMA_FLAGS); + tt->swap_storage = shmem; + + ttm_tt_fini(tt); + KUNIT_ASSERT_NULL(test, tt->swap_storage); +} + +static void ttm_tt_create_basic(struct kunit *test) +{ + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_device; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_NOT_NULL(test, bo->ttm); + + /* Free manually, as it was allocated outside of KUnit */ + kfree(bo->ttm); +} + +static void ttm_tt_create_invalid_bo_type(struct kunit *test) +{ + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_sg + 1; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -EINVAL); + KUNIT_EXPECT_NULL(test, bo->ttm); +} + +static void ttm_tt_create_ttm_exists(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + bo->ttm = tt; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + /* Expect to keep the previous TTM */ + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_PTR_EQ(test, tt, bo->ttm); +} + +static struct ttm_tt *ttm_tt_null_create(struct ttm_buffer_object *bo, + u32 page_flags) +{ + return NULL; +} + +static struct ttm_device_funcs ttm_dev_empty_funcs = { + .ttm_tt_create = ttm_tt_null_create, +}; + +static void ttm_tt_create_failed(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + /* Update ttm_device_funcs so we don't alloc ttm_tt */ + devs->ttm_dev->funcs = &ttm_dev_empty_funcs; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, -ENOMEM); +} + +static void ttm_tt_destroy_basic(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, bo->ttm); + + ttm_tt_destroy(devs->ttm_dev, bo->ttm); +} + +static void ttm_tt_populate_null_ttm(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_operation_ctx ctx = { }; + int err; + + err = ttm_tt_populate(devs->ttm_dev, NULL, &ctx); + KUNIT_ASSERT_EQ(test, err, -EINVAL); +} + +static void ttm_tt_populate_populated_ttm(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_operation_ctx ctx = { }; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + struct page *populated_page; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + populated_page = *tt->pages; + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_PTR_EQ(test, populated_page, *tt->pages); +} + +static void ttm_tt_unpopulate_basic(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_operation_ctx ctx = { }; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_TRUE(test, ttm_tt_is_populated(tt)); + + ttm_tt_unpopulate(devs->ttm_dev, tt); + KUNIT_ASSERT_FALSE(test, ttm_tt_is_populated(tt)); +} + +static void ttm_tt_unpopulate_empty_ttm(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_tt_unpopulate(devs->ttm_dev, tt); + /* Expect graceful handling of unpopulated TTs */ +} + +static void ttm_tt_swapin_basic(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + int expected_num_pages = BO_SIZE >> PAGE_SHIFT; + struct ttm_operation_ctx ctx = { }; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err, num_pages; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_TRUE(test, ttm_tt_is_populated(tt)); + + num_pages = ttm_tt_swapout(devs->ttm_dev, tt, GFP_KERNEL); + KUNIT_ASSERT_EQ(test, num_pages, expected_num_pages); + KUNIT_ASSERT_NOT_NULL(test, tt->swap_storage); + KUNIT_ASSERT_TRUE(test, tt->page_flags & TTM_TT_FLAG_SWAPPED); + + /* Swapout depopulates TT, allocate pages and then swap them in */ + err = ttm_pool_alloc(&devs->ttm_dev->pool, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_swapin(tt); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NULL(test, tt->swap_storage); + KUNIT_ASSERT_FALSE(test, tt->page_flags & TTM_TT_FLAG_SWAPPED); +} + +static struct kunit_case ttm_tt_test_cases[] = { + KUNIT_CASE_PARAM(ttm_tt_init_basic, ttm_tt_init_basic_gen_params), + KUNIT_CASE(ttm_tt_init_misaligned), + KUNIT_CASE(ttm_tt_fini_basic), + KUNIT_CASE(ttm_tt_fini_sg), + KUNIT_CASE(ttm_tt_fini_shmem), + KUNIT_CASE(ttm_tt_create_basic), + KUNIT_CASE(ttm_tt_create_invalid_bo_type), + KUNIT_CASE(ttm_tt_create_ttm_exists), + KUNIT_CASE(ttm_tt_create_failed), + KUNIT_CASE(ttm_tt_destroy_basic), + KUNIT_CASE(ttm_tt_populate_null_ttm), + KUNIT_CASE(ttm_tt_populate_populated_ttm), + KUNIT_CASE(ttm_tt_unpopulate_basic), + KUNIT_CASE(ttm_tt_unpopulate_empty_ttm), + KUNIT_CASE(ttm_tt_swapin_basic), + {} +}; + +static struct kunit_suite ttm_tt_test_suite = { + .name = "ttm_tt", + .init = ttm_test_devices_all_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_tt_test_cases, +}; + +kunit_test_suites(&ttm_tt_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_tt APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c new file mode 100644 index 000000000..147e25ee5 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + * Keith Packard. + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ttm_agp_backend { + struct ttm_tt ttm; + struct agp_memory *mem; + struct agp_bridge_data *bridge; +}; + +int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_resource *bo_mem) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + struct page *dummy_read_page = ttm_glob.dummy_read_page; + struct agp_memory *mem; + int ret, cached = ttm->caching == ttm_cached; + unsigned i; + + if (agp_be->mem) + return 0; + + mem = agp_allocate_memory(agp_be->bridge, ttm->num_pages, AGP_USER_MEMORY); + if (unlikely(mem == NULL)) + return -ENOMEM; + + mem->page_count = 0; + for (i = 0; i < ttm->num_pages; i++) { + struct page *page = ttm->pages[i]; + + if (!page) + page = dummy_read_page; + + mem->pages[mem->page_count++] = page; + } + agp_be->mem = mem; + + mem->is_flushed = 1; + mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; + + ret = agp_bind_memory(mem, bo_mem->start); + if (ret) + pr_err("AGP Bind memory failed\n"); + + return ret; +} +EXPORT_SYMBOL(ttm_agp_bind); + +void ttm_agp_unbind(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) { + if (agp_be->mem->is_bound) { + agp_unbind_memory(agp_be->mem); + return; + } + agp_free_memory(agp_be->mem); + agp_be->mem = NULL; + } +} +EXPORT_SYMBOL(ttm_agp_unbind); + +bool ttm_agp_is_bound(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (!ttm) + return false; + + return (agp_be->mem != NULL); +} +EXPORT_SYMBOL(ttm_agp_is_bound); + +void ttm_agp_destroy(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) + ttm_agp_unbind(ttm); + ttm_tt_fini(ttm); + kfree(agp_be); +} +EXPORT_SYMBOL(ttm_agp_destroy); + +struct ttm_tt *ttm_agp_tt_create(struct ttm_buffer_object *bo, + struct agp_bridge_data *bridge, + uint32_t page_flags) +{ + struct ttm_agp_backend *agp_be; + + agp_be = kmalloc_obj(*agp_be); + if (!agp_be) + return NULL; + + agp_be->mem = NULL; + agp_be->bridge = bridge; + + if (ttm_tt_init(&agp_be->ttm, bo, page_flags, ttm_write_combined, 0)) { + kfree(agp_be); + return NULL; + } + + return &agp_be->ttm; +} +EXPORT_SYMBOL(ttm_agp_tt_create); diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c new file mode 100644 index 000000000..0c2d53a13 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include + +#include +#include + +#include "ttm_pool_internal.h" + +/* + * Need to map shmem indices to handle since a handle value + * of 0 means error, following the swp_entry_t convention. + */ +static unsigned long ttm_backup_shmem_idx_to_handle(pgoff_t idx) +{ + return (unsigned long)idx + 1; +} + +static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle) +{ + return handle - 1; +} + +/** + * ttm_backup_drop() - release memory associated with a handle + * @backup: The struct backup pointer used to obtain the handle + * @handle: The handle obtained from the @backup_page function. + */ +void ttm_backup_drop(struct file *backup, pgoff_t handle) +{ + loff_t start = ttm_backup_handle_to_shmem_idx(handle); + + start <<= PAGE_SHIFT; + shmem_truncate_range(file_inode(backup), start, + start + PAGE_SIZE - 1); +} + +/** + * ttm_backup_copy_page() - Copy the contents of a previously backed + * up page + * @backup: The struct backup pointer used to back up the page. + * @dst: The struct page to copy into. + * @handle: The handle returned when the page was backed up. + * @intr: Try to perform waits interruptible or at least killable. + * @additional_gfp: GFP mask to add to the default GFP mask if any. + * + * Return: 0 on success, Negative error code on failure, notably + * -EINTR if @intr was set to true and a signal is pending. + */ +int ttm_backup_copy_page(struct file *backup, struct page *dst, + pgoff_t handle, bool intr, gfp_t additional_gfp) +{ + struct address_space *mapping = backup->f_mapping; + struct folio *from_folio; + pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle); + + from_folio = shmem_read_folio_gfp(mapping, idx, mapping_gfp_mask(mapping) + | additional_gfp); + if (IS_ERR(from_folio)) + return PTR_ERR(from_folio); + + copy_highpage(dst, folio_file_page(from_folio, idx)); + folio_put(from_folio); + + return 0; +} + +/** + * ttm_backup_backup_folio() - Backup a folio + * @backup: The struct backup pointer to use. + * @folio: The folio to back up. + * @order: The allocation order of @folio. Since TTM allocates higher-order + * pages without __GFP_COMP, folio_nr_pages(@folio) would always + * return 1; the caller must pass the true order explicitly. + * @writeback: Whether to perform immediate writeback of the folio's pages. + * This may have performance implications. + * @idx: A unique integer for the first page of the folio and each struct backup. + * This allows the backup implementation to avoid managing + * its address space separately. + * @folio_gfp: The gfp value used when the folio was allocated. + * Currently unused. + * @alloc_gfp: The gfp to be used when allocating memory. + * @nr_pages_backed: Output. On a successful return, set to the number of + * pages actually backed up, which may be less than (1 << @order) + * if an -ENOMEM was encountered mid-folio. + * + * Context: If called from reclaim context, the caller needs to + * assert that the shrinker gfp has __GFP_FS set, to avoid + * deadlocking on lock_page(). If @writeback is set to true and + * called from reclaim context, the caller also needs to assert + * that the shrinker gfp has __GFP_IO set, since without it, + * we're not allowed to start backup IO. + * + * Return: A handle for the first backed-up page on success (handles for + * subsequent pages follow sequentially). -ENOMEM if no pages could be backed + * up. Any other negative error code if a non-ENOMEM failure occurred; in that + * case any pages backed up so far are truncated before returning. + */ +s64 +ttm_backup_backup_folio(struct file *backup, struct folio *folio, + unsigned int order, bool writeback, pgoff_t idx, + gfp_t folio_gfp, gfp_t alloc_gfp, + pgoff_t *nr_pages_backed) +{ + struct address_space *mapping = backup->f_mapping; + int nr_pages = 1 << order; + struct folio *to_folio; + int ret, i; + + *nr_pages_backed = 0; + + for (i = 0; i < nr_pages; ) { + int to_nr, j; + + /* + * Only inject past the first subpage so *nr_pages_backed is + * always > 0 here, matching a genuine mid-compound -ENOMEM + * and driving the caller's reactive split fallback instead + * of an early, no-progress failure. + */ + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && i && + ttm_backup_fault_inject_folio()) + to_folio = ERR_PTR(-ENOMEM); + else + to_folio = shmem_read_folio_gfp(mapping, idx + i, alloc_gfp); + if (IS_ERR(to_folio)) { + int err = PTR_ERR(to_folio); + + if (err == -ENOMEM && *nr_pages_backed) + return ttm_backup_shmem_idx_to_handle(idx); + + if (*nr_pages_backed) { + shmem_truncate_range(file_inode(backup), + (loff_t)idx << PAGE_SHIFT, + ((loff_t)(idx + i) << PAGE_SHIFT) - 1); + /* + * The pages just truncated are no longer + * backed up; don't let the caller mistake + * them for valid handles. + */ + *nr_pages_backed = 0; + } + return err; + } + + to_nr = min_t(int, nr_pages - i, + folio_next_index(to_folio) - (idx + i)); + + folio_mark_accessed(to_folio); + folio_lock(to_folio); + folio_mark_dirty(to_folio); + + for (j = 0; j < to_nr; j++) + copy_highpage(folio_file_page(to_folio, idx + i + j), + folio_page(folio, i + j)); + + if (writeback && !folio_mapped(to_folio) && + folio_clear_dirty_for_io(to_folio)) { + folio_set_reclaim(to_folio); + ret = shmem_write_folio(to_folio); + if (!folio_test_writeback(to_folio)) + folio_clear_reclaim(to_folio); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(to_folio); + } else { + folio_unlock(to_folio); + } + + folio_put(to_folio); + i += to_nr; + *nr_pages_backed = i; + } + + return ttm_backup_shmem_idx_to_handle(idx); +} + +/** + * ttm_backup_fini() - Free the struct backup resources after last use. + * @backup: Pointer to the struct backup whose resources to free. + * + * After a call to this function, it's illegal to use the @backup pointer. + */ +void ttm_backup_fini(struct file *backup) +{ + fput(backup); +} + +/** + * ttm_backup_bytes_avail() - Report the approximate number of bytes of backup space + * left for backup. + * + * This function is intended also for driver use to indicate whether a + * backup attempt is meaningful. + * + * Return: An approximate size of backup space available. + */ +u64 ttm_backup_bytes_avail(void) +{ + /* + * The idea behind backing up to shmem is that shmem objects may + * eventually be swapped out. So no point swapping out if there + * is no or low swap-space available. But the accuracy of this + * number also depends on shmem actually swapping out backed-up + * shmem objects without too much buffering. + */ + return (u64)get_nr_swap_pages() << PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail); + +/** + * ttm_backup_shmem_create() - Create a shmem-based struct backup. + * @size: The maximum size (in bytes) to back up. + * + * Create a backup utilizing shmem objects. + * + * Return: A pointer to a struct file on success, + * an error pointer on error. + */ +struct file *ttm_backup_shmem_create(loff_t size) +{ + return shmem_file_setup("ttm shmem backup", size, + EMPTY_VMA_FLAGS); +} diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c new file mode 100644 index 000000000..ef56c18de --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -0,0 +1,1556 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ttm_module.h" +#include "ttm_bo_internal.h" + +static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX); + struct ttm_resource_manager *man; + int i, mem_type; + + for (i = 0; i < placement->num_placement; i++) { + mem_type = placement->placement[i].mem_type; + drm_printf(&p, " placement[%d]=0x%08X (%d)\n", + i, placement->placement[i].flags, mem_type); + man = ttm_manager_type(bo->bdev, mem_type); + ttm_resource_manager_debug(man, &p); + } +} + +/** + * ttm_bo_move_to_lru_tail + * + * @bo: The buffer object. + * + * Move this BO to the tail of all lru lists used to lookup and reserve an + * object. This function must be called with struct ttm_global::lru_lock + * held, and is used to make a BO less likely to be considered for eviction. + */ +void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + + if (bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); +} +EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); + +/** + * ttm_bo_set_bulk_move - update BOs bulk move object + * + * @bo: The buffer object. + * @bulk: bulk move structure + * + * Update the BOs bulk move object, making sure that resources are added/removed + * as well. A bulk move allows to move many resource on the LRU at once, + * resulting in much less overhead of maintaining the LRU. + * The only requirement is that the resources stay together on the LRU and are + * never separated. This is enforces by setting the bulk_move structure on a BO. + * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of + * their LRU list. + */ +void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, + struct ttm_lru_bulk_move *bulk) +{ + dma_resv_assert_held(bo->base.resv); + + if (bo->bulk_move == bulk) + return; + + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + bo->bulk_move = bulk; + if (bo->resource) + ttm_resource_add_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_set_bulk_move); + +static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + struct ttm_resource *mem, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) +{ + struct ttm_device *bdev = bo->bdev; + bool old_use_tt, new_use_tt; + int ret; + + old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; + new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; + + ttm_bo_unmap_virtual(bo); + + /* + * Create and bind a ttm if required. + */ + + if (new_use_tt) { + /* Zero init the new TTM structure if the old location should + * have used one as well. + */ + ret = ttm_tt_create(bo, old_use_tt); + if (ret) + goto out_err; + + if (mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_bo_populate(bo, ctx); + if (ret) + goto out_err; + } + } + + ret = dma_resv_reserve_fences(bo->base.resv, 1); + if (ret) + goto out_err; + + ret = bdev->funcs->move(bo, evict, ctx, mem, hop); + if (ret) { + if (ret == -EMULTIHOP) + return ret; + goto out_err; + } + + ctx->bytes_moved += bo->base.size; + return 0; + +out_err: + if (!old_use_tt) + ttm_bo_tt_destroy(bo); + + return ret; +} + +/* + * Call bo::reserved. + * Will release GPU memory type usage on destruction. + * This is the place to put in driver specific hooks to release + * driver private resources. + * Will release the bo::reserved lock. + */ + +static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) +{ + if (bo->bdev->funcs->delete_mem_notify) + bo->bdev->funcs->delete_mem_notify(bo); + + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->resource); +} + +static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) +{ + int r; + + if (bo->base.resv == &bo->base._resv) + return 0; + + BUG_ON(!dma_resv_trylock(&bo->base._resv)); + + r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); + dma_resv_unlock(&bo->base._resv); + if (r) + return r; + + if (bo->type != ttm_bo_type_sg) { + /* This works because the BO is about to be destroyed and nobody + * reference it any more. The only tricky case is the trylock on + * the resv object while holding the lru_lock. + */ + spin_lock(&bo->bdev->lru_lock); + bo->base.resv = &bo->base._resv; + spin_unlock(&bo->bdev->lru_lock); + } + + return r; +} + +static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) +{ + struct dma_resv *resv = &bo->base._resv; + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) + dma_fence_enable_signaling(fence); + dma_resv_iter_end(&cursor); +} + +/* + * Block for the dma_resv object to become idle, lock the buffer and clean up + * the resource and tt object. + */ +static void ttm_bo_delayed_delete(struct work_struct *work) +{ + struct ttm_buffer_object *bo; + + bo = container_of(work, typeof(*bo), delayed_delete); + + dma_resv_wait_timeout(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); +} + +static void ttm_bo_release(struct kref *kref) +{ + struct ttm_buffer_object *bo = + container_of(kref, struct ttm_buffer_object, kref); + struct ttm_device *bdev = bo->bdev; + int ret; + + WARN_ON_ONCE(bo->pin_count); + WARN_ON_ONCE(bo->bulk_move); + + if (!bo->deleted) { + ret = ttm_bo_individualize_resv(bo); + if (ret) { + /* Last resort, if we fail to allocate memory for the + * fences block for the BO to become idle + */ + dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, false, + 30 * HZ); + } + + if (bdev->funcs->release_notify) + bdev->funcs->release_notify(bo); + + drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); + ttm_mem_io_free(bdev, bo->resource); + + if (!dma_resv_test_signaled(&bo->base._resv, + DMA_RESV_USAGE_BOOKKEEP) || + (want_init_on_free() && (bo->ttm != NULL)) || + bo->type == ttm_bo_type_sg || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; + + spin_lock(&bdev->lru_lock); + + /* + * Make pinned bos immediately available to + * shrinkers, now that they are queued for + * destruction. + * + * FIXME: QXL is triggering this. Can be removed when the + * driver is fixed. + */ + if (bo->pin_count) { + bo->pin_count = 0; + ttm_resource_move_to_lru_tail(bo->resource); + } + + kref_init(&bo->kref); + spin_unlock(&bdev->lru_lock); + + INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); + + /* Schedule the worker on the closest NUMA node. This + * improves performance since system memory might be + * cleared on free and that is best done on a CPU core + * close to it. + */ + queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete); + return; + } + + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + } + + atomic_dec(&ttm_glob.bo_count); + bo->destroy(bo); +} + +/* TODO: remove! */ +void ttm_bo_put(struct ttm_buffer_object *bo) +{ + kref_put(&bo->kref, ttm_bo_release); +} + +void ttm_bo_fini(struct ttm_buffer_object *bo) +{ + ttm_bo_put(bo); +} +EXPORT_SYMBOL(ttm_bo_fini); + +static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) +{ + struct ttm_placement hop_placement; + struct ttm_resource *hop_mem; + int ret; + + hop_placement.num_placement = 1; + hop_placement.placement = hop; + + /* find space in the bounce domain */ + ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); + if (ret) + return ret; + /* move to the bounce domain */ + ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); + if (ret) { + ttm_resource_free(bo, &hop_mem); + return ret; + } + return 0; +} + +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + struct ttm_resource *evict_mem; + struct ttm_placement placement; + struct ttm_place hop; + int ret = 0; + + memset(&hop, 0, sizeof(hop)); + + dma_resv_assert_held(bo->base.resv); + + placement.num_placement = 0; + bo->bdev->funcs->evict_flags(bo, &placement); + + if (!placement.num_placement) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + return ret; + + /* + * Since we've already synced, this frees backing store + * immediately. + */ + return ttm_bo_pipeline_gutting(bo); + } + + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); + if (ret) { + if (ret != -ERESTARTSYS) { + pr_err("Failed to find memory space for buffer 0x%p eviction\n", + bo); + ttm_bo_mem_space_debug(bo, &placement); + } + goto out; + } + + do { + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret != -EMULTIHOP) + break; + + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + } while (!ret); + + if (ret) { + ttm_resource_free(bo, &evict_mem); + if (ret != -ERESTARTSYS && ret != -EINTR) + pr_err("Buffer eviction failed\n"); + } +out: + return ret; +} + +/** + * ttm_bo_eviction_valuable + * + * @bo: The buffer object to evict + * @place: the placement we need to make room for + * + * Check if it is valuable to evict the BO to make room for the given placement. + */ +bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, + const struct ttm_place *place) +{ + struct ttm_resource *res = bo->resource; + + dma_resv_assert_held(bo->base.resv); + + if (res->mem_type == TTM_PL_SYSTEM) + return true; + + /* Don't evict this BO if it's outside of the + * requested placement range + */ + return ttm_resource_intersects(bo->bdev, res, place, bo->base.size); +} +EXPORT_SYMBOL(ttm_bo_eviction_valuable); + +/** + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. + * @bdev: The ttm device. + * @man: The manager whose bo to evict. + * @ctx: The TTM operation ctx governing the eviction. + * + * Return: 0 if successful or the resource disappeared. Negative error code on error. + */ +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx) +{ + struct ttm_resource_cursor cursor; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + unsigned int mem_type; + int ret = 0; + + spin_lock(&bdev->lru_lock); + ttm_resource_cursor_init(&cursor, man); + res = ttm_resource_manager_first(&cursor); + ttm_resource_cursor_fini(&cursor); + if (!res) { + ret = -ENOENT; + goto out_no_ref; + } + bo = res->bo; + if (!ttm_bo_get_unless_zero(bo)) + goto out_no_ref; + mem_type = res->mem_type; + spin_unlock(&bdev->lru_lock); + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); + if (ret) + goto out_no_lock; + if (!bo->resource || bo->resource->mem_type != mem_type) + goto out_bo_moved; + + if (bo->deleted) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (!ret) + ttm_bo_cleanup_memtype_use(bo); + } else { + ret = ttm_bo_evict(bo, ctx); + } +out_bo_moved: + dma_resv_unlock(bo->base.resv); +out_no_lock: + ttm_bo_put(bo); + return ret; + +out_no_ref: + spin_unlock(&bdev->lru_lock); + return ret; +} + +struct ttm_bo_alloc_state { + /** @charge_pool: The memory pool the resource is charged to */ + struct dmem_cgroup_pool_state *charge_pool; + /** @limit_pool: Which pool limit we should test against */ + struct dmem_cgroup_pool_state *limit_pool; + /** @in_evict: Whether we are currently evicting buffers */ + bool in_evict; + /** @may_try_low: If only unprotected BOs, i.e. BOs whose cgroup + * is exceeding its dmem low/min protection, should be considered for eviction + */ + bool may_try_low; +}; + +/** + * ttm_bo_alloc_at_place - Attempt allocating a BO's backing store in a place + * + * @bo: The buffer to allocate the backing store of + * @place: The place to attempt allocation in + * @ctx: ttm_operation_ctx associated with this allocation + * @force_space: If we should evict buffers to force space + * @res: On allocation success, the resulting struct ttm_resource. + * @alloc_state: Object holding allocation state such as charged cgroups. + * + * Returns: + * -EBUSY: No space available, but allocation should be retried with ttm_bo_evict_alloc. + * -ENOSPC: No space available, allocation should not be retried. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. + * + */ +static int ttm_bo_alloc_at_place(struct ttm_buffer_object *bo, + const struct ttm_place *place, + bool force_space, + struct ttm_resource **res, + struct ttm_bo_alloc_state *alloc_state) +{ + bool may_evict; + int ret; + + may_evict = !alloc_state->in_evict && force_space && + place->mem_type != TTM_PL_SYSTEM; + if (!alloc_state->charge_pool) { + ret = ttm_resource_try_charge(bo, place, &alloc_state->charge_pool, + force_space ? &alloc_state->limit_pool + : NULL); + if (ret) { + /* + * -EAGAIN means the charge failed, which we treat + * like an allocation failure. Therefore, return an + * error code indicating the allocation failed - + * either -EBUSY if the allocation should be + * retried with eviction, or -ENOSPC if there should + * be no second attempt. + */ + if (!alloc_state->in_evict) + alloc_state->may_try_low = may_evict; + if (ret == -EAGAIN) + ret = may_evict ? -EBUSY : -ENOSPC; + return ret; + } + } + + /* + * cgroup protection plays a special role in eviction. + * Conceptually, protection of memory via the dmem cgroup controller + * entitles the protected cgroup to use a certain amount of memory. + * There are two types of protection - the 'low' limit is a + * "best-effort" protection, whereas the 'min' limit provides a hard + * guarantee that memory within the cgroup's allowance will not be + * evicted under any circumstance. + * + * To faithfully model this concept in TTM, we also need to take cgroup + * protection into account when allocating. When allocation in one + * place fails, TTM will default to trying other places first before + * evicting. + * If the allocation is covered by dmem cgroup protection, however, + * this prevents the allocation from using the memory it is "entitled" + * to. To make sure unprotected allocations cannot push new protected + * allocations out of places they are "entitled" to use, we should + * evict buffers not covered by any cgroup protection, if this + * allocation is covered by cgroup protection. + * + * Buffers covered by 'min' protection are a special case - the 'min' + * limit is a stronger guarantee than 'low', and thus buffers protected + * by 'low' but not 'min' should also be considered for eviction. + * Buffers protected by 'min' will never be considered for eviction + * anyway, so the regular eviction path should be triggered here. + * Buffers protected by 'low' but not 'min' will take a special + * eviction path that only evicts buffers covered by neither 'low' or + * 'min' protections. + */ + if (!alloc_state->in_evict) { + may_evict |= dmem_cgroup_below_min(NULL, alloc_state->charge_pool); + alloc_state->may_try_low = may_evict; + + may_evict |= dmem_cgroup_below_low(NULL, alloc_state->charge_pool); + } + + ret = ttm_resource_alloc(bo, place, res, alloc_state->charge_pool); + if (ret) { + if (ret == -ENOSPC && may_evict) + ret = -EBUSY; + return ret; + } + + /* + * Ownership of charge_pool has been transferred to the TTM resource, + * don't make the caller think we still hold a reference to it. + */ + alloc_state->charge_pool = NULL; + return 0; +} + +/** + * struct ttm_bo_evict_walk - Parameters for the evict walk. + */ +struct ttm_bo_evict_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @place: The place passed to the resource allocation. */ + const struct ttm_place *place; + /** @evictor: The buffer object we're trying to make room for. */ + struct ttm_buffer_object *evictor; + /** @res: The allocated resource if any. */ + struct ttm_resource **res; + /** @evicted: Number of successful evictions. */ + unsigned long evicted; + + /** @try_low: Whether we should attempt to evict BO's with low watermark threshold */ + bool try_low; + /** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */ + bool hit_low; + + /** @alloc_state: State associated with the allocation attempt. */ + struct ttm_bo_alloc_state *alloc_state; +}; + +static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) +{ + struct ttm_bo_evict_walk *evict_walk = + container_of(walk, typeof(*evict_walk), walk); + struct dmem_cgroup_pool_state *limit_pool, *ancestor = NULL; + s64 bo_size = bo->base.size; + bool evict_valuable; + s64 lret; + + /* + * If may_try_low is not set, then we're trying to evict unprotected + * buffers in favor of a protected allocation for charge_pool. Explicitly skip + * buffers belonging to the same cgroup here - that cgroup is definitely protected, + * even though dmem_cgroup_state_evict_valuable would allow the eviction because a + * cgroup is always allowed to evict from itself even if it is protected. + */ + if (!evict_walk->alloc_state->may_try_low && + bo->resource->css == evict_walk->alloc_state->charge_pool) + return 0; + + limit_pool = evict_walk->alloc_state->limit_pool; + /* + * If there is no explicit limit pool, find the root of the shared subtree between + * evictor and evictee. This is important so that recursive protection rules can + * apply properly: Recursive protection distributes cgroup protection afforded + * to a parent cgroup but not used explicitly by a child cgroup between all child + * cgroups (see docs of effective_protection in mm/page_counter.c). However, when + * direct siblings compete for memory, siblings that were explicitly protected + * should get prioritized over siblings that weren't. This only happens correctly + * when the root of the shared subtree is passed to + * dmem_cgroup_state_evict_valuable. Otherwise, the effective-protection + * calculation cannot distinguish direct siblings from unrelated subtrees and the + * calculated protection ends up wrong. + */ + if (!limit_pool) { + ancestor = dmem_cgroup_get_common_ancestor(bo->resource->css, + evict_walk->alloc_state->charge_pool); + limit_pool = ancestor; + } + + evict_valuable = dmem_cgroup_state_evict_valuable(limit_pool, bo->resource->css, + evict_walk->try_low, + &evict_walk->hit_low); + if (ancestor) + dmem_cgroup_pool_state_put(ancestor); + + if (!evict_valuable) + return 0; + + /* + * evict_walk->place is NULL in cgroup drain mode. Drivers' + * eviction_valuable() callbacks must handle a NULL place, treating it + * as "any placement": the TTM base implementation already does so via + * ttm_resource_intersects(). + */ + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) + return 0; + + if (bo->deleted) { + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); + if (!lret) + ttm_bo_cleanup_memtype_use(bo); + } else { + lret = ttm_bo_evict(bo, walk->arg.ctx); + } + + if (lret) + goto out; + + evict_walk->evicted++; + if (evict_walk->res) { + lret = ttm_bo_alloc_at_place(evict_walk->evictor, + evict_walk->place, false, + evict_walk->res, + evict_walk->alloc_state); + if (lret == 0) + return 1; + } else { + /* Cgroup drain: return bytes freed for byte-denominated progress. */ + return bo_size; + } +out: + /* Errors that should terminate the walk. */ + if (lret == -ENOSPC) + return -EBUSY; + + return lret; +} + +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { + .process_bo = ttm_bo_evict_cb, +}; + +static int ttm_bo_evict_alloc(struct ttm_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_buffer_object *evictor, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket, + struct ttm_resource **res, + struct ttm_bo_alloc_state *state) +{ + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { + .ctx = ctx, + .ticket = ticket, + } + }, + .place = place, + .evictor = evictor, + .res = res, + .alloc_state = state, + }; + s64 lret; + + state->in_evict = true; + + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + + /* If we failed to find enough BOs to evict, but we skipped over + * some BOs because they were covered by dmem low protection, retry + * evicting these protected BOs too, except if we're told not to + * consider protected BOs at all. + */ + if (!lret && evict_walk.hit_low && state->may_try_low) { + evict_walk.try_low = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } + if (lret || !ticket) + goto out; + + /* Reset low limit */ + evict_walk.try_low = evict_walk.hit_low = false; + /* If ticket-locking, repeat while making progress. */ + evict_walk.walk.arg.trylock_only = false; + +retry: + do { + /* The walk may clear the evict_walk.walk.ticket field */ + evict_walk.walk.arg.ticket = ticket; + evict_walk.evicted = 0; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } while (!lret && evict_walk.evicted); + + /* We hit the low limit? Try once more */ + if (!lret && evict_walk.hit_low && !evict_walk.try_low && + state->may_try_low) { + evict_walk.try_low = true; + goto retry; + } +out: + state->in_evict = false; + if (lret < 0) + return lret; + if (lret == 0) + return -EBUSY; + return 0; +} + +/** + * ttm_bo_evict_cgroup() - Evict buffer objects charged to a specific cgroup. + * @bdev: The TTM device. + * @man: The resource manager whose LRU to walk. + * @limit_pool: The cgroup pool state whose members should be evicted. + * @target_bytes: Number of bytes to free. + * @ctx: The TTM operation context. + * + * Walk the LRU of @man and evict buffer objects that are charged to the + * cgroup identified by @limit_pool, until at least @target_bytes have been + * freed. Mirrors the two-pass (trylock -> sleeping-lock, low-watermark) + * strategy used by ttm_bo_evict_alloc(). + * + * Return: >= @target_bytes on full success, 0..target_bytes-1 if partial, + * negative error code on fatal error. + */ +s64 ttm_bo_evict_cgroup(struct ttm_device *bdev, + struct ttm_resource_manager *man, + struct dmem_cgroup_pool_state *limit_pool, + s64 target_bytes, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { .ctx = ctx }, + }, + .alloc_state = &(struct ttm_bo_alloc_state) { + .limit_pool = limit_pool, + .in_evict = true, + }, + /* place, evictor, res left NULL: selects cgroup drain mode */ + }; + s64 lret, pass; + + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, target_bytes); + if (lret < 0 || lret >= target_bytes) + return lret; + + /* Second pass: also evict BOs at the low watermark. */ + if (evict_walk.hit_low) { + evict_walk.try_low = true; + pass = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, + target_bytes - lret); + if (pass < 0) + return pass; + lret += pass; + if (lret >= target_bytes) + return lret; + } + + /* Full sleeping-lock pass for remaining target. */ + evict_walk.try_low = evict_walk.hit_low = false; + evict_walk.walk.arg.trylock_only = false; + +retry: + evict_walk.walk.arg.sleeping_lock = true; + do { + evict_walk.evicted = 0; + pass = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, + target_bytes - lret); + if (pass < 0) { + lret = pass; + goto out; + } + lret += pass; + } while (lret < target_bytes && evict_walk.evicted); + + /* One more attempt if we hit the low limit during sleeping-lock pass. */ + if (lret < target_bytes && evict_walk.hit_low && !evict_walk.try_low) { + evict_walk.try_low = true; + goto retry; + } + +out: + return lret; +} +EXPORT_SYMBOL(ttm_bo_evict_cgroup); + +/** + * ttm_bo_pin - Pin the buffer object. + * @bo: The buffer object to pin + * + * Make sure the buffer is not evicted any more during memory pressure. + * @bo must be unpinned again by calling ttm_bo_unpin(). + */ +void ttm_bo_pin(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + if (!bo->pin_count++ && bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_pin); + +/** + * ttm_bo_unpin - Unpin the buffer object. + * @bo: The buffer object to unpin + * + * Allows the buffer object to be evicted again during memory pressure. + */ +void ttm_bo_unpin(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + if (WARN_ON_ONCE(!bo->pin_count)) + return; + + spin_lock(&bo->bdev->lru_lock); + if (!--bo->pin_count && bo->resource) { + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + } + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_unpin); + +/* + * Add the pipelined eviction fencesto the BO as kernel dependency and reserve new + * fence slots. + */ +static int ttm_bo_add_pipelined_eviction_fences(struct ttm_buffer_object *bo, + struct ttm_resource_manager *man, + bool no_wait_gpu) +{ + struct dma_fence *fence; + int i; + + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (!fence) + continue; + + if (no_wait_gpu) { + if (!dma_fence_is_signaled(fence)) { + spin_unlock(&man->eviction_lock); + return -EBUSY; + } + } else { + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + } + } + spin_unlock(&man->eviction_lock); + + /* TODO: this call should be removed. */ + return dma_resv_reserve_fences(bo->base.resv, 1); +} + +/** + * ttm_bo_alloc_resource - Allocate backing store for a BO + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @ctx: if and how to sleep, lock buffers and alloc memory + * @force_space: If we should evict buffers to force space + * @res: The resulting struct ttm_resource. + * + * Allocates a resource for the buffer object pointed to by @bo, using the + * placement flags in @placement, potentially evicting other buffer objects when + * @force_space is true. + * This function may sleep while waiting for resources to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == true). + * -ENOSPC: Could not allocate space for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. + */ +static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx, + bool force_space, + struct ttm_resource **res) +{ + struct ttm_device *bdev = bo->bdev; + struct ww_acquire_ctx *ticket; + int i, ret; + + ticket = dma_resv_locking_ctx(bo->base.resv); + ret = dma_resv_reserve_fences(bo->base.resv, TTM_NUM_MOVE_FENCES); + if (unlikely(ret)) + return ret; + + for (i = 0; i < placement->num_placement; ++i) { + const struct ttm_place *place = &placement->placement[i]; + struct ttm_bo_alloc_state alloc_state = {}; + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) + continue; + + if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) + continue; + + ret = ttm_bo_alloc_at_place(bo, place, force_space, res, + &alloc_state); + + if (ret == -ENOSPC) { + dmem_cgroup_uncharge(alloc_state.charge_pool, bo->base.size); + dmem_cgroup_pool_state_put(alloc_state.limit_pool); + continue; + } else if (ret == -EBUSY) { + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, + ticket, res, &alloc_state); + + dmem_cgroup_pool_state_put(alloc_state.limit_pool); + + if (ret) { + dmem_cgroup_uncharge(alloc_state.charge_pool, + bo->base.size); + if (ret == -EBUSY) + continue; + return ret; + } + } else if (ret) { + dmem_cgroup_uncharge(alloc_state.charge_pool, bo->base.size); + dmem_cgroup_pool_state_put(alloc_state.limit_pool); + return ret; + } + + ret = ttm_bo_add_pipelined_eviction_fences(bo, man, ctx->no_wait_gpu); + if (unlikely(ret)) { + ttm_resource_free(bo, res); + if (ret == -EBUSY) + continue; + + return ret; + } + return 0; + } + + return -ENOSPC; +} + +/* + * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @res: The resulting struct ttm_resource. + * @ctx: if and how to sleep, lock buffers and alloc memory + * + * Tries both idle allocation and forcefully eviction of buffers. See + * ttm_bo_alloc_resource for details. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_resource **res, + struct ttm_operation_ctx *ctx) +{ + bool force_space = false; + int ret; + + do { + ret = ttm_bo_alloc_resource(bo, placement, ctx, + force_space, res); + force_space = !force_space; + } while (ret == -ENOSPC && force_space); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_mem_space); + +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @ctx: validation parameters. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ +int ttm_bo_validate(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) +{ + struct ttm_resource *res; + struct ttm_place hop; + bool force_space; + int ret; + + dma_resv_assert_held(bo->base.resv); + + /* + * Remove the backing store if no placement is given. + */ + if (!placement->num_placement) + return ttm_bo_pipeline_gutting(bo); + + force_space = false; + do { + /* Check whether we need to move buffer. */ + if (bo->resource && + ttm_resource_compatible(bo->resource, placement, + force_space)) + return 0; + + /* Moving of pinned BOs is forbidden */ + if (bo->pin_count) + return -EINVAL; + + /* + * Determine where to move the buffer. + * + * If driver determines move is going to need + * an extra step then it will return -EMULTIHOP + * and the buffer will be moved to the temporary + * stop and the driver will be called to make + * the second hop. + */ + ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space, + &res); + force_space = !force_space; + if (ret == -ENOSPC) + continue; + if (ret) + return ret; + +bounce: + ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop); + if (ret == -EMULTIHOP) { + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + /* try and move to final place now. */ + if (!ret) + goto bounce; + } + if (ret) { + ttm_resource_free(bo, &res); + return ret; + } + + } while (ret && force_space); + + /* For backward compatibility with userspace */ + if (ret == -ENOSPC) + return bo->bdev->alloc_flags & TTM_ALLOCATION_PROPAGATE_ENOSPC ? + ret : -ENOMEM; + + /* + * We might need to add a TTM. + */ + if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { + ret = ttm_tt_create(bo, true); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL(ttm_bo_validate); + +/** + * ttm_bo_init_reserved + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @ctx: TTM operation context for memory allocation. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, enables driver-specific objects + * derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * Furthermore, if resv == NULL, the buffer's reservation lock will be held, + * and it is the caller's responsibility to call ttm_bo_unreserve. + * + * If a failure occurs, the function will call the @destroy function. Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, struct ttm_operation_ctx *ctx, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) +{ + int ret; + + kref_init(&bo->kref); + bo->bdev = bdev; + bo->type = type; + bo->page_alignment = alignment; + bo->destroy = destroy; + bo->pin_count = 0; + bo->sg = sg; + bo->bulk_move = NULL; + if (resv) + bo->base.resv = resv; + else + bo->base.resv = &bo->base._resv; + atomic_inc(&ttm_glob.bo_count); + + /* + * For ttm_bo_type_device buffers, allocate + * address space from the device. + */ + if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { + ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, + PFN_UP(bo->base.size)); + if (ret) + goto err_put; + } + + /* passed reservation objects should already be locked, + * since otherwise lockdep will be angered in radeon. + */ + if (!resv) + WARN_ON(!dma_resv_trylock(bo->base.resv)); + else + dma_resv_assert_held(resv); + + ret = ttm_bo_validate(bo, placement, ctx); + if (unlikely(ret)) + goto err_unlock; + + return 0; + +err_unlock: + if (!resv) + dma_resv_unlock(bo->base.resv); + +err_put: + ttm_bo_put(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_init_reserved); + +/** + * ttm_bo_init_validate + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @interruptible: If needing to sleep to wait for GPU resources, + * sleep interruptible. + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, + * enables driver-specific objects derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * + * If a failure occurs, the function will call the @destroy function, Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, bool interruptible, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) +{ + struct ttm_operation_ctx ctx = { .interruptible = interruptible }; + int ret; + + ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, + sg, resv, destroy); + if (ret) + return ret; + + if (!resv) + ttm_bo_unreserve(bo); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_init_validate); + +/* + * buffer object vm functions. + */ + +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ +void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) +{ + struct ttm_device *bdev = bo->bdev; + + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); + ttm_mem_io_free(bdev, bo->resource); +} +EXPORT_SYMBOL(ttm_bo_unmap_virtual); + +/** + * ttm_bo_wait_ctx - wait for buffer idle. + * + * @bo: The buffer object. + * @ctx: defines how to wait + * + * Waits for the buffer to be idle. Used timeout depends on the context. + * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or + * zero on success. + */ +int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) +{ + long ret; + + if (ctx->no_wait_gpu) { + if (dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + return 0; + else + return -EBUSY; + } + + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, 15 * HZ); + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret == 0)) + return -EBUSY; + return 0; +} +EXPORT_SYMBOL(ttm_bo_wait_ctx); + +/** + * struct ttm_bo_swapout_walk - Parameters for the swapout walk + */ +struct ttm_bo_swapout_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ + gfp_t gfp_flags; + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ + bool hit_low, evict_low; +}; + +static s64 +ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) +{ + struct ttm_place place = { .mem_type = bo->resource->mem_type }; + struct ttm_bo_swapout_walk *swapout_walk = + container_of(walk, typeof(*swapout_walk), walk); + struct ttm_operation_ctx *ctx = walk->arg.ctx; + struct ttm_device *bdev = bo->bdev; + struct ttm_tt *tt = bo->ttm; + s64 ret; + + /* + * While the bo may already reside in SYSTEM placement, set + * SYSTEM as new placement to cover also the move further below. + * The driver may use the fact that we're moving from SYSTEM + * as an indication that we're about to swap out. + */ + if (bo->pin_count || !bdev->funcs->eviction_valuable(bo, &place)) { + ret = -EBUSY; + goto out; + } + + if (!tt || !ttm_tt_is_populated(tt) || + tt->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_SWAPPED)) { + ret = -EBUSY; + goto out; + } + + if (bo->deleted) { + pgoff_t num_pages = tt->num_pages; + + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; + + ttm_bo_cleanup_memtype_use(bo); + ret = num_pages; + goto out; + } + + /* + * Move to system cached + */ + if (bo->resource->mem_type != TTM_PL_SYSTEM) { + struct ttm_resource *evict_mem; + struct ttm_place hop; + + memset(&hop, 0, sizeof(hop)); + place.mem_type = TTM_PL_SYSTEM; + ret = ttm_resource_alloc(bo, &place, &evict_mem, NULL); + if (ret) + goto out; + + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret) { + WARN(ret == -EMULTIHOP, + "Unexpected multihop in swapout - likely driver bug.\n"); + ttm_resource_free(bo, &evict_mem); + goto out; + } + } + + /* + * Make sure BO is idle. + */ + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; + + ttm_bo_unmap_virtual(bo); + if (bdev->funcs->swap_notify) + bdev->funcs->swap_notify(bo); + + if (ttm_tt_is_populated(tt)) { + ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags); + if (!ret) { + spin_lock(&bdev->lru_lock); + ttm_resource_del_bulk_move_unevictable(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } + } + +out: + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; + + return ret; +} + +const struct ttm_lru_walk_ops ttm_swap_ops = { + .process_bo = ttm_bo_swapout_cb, +}; + +/** + * ttm_bo_swapout() - Swap out buffer objects on the LRU list to shmem. + * @bdev: The ttm device. + * @ctx: The ttm_operation_ctx governing the swapout operation. + * @man: The resource manager whose resources / buffer objects are + * goint to be swapped out. + * @gfp_flags: The gfp flags used for shmem page allocations. + * @target: The desired number of pages to swap out. + * + * Return: The number of pages actually swapped out, or negative error code + * on error. + */ +s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + struct ttm_resource_manager *man, gfp_t gfp_flags, + s64 target) +{ + struct ttm_bo_swapout_walk swapout_walk = { + .walk = { + .ops = &ttm_swap_ops, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, + }, + .gfp_flags = gfp_flags, + }; + + return ttm_lru_walk_for_evict(&swapout_walk.walk, bdev, man, target); +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_bo_swapout); + +void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) +{ + if (bo->ttm == NULL) + return; + + ttm_tt_unpopulate(bo->bdev, bo->ttm); + ttm_tt_destroy(bo->bdev, bo->ttm); + bo->ttm = NULL; +} + +/** + * ttm_bo_populate() - Ensure that a buffer object has backing pages + * @bo: The buffer object + * @ctx: The ttm_operation_ctx governing the operation. + * + * For buffer objects in a memory type whose manager uses + * struct ttm_tt for backing pages, ensure those backing pages + * are present and with valid content. The bo's resource is also + * placed on the correct LRU list if it was previously swapped + * out. + * + * Return: 0 if successful, negative error code on failure. + * Note: May return -EINTR or -ERESTARTSYS if @ctx::interruptible + * is set to true. + */ +int ttm_bo_populate(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + struct ttm_device *bdev = bo->bdev; + struct ttm_tt *tt = bo->ttm; + bool swapped; + int ret; + + dma_resv_assert_held(bo->base.resv); + + if (!tt) + return 0; + + swapped = ttm_tt_is_swapped(tt); + ret = ttm_tt_populate(bdev, tt, ctx); + if (ret) + return ret; + + if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count && + bo->resource) { + spin_lock(&bdev->lru_lock); + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_populate); + +int ttm_bo_setup_export(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + int ret; + + ret = ttm_bo_reserve(bo, false, false, NULL); + if (ret != 0) + return ret; + + ret = ttm_bo_populate(bo, ctx); + ttm_bo_unreserve(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_setup_export); diff --git a/drivers/gpu/drm/ttm/ttm_bo_internal.h b/drivers/gpu/drm/ttm/ttm_bo_internal.h new file mode 100644 index 000000000..e0d48eac7 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_internal.h @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ + +#ifndef _TTM_BO_INTERNAL_H_ +#define _TTM_BO_INTERNAL_H_ + +#include + +/** + * ttm_bo_get - reference a struct ttm_buffer_object + * + * @bo: The buffer object. + */ +static inline void ttm_bo_get(struct ttm_buffer_object *bo) +{ + kref_get(&bo->kref); +} + +/** + * ttm_bo_get_unless_zero - reference a struct ttm_buffer_object unless + * its refcount has already reached zero. + * @bo: The buffer object. + * + * Used to reference a TTM buffer object in lookups where the object is removed + * from the lookup structure during the destructor and for RCU lookups. + * + * Returns: @bo if the referencing was successful, NULL otherwise. + */ +static inline __must_check struct ttm_buffer_object * +ttm_bo_get_unless_zero(struct ttm_buffer_object *bo) +{ + if (!kref_get_unless_zero(&bo->kref)) + return NULL; + return bo; +} + +void ttm_bo_put(struct ttm_buffer_object *bo); + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c new file mode 100644 index 000000000..bd0b23ac2 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -0,0 +1,1167 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include +#include +#include + +#include +#include +#include + +#include + +#include "ttm_bo_internal.h" + +struct ttm_transfer_obj { + struct ttm_buffer_object base; + struct ttm_buffer_object *bo; +}; + +int ttm_mem_io_reserve(struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (mem->bus.offset || mem->bus.addr) + return 0; + + mem->bus.is_iomem = false; + if (!bdev->funcs->io_mem_reserve) + return 0; + + return bdev->funcs->io_mem_reserve(bdev, mem); +} + +void ttm_mem_io_free(struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (!mem) + return; + + if (!mem->bus.offset && !mem->bus.addr) + return; + + if (bdev->funcs->io_mem_free) + bdev->funcs->io_mem_free(bdev, mem); + + mem->bus.offset = 0; + mem->bus.addr = NULL; +} + +/** + * ttm_move_memcpy - Helper to perform a memcpy ttm move operation. + * @clear: Whether to clear rather than copy. + * @num_pages: Number of pages of the operation. + * @dst_iter: A struct ttm_kmap_iter representing the destination resource. + * @src_iter: A struct ttm_kmap_iter representing the source resource. + * + * This function is intended to be able to move out async under a + * dma-fence if desired. + */ +void ttm_move_memcpy(bool clear, + u32 num_pages, + struct ttm_kmap_iter *dst_iter, + struct ttm_kmap_iter *src_iter) +{ + const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops; + const struct ttm_kmap_iter_ops *src_ops = src_iter->ops; + struct iosys_map src_map, dst_map; + pgoff_t i; + + /* Single TTM move. NOP */ + if (dst_ops->maps_tt && src_ops->maps_tt) + return; + + /* Don't move nonexistent data. Clear destination instead. */ + if (clear) { + for (i = 0; i < num_pages; ++i) { + dst_ops->map_local(dst_iter, &dst_map, i); + if (dst_map.is_iomem) + memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE); + else + memset(dst_map.vaddr, 0, PAGE_SIZE); + if (dst_ops->unmap_local) + dst_ops->unmap_local(dst_iter, &dst_map); + } + return; + } + + for (i = 0; i < num_pages; ++i) { + dst_ops->map_local(dst_iter, &dst_map, i); + src_ops->map_local(src_iter, &src_map, i); + + drm_memcpy_from_wc(&dst_map, &src_map, PAGE_SIZE); + + if (src_ops->unmap_local) + src_ops->unmap_local(src_iter, &src_map); + if (dst_ops->unmap_local) + dst_ops->unmap_local(dst_iter, &dst_map); + } +} +EXPORT_SYMBOL(ttm_move_memcpy); + +/** + * ttm_bo_move_memcpy + * + * @bo: A pointer to a struct ttm_buffer_object. + * @ctx: operation context + * @dst_mem: struct ttm_resource indicating where to move. + * + * Fallback move function for a mappable buffer object in mappable memory. + * The function will, if successful, + * free any old aperture space, and set (@new_mem)->mm_node to NULL, + * and update the (@bo)->mem placement flags. If unsuccessful, the old + * data remains untouched, and it's up to the caller to free the + * memory space indicated by @new_mem. + * Returns: + * !0: Failure. + */ +int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_resource *dst_mem) +{ + struct ttm_device *bdev = bo->bdev; + struct ttm_resource_manager *dst_man = + ttm_manager_type(bo->bdev, dst_mem->mem_type); + struct ttm_tt *ttm = bo->ttm; + struct ttm_resource *src_mem = bo->resource; + struct ttm_resource_manager *src_man; + union { + struct ttm_kmap_iter_tt tt; + struct ttm_kmap_iter_linear_io io; + } _dst_iter, _src_iter; + struct ttm_kmap_iter *dst_iter, *src_iter; + bool clear; + int ret = 0; + + if (WARN_ON(!src_mem)) + return -EINVAL; + + src_man = ttm_manager_type(bdev, src_mem->mem_type); + if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) || + dst_man->use_tt)) { + ret = ttm_bo_populate(bo, ctx); + if (ret) + return ret; + } + + dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem); + if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt) + dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, ttm); + if (IS_ERR(dst_iter)) + return PTR_ERR(dst_iter); + + src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem); + if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt) + src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, ttm); + if (IS_ERR(src_iter)) { + ret = PTR_ERR(src_iter); + goto out_src_iter; + } + + clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm)); + if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) + ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter); + + if (!src_iter->ops->maps_tt) + ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, src_mem); + ttm_bo_move_sync_cleanup(bo, dst_mem); + +out_src_iter: + if (!dst_iter->ops->maps_tt) + ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_move_memcpy); + +static void ttm_transfered_destroy(struct ttm_buffer_object *bo) +{ + struct ttm_transfer_obj *fbo; + + fbo = container_of(bo, struct ttm_transfer_obj, base); + dma_resv_fini(&fbo->base.base._resv); + ttm_bo_put(fbo->bo); + kfree(fbo); +} + +/** + * ttm_buffer_object_transfer + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, + * holding the data of @bo with the old placement. + * + * This is a utility function that may be called after an accelerated move + * has been scheduled. A new buffer object is created as a placeholder for + * the old data while it's being copied. When that buffer object is idle, + * it can be destroyed, releasing the space of the old placement. + * Returns: + * !0: Failure. + */ + +static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + struct ttm_buffer_object **new_obj) +{ + struct ttm_transfer_obj *fbo; + int ret; + + fbo = kmalloc_obj(*fbo); + if (!fbo) + return -ENOMEM; + + fbo->base = *bo; + + /** + * Fix up members that we shouldn't copy directly: + * TODO: Explicit member copy would probably be better here. + */ + + atomic_inc(&ttm_glob.bo_count); + drm_vma_node_reset(&fbo->base.base.vma_node); + + kref_init(&fbo->base.kref); + fbo->base.destroy = &ttm_transfered_destroy; + fbo->base.pin_count = 0; + if (bo->type != ttm_bo_type_sg) + fbo->base.base.resv = &fbo->base.base._resv; + + dma_resv_init(&fbo->base.base._resv); + fbo->base.base.dev = NULL; + ret = dma_resv_trylock(&fbo->base.base._resv); + WARN_ON(!ret); + + ret = dma_resv_reserve_fences(&fbo->base.base._resv, TTM_NUM_MOVE_FENCES); + if (ret) { + dma_resv_unlock(&fbo->base.base._resv); + kfree(fbo); + return ret; + } + + if (fbo->base.resource) { + ttm_resource_set_bo(fbo->base.resource, &fbo->base); + bo->resource = NULL; + ttm_bo_set_bulk_move(&fbo->base, NULL); + } else { + fbo->base.bulk_move = NULL; + } + + ttm_bo_get(bo); + fbo->bo = bo; + + ttm_bo_move_to_lru_tail_unlocked(&fbo->base); + + *new_obj = &fbo->base; + return 0; +} + +/** + * ttm_io_prot + * + * @bo: ttm buffer object + * @res: ttm resource object + * @tmp: Page protection flag for a normal, cached mapping. + * + * Utility function that returns the pgprot_t that should be used for + * setting up a PTE with the caching model indicated by @c_state. + */ +pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, + pgprot_t tmp) +{ + struct ttm_resource_manager *man; + enum ttm_caching caching; + + man = ttm_manager_type(bo->bdev, res->mem_type); + if (man->use_tt) { + caching = bo->ttm->caching; + if (bo->ttm->page_flags & TTM_TT_FLAG_DECRYPTED) + tmp = pgprot_decrypted(tmp); + } else { + caching = res->bus.caching; + } + + return ttm_prot_from_caching(caching, tmp); +} +EXPORT_SYMBOL(ttm_io_prot); + +static int ttm_bo_ioremap(struct ttm_buffer_object *bo, + unsigned long offset, + unsigned long size, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *mem = bo->resource; + + if (mem->bus.addr) { + map->bo_kmap_type = ttm_bo_map_premapped; + map->virtual = ((u8 *)mem->bus.addr) + offset; + } else { + resource_size_t res = mem->bus.offset + offset; + + map->bo_kmap_type = ttm_bo_map_iomap; + if (mem->bus.caching == ttm_write_combined) + map->virtual = ioremap_wc(res, size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + map->virtual = ioremap_cache(res, size); +#endif + else + map->virtual = ioremap(res, size); + } + return (!map->virtual) ? -ENOMEM : 0; +} + +static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, + unsigned long start_page, + unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *mem = bo->resource; + struct ttm_operation_ctx ctx = { }; + struct ttm_tt *ttm = bo->ttm; + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, mem->mem_type); + pgprot_t prot; + int ret; + + BUG_ON(!ttm); + + ret = ttm_bo_populate(bo, &ctx); + if (ret) + return ret; + + if (num_pages == 1 && ttm->caching == ttm_cached && + !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) { + /* + * We're mapping a single page, and the desired + * page protection is consistent with the bo. + */ + + map->bo_kmap_type = ttm_bo_map_kmap; + map->page = ttm->pages[start_page]; + map->virtual = kmap(map->page); + } else { + /* + * We need to use vmap to get the desired page protection + * or to make the buffer object look contiguous. + */ + prot = ttm_io_prot(bo, mem, PAGE_KERNEL); + map->bo_kmap_type = ttm_bo_map_vmap; + map->virtual = vmap(ttm->pages + start_page, num_pages, + 0, prot); + } + return (!map->virtual) ? -ENOMEM : 0; +} + +/** + * ttm_bo_kmap_try_from_panic + * + * @bo: The buffer object + * @page: The page to map + * + * Sets up a kernel virtual mapping using kmap_local_page_try_from_panic(). + * This should only be called from the panic handler, if you make sure the bo + * is the one being displayed, so is properly allocated, and protected. + * + * Returns the vaddr, that you can use to write to the bo, and that you should + * pass to kunmap_local() when you're done with this page, or NULL if the bo + * is in iomem. + */ +void *ttm_bo_kmap_try_from_panic(struct ttm_buffer_object *bo, unsigned long page) +{ + if (page + 1 > PFN_UP(bo->resource->size)) + return NULL; + + if (!bo->resource->bus.is_iomem && bo->ttm->pages && bo->ttm->pages[page]) + return kmap_local_page_try_from_panic(bo->ttm->pages[page]); + + return NULL; +} +EXPORT_SYMBOL(ttm_bo_kmap_try_from_panic); + +/** + * ttm_bo_kmap + * + * @bo: The buffer object. + * @start_page: The first page to map. + * @num_pages: Number of pages to map. + * @map: pointer to a struct ttm_bo_kmap_obj representing the map. + * + * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the + * data in the buffer object. The ttm_kmap_obj_virtual function can then be + * used to obtain a virtual address to the data. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ +int ttm_bo_kmap(struct ttm_buffer_object *bo, + unsigned long start_page, unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *res = bo->resource; + unsigned long offset, size; + int ret; + + map->virtual = NULL; + map->bo = bo; + if (num_pages > PFN_UP(res->size)) + return -EINVAL; + if ((start_page + num_pages) > PFN_UP(res->size)) + return -EINVAL; + + ret = ttm_mem_io_reserve(bo->bdev, res); + if (ret) + return ret; + if (!res->bus.is_iomem) { + return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); + } else { + offset = start_page << PAGE_SHIFT; + size = num_pages << PAGE_SHIFT; + return ttm_bo_ioremap(bo, offset, size, map); + } +} +EXPORT_SYMBOL(ttm_bo_kmap); + +/** + * ttm_bo_kunmap + * + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_kmap. + */ +void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) +{ + if (!map->virtual) + return; + switch (map->bo_kmap_type) { + case ttm_bo_map_iomap: + iounmap(map->virtual); + break; + case ttm_bo_map_vmap: + vunmap(map->virtual); + break; + case ttm_bo_map_kmap: + kunmap(map->page); + break; + case ttm_bo_map_premapped: + break; + default: + BUG(); + } + ttm_mem_io_free(map->bo->bdev, map->bo->resource); + map->virtual = NULL; + map->page = NULL; +} +EXPORT_SYMBOL(ttm_bo_kunmap); + +/** + * ttm_bo_vmap + * + * @bo: The buffer object. + * @map: pointer to a struct iosys_map representing the map. + * + * Sets up a kernel virtual mapping, using ioremap or vmap to the + * data in the buffer object. The parameter @map returns the virtual + * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap(). + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ +int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) +{ + struct ttm_resource *mem = bo->resource; + int ret; + + dma_resv_assert_held(bo->base.resv); + + ret = ttm_mem_io_reserve(bo->bdev, mem); + if (ret) + return ret; + + if (mem->bus.is_iomem) { + void __iomem *vaddr_iomem; + + if (mem->bus.addr) + vaddr_iomem = (void __iomem *)mem->bus.addr; + else if (mem->bus.caching == ttm_write_combined) + vaddr_iomem = ioremap_wc(mem->bus.offset, + bo->base.size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + vaddr_iomem = ioremap_cache(mem->bus.offset, + bo->base.size); +#endif + else + vaddr_iomem = ioremap(mem->bus.offset, bo->base.size); + + if (!vaddr_iomem) + return -ENOMEM; + + iosys_map_set_vaddr_iomem(map, vaddr_iomem); + + } else { + struct ttm_operation_ctx ctx = { }; + struct ttm_tt *ttm = bo->ttm; + pgprot_t prot; + void *vaddr; + + ret = ttm_bo_populate(bo, &ctx); + if (ret) + return ret; + + /* + * We need to use vmap to get the desired page protection + * or to make the buffer object look contiguous. + */ + prot = ttm_io_prot(bo, mem, PAGE_KERNEL); + vaddr = vmap(ttm->pages, ttm->num_pages, 0, prot); + if (!vaddr) + return -ENOMEM; + + iosys_map_set_vaddr(map, vaddr); + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_vmap); + +/** + * ttm_bo_vunmap + * + * @bo: The buffer object. + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_vmap(). + */ +void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map) +{ + struct ttm_resource *mem = bo->resource; + + dma_resv_assert_held(bo->base.resv); + + if (iosys_map_is_null(map)) + return; + + if (!map->is_iomem) + vunmap(map->vaddr); + else if (!mem->bus.addr) + iounmap(map->vaddr_iomem); + iosys_map_clear(map); + + ttm_mem_io_free(bo->bdev, mem); +} +EXPORT_SYMBOL(ttm_bo_vunmap); + +static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, + bool dst_use_tt) +{ + long ret; + + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, 15 * HZ); + if (ret == 0) + return -EBUSY; + if (ret < 0) + return ret; + + if (!dst_use_tt) + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->resource); + return 0; +} + +static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, + struct dma_fence *fence, + bool dst_use_tt) +{ + struct ttm_buffer_object *ghost_obj; + int ret; + + /** + * This should help pipeline ordinary buffer moves. + * + * Hang old buffer memory on a new buffer object, + * and leave it to be released when the GPU + * operation has completed. + */ + + ret = ttm_buffer_object_transfer(bo, &ghost_obj); + if (ret) + return ret; + + dma_resv_add_fence(&ghost_obj->base._resv, fence, + DMA_RESV_USAGE_KERNEL); + + /** + * If we're not moving to fixed memory, the TTM object + * needs to stay alive. Otherwhise hang it on the ghost + * bo to be unbound and destroyed. + */ + + if (dst_use_tt) + ghost_obj->ttm = NULL; + else + bo->ttm = NULL; + + dma_resv_unlock(&ghost_obj->base._resv); + ttm_bo_put(ghost_obj); + return 0; +} + +static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, + struct dma_fence *fence) +{ + struct ttm_resource_manager *from; + struct dma_fence *tmp; + int i; + + from = ttm_manager_type(bo->bdev, bo->resource->mem_type); + + /** + * BO doesn't have a TTM we need to bind/unbind. Just remember + * this eviction and free up the allocation. + * The fence will be saved in the first free slot or in the slot + * already used to store a fence from the same context. Since + * drivers can't use more than TTM_NUM_MOVE_FENCES contexts for + * evictions we should always find a slot to use. + */ + spin_lock(&from->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + tmp = from->eviction_fences[i]; + if (!tmp) + break; + if (fence->context != tmp->context) + continue; + if (dma_fence_is_later(fence, tmp)) { + dma_fence_put(tmp); + break; + } + goto unlock; + } + if (i < TTM_NUM_MOVE_FENCES) { + from->eviction_fences[i] = dma_fence_get(fence); + } else { + WARN(1, "not enough fence slots for all fence contexts"); + spin_unlock(&from->eviction_lock); + dma_fence_wait(fence, false); + goto end; + } + +unlock: + spin_unlock(&from->eviction_lock); +end: + ttm_resource_free(bo, &bo->resource); +} + +/** + * ttm_bo_move_accel_cleanup - cleanup helper for hw copies + * + * @bo: A pointer to a struct ttm_buffer_object. + * @fence: A fence object that signals when moving is complete. + * @evict: This is an evict move. Don't return until the buffer is idle. + * @pipeline: evictions are to be pipelined. + * @new_mem: struct ttm_resource indicating where to move. + * + * Accelerated move function to be called when an accelerated move + * has been scheduled. The function will create a new temporary buffer object + * representing the old placement, and put the sync object on both buffer + * objects. After that the newly created buffer object is unref'd to be + * destroyed when the move is complete. This will help pipeline + * buffer moves. + */ +int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, + struct dma_fence *fence, + bool evict, + bool pipeline, + struct ttm_resource *new_mem) +{ + struct ttm_device *bdev = bo->bdev; + struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->resource->mem_type); + struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); + int ret = 0; + + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + if (!evict) + ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt); + else if (!from->use_tt && pipeline) + ttm_bo_move_pipeline_evict(bo, fence); + else + ret = ttm_bo_wait_free_node(bo, man->use_tt); + + if (ret) + return ret; + + ttm_bo_assign_mem(bo, new_mem); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); + +/** + * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_mem: struct ttm_resource indicating where to move. + * + * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed + * by the caller to be idle. Typically used after memcpy buffer moves. + */ +void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo, + struct ttm_resource *new_mem) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, new_mem->mem_type); + int ret; + + ret = ttm_bo_wait_free_node(bo, man->use_tt); + if (WARN_ON(ret)) + return; + + ttm_bo_assign_mem(bo, new_mem); +} +EXPORT_SYMBOL(ttm_bo_move_sync_cleanup); + +/** + * ttm_bo_pipeline_gutting - purge the contents of a bo + * @bo: The buffer object + * + * Purge the contents of a bo, async if the bo is not idle. + * After a successful call, the bo is left unpopulated in + * system placement. The function may wait uninterruptible + * for idle on OOM. + * + * Return: 0 if successful, negative error code on failure. + */ +int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) +{ + struct ttm_buffer_object *ghost; + struct ttm_tt *ttm; + int ret; + + /* If already idle, no need for ghost object dance. */ + if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) { + if (!bo->ttm) { + /* See comment below about clearing. */ + ret = ttm_tt_create(bo, true); + if (ret) + return ret; + } else { + ttm_tt_unpopulate(bo->bdev, bo->ttm); + if (bo->type == ttm_bo_type_device) + ttm_tt_mark_for_clear(bo->ttm); + } + ttm_resource_free(bo, &bo->resource); + return 0; + } + + /* + * We need an unpopulated ttm_tt after giving our current one, + * if any, to the ghost object. And we can't afford to fail + * creating one *after* the operation. If the bo subsequently gets + * resurrected, make sure it's cleared (if ttm_bo_type_device) + * to avoid leaking sensitive information to user-space. + */ + + ttm = bo->ttm; + bo->ttm = NULL; + ret = ttm_tt_create(bo, true); + swap(bo->ttm, ttm); + if (ret) + return ret; + + ret = ttm_buffer_object_transfer(bo, &ghost); + if (ret) + goto error_destroy_tt; + + ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); + /* Last resort, wait for the BO to be idle when we are OOM */ + if (ret) { + dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + } + + dma_resv_unlock(&ghost->base._resv); + ttm_bo_put(ghost); + bo->ttm = ttm; + return 0; + +error_destroy_tt: + ttm_tt_destroy(bo->bdev, ttm); + return ret; +} + +static bool ttm_lru_walk_trylock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) +{ + struct ttm_operation_ctx *ctx = curs->arg->ctx; + + curs->needs_unlock = false; + + if (dma_resv_trylock(bo->base.resv)) { + curs->needs_unlock = true; + return true; + } + + if (bo->base.resv == ctx->resv && ctx->allow_res_evict) { + dma_resv_assert_held(bo->base.resv); + return true; + } + + return false; +} + +static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) +{ + struct ttm_lru_walk_arg *arg = curs->arg; + int ret; + + if (arg->ctx->interruptible) + ret = dma_resv_lock_interruptible(bo->base.resv, arg->ticket); + else + ret = dma_resv_lock(bo->base.resv, arg->ticket); + + if (!ret) { + curs->needs_unlock = true; + /* + * Only a single ticketlock per loop. Ticketlocks are prone + * to return -EDEADLK causing the eviction to fail, so + * after waiting for the ticketlock, revert back to + * trylocking for this walk. + */ + arg->ticket = NULL; + } else if (ret == -EDEADLK) { + /* Caller needs to exit the ww transaction. */ + ret = -ENOSPC; + } + + return ret; +} + +/** + * ttm_lru_walk_for_evict() - Perform a LRU list walk, with actions taken on + * valid items. + * @walk: describe the walks and actions taken + * @bdev: The TTM device. + * @man: The struct ttm_resource manager whose LRU lists we're walking. + * @target: The end condition for the walk. + * + * The LRU lists of @man are walk, and for each struct ttm_resource encountered, + * the corresponding ttm_buffer_object is locked and taken a reference on, and + * the LRU lock is dropped. the LRU lock may be dropped before locking and, in + * that case, it's verified that the item actually remains on the LRU list after + * the lock, and that the buffer object didn't switch resource in between. + * + * With a locked object, the actions indicated by @walk->process_bo are + * performed, and after that, the bo is unlocked, the refcount dropped and the + * next struct ttm_resource is processed. Here, the walker relies on + * TTM's restartable LRU list implementation. + * + * Typically @walk->process_bo() would return the number of pages evicted, + * swapped or shrunken, so that when the total exceeds @target, or when the + * LRU list has been walked in full, iteration is terminated. It's also terminated + * on error. Note that the definition of @target is done by the caller, it + * could have a different meaning than the number of pages. + * + * Note that the way dma_resv individualization is done, locking needs to be done + * either with the LRU lock held (trylocking only) or with a reference on the + * object. + * + * Return: The progress made towards target or negative error code on error. + */ +s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev, + struct ttm_resource_manager *man, s64 target) +{ + struct ttm_bo_lru_cursor cursor; + struct ttm_buffer_object *bo; + s64 progress = 0; + s64 lret; + + ttm_bo_lru_for_each_reserved_guarded(&cursor, man, &walk->arg, bo) { + lret = walk->ops->process_bo(walk, bo); + if (lret == -EBUSY || lret == -EALREADY) + lret = 0; + progress = (lret < 0) ? lret : progress + lret; + if (progress < 0 || progress >= target) + break; + } + if (IS_ERR(bo)) + return PTR_ERR(bo); + + return progress; +} +EXPORT_SYMBOL(ttm_lru_walk_for_evict); + +static void ttm_bo_lru_cursor_cleanup_bo(struct ttm_bo_lru_cursor *curs) +{ + struct ttm_buffer_object *bo = curs->bo; + + if (bo) { + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); + curs->bo = NULL; + } +} + +/** + * ttm_bo_lru_cursor_fini() - Stop using a struct ttm_bo_lru_cursor + * and clean up any iteration it was used for. + * @curs: The cursor. + */ +void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs) +{ + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + + ttm_bo_lru_cursor_cleanup_bo(curs); + spin_lock(lru_lock); + ttm_resource_cursor_fini(&curs->res_curs); + spin_unlock(lru_lock); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_fini); + +/** + * ttm_bo_lru_cursor_init() - Initialize a struct ttm_bo_lru_cursor + * @curs: The ttm_bo_lru_cursor to initialize. + * @man: The ttm resource_manager whose LRU lists to iterate over. + * @arg: The ttm_lru_walk_arg to govern the walk. + * + * Initialize a struct ttm_bo_lru_cursor. + * + * Return: Pointer to @curs. The function does not fail. + */ +struct ttm_bo_lru_cursor * +ttm_bo_lru_cursor_init(struct ttm_bo_lru_cursor *curs, + struct ttm_resource_manager *man, + struct ttm_lru_walk_arg *arg) +{ + memset(curs, 0, sizeof(*curs)); + ttm_resource_cursor_init(&curs->res_curs, man); + curs->arg = arg; + + return curs; +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_init); + +static struct ttm_buffer_object * +__ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) +{ + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + struct ttm_resource *res = NULL; + struct ttm_buffer_object *bo; + struct ttm_lru_walk_arg *arg = curs->arg; + bool first = !curs->bo; + + ttm_bo_lru_cursor_cleanup_bo(curs); + + spin_lock(lru_lock); + for (;;) { + int mem_type, ret = 0; + bool bo_locked = false; + + if (first) { + res = ttm_resource_manager_first(&curs->res_curs); + first = false; + } else { + res = ttm_resource_manager_next(&curs->res_curs); + } + if (!res) + break; + + bo = res->bo; + if (ttm_lru_walk_trylock(curs, bo)) + bo_locked = true; + else if ((!arg->ticket && !arg->sleeping_lock) || arg->ctx->no_wait_gpu || + arg->trylock_only) + continue; + + if (!ttm_bo_get_unless_zero(bo)) { + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); + continue; + } + + mem_type = res->mem_type; + spin_unlock(lru_lock); + if (!bo_locked) + ret = ttm_lru_walk_ticketlock(curs, bo); + + /* + * Note that in between the release of the lru lock and the + * ticketlock, the bo may have switched resource, + * and also memory type, since the resource may have been + * freed and allocated again with a different memory type. + * In that case, just skip it. + */ + curs->bo = bo; + if (!ret && bo->resource && bo->resource->mem_type == mem_type) + return bo; + + ttm_bo_lru_cursor_cleanup_bo(curs); + if (ret && ret != -EALREADY) + return ERR_PTR(ret); + + spin_lock(lru_lock); + } + + spin_unlock(lru_lock); + return res ? bo : NULL; +} + +/** + * ttm_bo_lru_cursor_next() - Continue iterating a manager's LRU lists + * to find and lock buffer object. + * @curs: The cursor initialized using ttm_bo_lru_cursor_init() and + * ttm_bo_lru_cursor_first(). + * + * Return: A pointer to a locked and reference-counted buffer object, + * or NULL if none could be found and looping should be terminated. + */ +struct ttm_buffer_object *ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) +{ + return __ttm_bo_lru_cursor_next(curs); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_next); + +/** + * ttm_bo_lru_cursor_first() - Start iterating a manager's LRU lists + * to find and lock buffer object. + * @curs: The cursor initialized using ttm_bo_lru_cursor_init(). + * + * Return: A pointer to a locked and reference-counted buffer object, + * or NULL if none could be found and looping should be terminated. + */ +struct ttm_buffer_object *ttm_bo_lru_cursor_first(struct ttm_bo_lru_cursor *curs) +{ + ttm_bo_lru_cursor_cleanup_bo(curs); + return __ttm_bo_lru_cursor_next(curs); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_first); + +/** + * ttm_bo_shrink() - Helper to shrink a ttm buffer object. + * @ctx: The struct ttm_operation_ctx used for the shrinking operation. + * @bo: The buffer object. + * @flags: Flags governing the shrinking behaviour. + * + * The function uses the ttm_tt_back_up functionality to back up or + * purge a struct ttm_tt. If the bo is not in system, it's first + * moved there. + * + * Return: The number of pages shrunken or purged, or + * negative error code on failure. + */ +long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, + const struct ttm_bo_shrink_flags flags) +{ + static const struct ttm_place sys_placement_flags = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_SYSTEM, + .flags = 0, + }; + static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_placement_flags, + }; + struct ttm_device *bdev = bo->bdev; + long lret; + + dma_resv_assert_held(bo->base.resv); + + if (flags.allow_move && bo->resource->mem_type != TTM_PL_SYSTEM) { + int ret = ttm_bo_validate(bo, &sys_placement, ctx); + + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret) { + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; + return ret; + } + } + + ttm_bo_unmap_virtual(bo); + lret = ttm_bo_wait_ctx(bo, ctx); + if (lret < 0) + return lret; + + lret = ttm_tt_backup(bdev, bo->ttm, (struct ttm_backup_flags) + {.purge = flags.purge, + .writeback = flags.writeback}); + + if (lret > 0) { + spin_lock(&bdev->lru_lock); + ttm_resource_del_bulk_move_unevictable(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } + + if (lret < 0 && lret != -EINTR) + return -EBUSY; + + return lret; +} +EXPORT_SYMBOL(ttm_bo_shrink); + +/** + * ttm_bo_shrink_suitable() - Whether a bo is suitable for shinking + * @ctx: The struct ttm_operation_ctx governing the shrinking. + * @bo: The candidate for shrinking. + * + * Check whether the object, given the information available to TTM, + * is suitable for shinking, This function can and should be used + * before attempting to shrink an object. + * + * Return: true if suitable. false if not. + */ +bool ttm_bo_shrink_suitable(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) +{ + return bo->ttm && ttm_tt_is_populated(bo->ttm) && !bo->pin_count && + (!ctx->no_wait_gpu || + dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)); +} +EXPORT_SYMBOL(ttm_bo_shrink_suitable); + +/** + * ttm_bo_shrink_avoid_wait() - Whether to avoid waiting for GPU + * during shrinking + * + * In some situations, like direct reclaim, waiting (in particular gpu waiting) + * should be avoided since it may stall a system that could otherwise make progress + * shrinking something else less time consuming. + * + * Return: true if gpu waiting should be avoided, false if not. + */ +bool ttm_bo_shrink_avoid_wait(void) +{ + return !current_is_kswapd(); +} +EXPORT_SYMBOL(ttm_bo_shrink_avoid_wait); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c new file mode 100644 index 000000000..872bf444b --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -0,0 +1,514 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include + +#include +#include +#include + +#include +#include + +static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + struct vm_fault *vmf) +{ + long err = 0; + + /* + * Quick non-stalling check for idle. + */ + if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_KERNEL)) + return 0; + + /* + * If possible, avoid waiting for GPU with mmap_lock + * held. We only do this if the fault allows retry and this + * is the first attempt. + */ + if (fault_flag_allow_retry_first(vmf->flags)) { + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + return VM_FAULT_RETRY; + + drm_gem_object_get(&bo->base); + mmap_read_unlock(vmf->vma->vm_mm); + (void)dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_KERNEL, true, + MAX_SCHEDULE_TIMEOUT); + dma_resv_unlock(bo->base.resv); + drm_gem_object_put(&bo->base); + return VM_FAULT_RETRY; + } + + /* + * Ordinary wait. + */ + err = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_KERNEL, true, + MAX_SCHEDULE_TIMEOUT); + if (unlikely(err < 0)) { + return (err != -ERESTARTSYS) ? VM_FAULT_SIGBUS : + VM_FAULT_NOPAGE; + } + + return 0; +} + +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_device *bdev = bo->bdev; + + if (bdev->funcs->io_mem_pfn) + return bdev->funcs->io_mem_pfn(bo, page_offset); + + return (bo->resource->bus.offset >> PAGE_SHIFT) + page_offset; +} + +/** + * ttm_bo_vm_reserve - Reserve a buffer object in a retryable vm callback + * @bo: The buffer object + * @vmf: The fault structure handed to the callback + * + * vm callbacks like fault() and *_mkwrite() allow for the mmap_lock to be dropped + * during long waits, and after the wait the callback will be restarted. This + * is to allow other threads using the same virtual memory space concurrent + * access to map(), unmap() completely unrelated buffer objects. TTM buffer + * object reservations sometimes wait for GPU and should therefore be + * considered long waits. This function reserves the buffer object interruptibly + * taking this into account. Starvation is avoided by the vm system not + * allowing too many repeated restarts. + * This function is intended to be used in customized fault() and _mkwrite() + * handlers. + * + * Return: + * 0 on success and the bo was reserved. + * VM_FAULT_RETRY if blocking wait. + * VM_FAULT_NOPAGE if blocking wait and retrying was not allowed. + */ +vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, + struct vm_fault *vmf) +{ + /* + * Work around locking order reversal in fault / nopfn + * between mmap_lock and bo_reserve: Perform a trylock operation + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. + */ + if (unlikely(!dma_resv_trylock(bo->base.resv))) { + /* + * If the fault allows retry and this is the first + * fault attempt, we try to release the mmap_lock + * before waiting + */ + if (fault_flag_allow_retry_first(vmf->flags)) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + drm_gem_object_get(&bo->base); + mmap_read_unlock(vmf->vma->vm_mm); + if (!dma_resv_lock_interruptible(bo->base.resv, + NULL)) + dma_resv_unlock(bo->base.resv); + drm_gem_object_put(&bo->base); + } + + return VM_FAULT_RETRY; + } + + if (dma_resv_lock_interruptible(bo->base.resv, NULL)) + return VM_FAULT_NOPAGE; + } + + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + if (!(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE)) { + dma_resv_unlock(bo->base.resv); + return VM_FAULT_SIGBUS; + } + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_vm_reserve); + +/** + * ttm_bo_vm_fault_reserved - TTM fault helper + * @vmf: The struct vm_fault given as argument to the fault callback + * @prot: The page protection to be used for this memory area. + * @num_prefault: Maximum number of prefault pages. The caller may want to + * specify this based on madvice settings and the size of the GPU object + * backed by the memory. + * + * This function inserts one or more page table entries pointing to the + * memory backing the buffer object, and then returns a return code + * instructing the caller to retry the page access. + * + * Return: + * VM_FAULT_NOPAGE on success or pending signal + * VM_FAULT_SIGBUS on unspecified error + * VM_FAULT_OOM on out-of-memory + * VM_FAULT_RETRY if retryable wait + */ +vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, + pgprot_t prot, + pgoff_t num_prefault) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct page *page; + int err; + pgoff_t i; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address = vmf->address; + + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + ret = ttm_bo_vm_fault_idle(bo, vmf); + if (unlikely(ret != 0)) + return ret; + + err = ttm_mem_io_reserve(bo->bdev, bo->resource); + if (unlikely(err != 0)) + return VM_FAULT_SIGBUS; + + page_offset = linear_page_index(vma, address) - + drm_vma_node_start(&bo->base.vma_node); + page_last = vma_end_pgoff(vma) - + drm_vma_node_start(&bo->base.vma_node); + + if (unlikely(page_offset >= PFN_UP(bo->base.size))) + return VM_FAULT_SIGBUS; + + prot = ttm_io_prot(bo, bo->resource, prot); + if (!bo->resource->bus.is_iomem) { + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + }; + + ttm = bo->ttm; + err = ttm_bo_populate(bo, &ctx); + if (err) { + if (err == -EINTR || err == -ERESTARTSYS || + err == -EAGAIN) + return VM_FAULT_NOPAGE; + + pr_debug("TTM fault hit %pe.\n", ERR_PTR(err)); + return VM_FAULT_SIGBUS; + } + } else { + /* Iomem should not be marked encrypted */ + prot = pgprot_decrypted(prot); + } + + /* + * Speculatively prefault a number of pages. Only error on + * first page. + */ + for (i = 0; i < num_prefault; ++i) { + if (bo->resource->bus.is_iomem) { + pfn = ttm_bo_io_mem_pfn(bo, page_offset); + } else { + page = ttm->pages[page_offset]; + if (unlikely(!page && i == 0)) { + return VM_FAULT_OOM; + } else if (unlikely(!page)) { + break; + } + pfn = page_to_pfn(page); + } + + /* + * Note that the value of @prot at this point may differ from + * the value of @vma->vm_page_prot in the caching- and + * encryption bits. This is because the exact location of the + * data may not be known at mmap() time and may also change + * at arbitrary times while the data is mmap'ed. + * See vmf_insert_pfn_prot() for a discussion. + */ + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + + /* Never error on prefaulted PTEs */ + if (unlikely((ret & VM_FAULT_ERROR))) { + if (i == 0) + return VM_FAULT_NOPAGE; + else + break; + } + + address += PAGE_SIZE; + if (unlikely(++page_offset >= page_last)) + break; + } + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); + +static void ttm_bo_release_dummy_page(struct drm_device *dev, void *res) +{ + struct page *dummy_page = (struct page *)res; + + __free_page(dummy_page); +} + +vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address; + unsigned long pfn; + struct page *page; + + /* Allocate new dummy page to map all the VA range in this VMA to it*/ + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + /* Set the page to be freed using drmm release action */ + if (drmm_add_action_or_reset(bo->base.dev, ttm_bo_release_dummy_page, + page)) + return VM_FAULT_OOM; + + pfn = page_to_pfn(page); + + /* Prefault the entire VMA range right away to avoid further faults */ + for (address = vma->vm_start; address < vma->vm_end; + address += PAGE_SIZE) + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_dummy_page); + +vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + vm_fault_t ret; + pgprot_t prot; + int idx; + + ret = ttm_bo_vm_reserve(bo, vmf); + if (ret) + return ret; + + prot = vma->vm_page_prot; + if (drm_dev_enter(bo->base.dev, &idx)) { + ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); + drm_dev_exit(idx); + } else { + ret = ttm_bo_vm_dummy_page(vmf, prot); + } + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return ret; + + dma_resv_unlock(bo->base.resv); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_fault); + +void ttm_bo_vm_open(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + + WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); + + drm_gem_object_get(&bo->base); +} +EXPORT_SYMBOL(ttm_bo_vm_open); + +void ttm_bo_vm_close(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + + drm_gem_object_put(&bo->base); + vma->vm_private_data = NULL; +} +EXPORT_SYMBOL(ttm_bo_vm_close); + +static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo, + unsigned long offset, + uint8_t *buf, int len, int write) +{ + unsigned long page = offset >> PAGE_SHIFT; + unsigned long bytes_left = len; + int ret; + + /* Copy a page at a time, that way no extra virtual address + * mapping is needed + */ + offset -= page << PAGE_SHIFT; + do { + unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); + struct ttm_bo_kmap_obj map; + void *ptr; + bool is_iomem; + + ret = ttm_bo_kmap(bo, page, 1, &map); + if (ret) + return ret; + + ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset; + WARN_ON_ONCE(is_iomem); + if (write) + memcpy(ptr, buf, bytes); + else + memcpy(buf, ptr, bytes); + ttm_bo_kunmap(&map); + + page++; + buf += bytes; + bytes_left -= bytes; + offset = 0; + } while (bytes_left); + + return len; +} + +/** + * ttm_bo_access - Helper to access a buffer object + * + * @bo: ttm buffer object + * @offset: access offset into buffer object + * @buf: pointer to caller memory to read into or write from + * @len: length of access + * @write: write access + * + * Utility function to access a buffer object. Useful when buffer object cannot + * be easily mapped (non-contiguous, non-visible, etc...). Should not directly + * be exported to user space via a peak / poke interface. + * + * Returns: + * @len if successful, negative error code on failure. + */ +int ttm_bo_access(struct ttm_buffer_object *bo, unsigned long offset, + void *buf, int len, int write) +{ + int ret; + + if (len < 1 || (offset + len) > bo->base.size) + return -EIO; + + ret = ttm_bo_reserve(bo, true, false, NULL); + if (ret) + return ret; + + if (!bo->resource) { + ret = -ENODATA; + goto unlock; + } + + switch (bo->resource->mem_type) { + case TTM_PL_SYSTEM: + fallthrough; + case TTM_PL_TT: + ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write); + break; + default: + if (bo->bdev->funcs->access_memory) + ret = bo->bdev->funcs->access_memory + (bo, offset, buf, len, write); + else + ret = -EIO; + } + +unlock: + ttm_bo_unreserve(bo); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_access); + +int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + unsigned long offset = (addr) - vma->vm_start + + ((vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node)) + << PAGE_SHIFT); + + return ttm_bo_access(bo, offset, buf, len, write); +} +EXPORT_SYMBOL(ttm_bo_vm_access); + +static const struct vm_operations_struct ttm_bo_vm_ops = { + .fault = ttm_bo_vm_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access, +}; + +/** + * ttm_bo_mmap_obj - mmap memory backed by a ttm buffer object. + * + * @vma: vma as input from the fbdev mmap method. + * @bo: The bo backing the address space. + * + * Maps a buffer object. + */ +int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) +{ + /* Enforce no COW since would have really strange behavior with it. */ + if (vma_is_cow_mapping(vma)) + return -EINVAL; + + drm_gem_object_get(&bo->base); + + /* + * Drivers may want to override the vm_ops field. Otherwise we + * use TTM's default callbacks. + */ + if (!vma->vm_ops) + vma->vm_ops = &ttm_bo_vm_ops; + + /* + * Note: We're transferring the bo reference to + * vma->vm_private_data here. + */ + + vma->vm_private_data = bo; + + vm_flags_set(vma, VM_PFNMAP | VM_IO | VM_DONTEXPAND | VM_DONTDUMP); + return 0; +} +EXPORT_SYMBOL(ttm_bo_mmap_obj); diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c new file mode 100644 index 000000000..d3bfb9a69 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -0,0 +1,319 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +/* + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#define pr_fmt(fmt) "[TTM DEVICE] " fmt + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "ttm_module.h" +#include "ttm_bo_internal.h" + +/* + * ttm_global_mutex - protecting the global state + */ +static DEFINE_MUTEX(ttm_global_mutex); +static unsigned ttm_glob_use_count; +struct ttm_global ttm_glob; +EXPORT_SYMBOL(ttm_glob); + +struct dentry *ttm_debugfs_root; + +static void ttm_global_release(void) +{ + struct ttm_global *glob = &ttm_glob; + + mutex_lock(&ttm_global_mutex); + if (--ttm_glob_use_count > 0) + goto out; + + ttm_pool_mgr_fini(); + debugfs_remove(ttm_debugfs_root); + + __free_page(glob->dummy_read_page); + memset(glob, 0, sizeof(*glob)); +out: + mutex_unlock(&ttm_global_mutex); +} + +static int ttm_global_init(void) +{ + struct ttm_global *glob = &ttm_glob; + unsigned long num_pages, num_dma32; + struct sysinfo si; + int ret = 0; + + mutex_lock(&ttm_global_mutex); + if (++ttm_glob_use_count > 1) + goto out; + + si_meminfo(&si); + + ttm_debugfs_root = debugfs_create_dir("ttm", NULL); + if (IS_ERR(ttm_debugfs_root)) { + ttm_debugfs_root = NULL; + } + + /* Limit the number of pages in the pool to about 50% of the total + * system memory. + */ + num_pages = ((u64)si.totalram * si.mem_unit) >> PAGE_SHIFT; + num_pages /= 2; + + /* But for DMA32 we limit ourself to only use 2GiB maximum. */ + num_dma32 = (u64)(si.totalram - si.totalhigh) * si.mem_unit + >> PAGE_SHIFT; + num_dma32 = min(num_dma32, 2UL << (30 - PAGE_SHIFT)); + + ttm_pool_mgr_init(num_pages); + ttm_tt_mgr_init(num_pages, num_dma32); + + glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32 | + __GFP_NOWARN); + + /* Retry without GFP_DMA32 for platforms DMA32 is not available */ + if (unlikely(glob->dummy_read_page == NULL)) { + glob->dummy_read_page = alloc_page(__GFP_ZERO); + if (unlikely(glob->dummy_read_page == NULL)) { + ret = -ENOMEM; + goto out; + } + pr_warn("Using GFP_DMA32 fallback for dummy_read_page\n"); + } + + INIT_LIST_HEAD(&glob->device_list); + atomic_set(&glob->bo_count, 0); + + debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root, + &glob->bo_count); +out: + if (ret && ttm_debugfs_root) + debugfs_remove(ttm_debugfs_root); + if (ret) + --ttm_glob_use_count; + mutex_unlock(&ttm_global_mutex); + return ret; +} + +/** + * ttm_device_prepare_hibernation - move GTT BOs to shmem for hibernation. + * + * @bdev: A pointer to a struct ttm_device to prepare hibernation for. + * + * Return: 0 on success, negative number on failure. + */ +int ttm_device_prepare_hibernation(struct ttm_device *bdev) +{ + struct ttm_operation_ctx ctx = { }; + int ret; + + do { + ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL); + } while (ret > 0); + return ret; +} +EXPORT_SYMBOL(ttm_device_prepare_hibernation); + +/* + * A buffer object shrink method that tries to swap out the first + * buffer object on the global::swap_lru list. + */ +int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags) +{ + struct ttm_global *glob = &ttm_glob; + struct ttm_device *bdev; + int ret = 0; + + mutex_lock(&ttm_global_mutex); + list_for_each_entry(bdev, &glob->device_list, device_list) { + ret = ttm_device_swapout(bdev, ctx, gfp_flags); + if (ret > 0) { + list_move_tail(&bdev->device_list, &glob->device_list); + break; + } + } + mutex_unlock(&ttm_global_mutex); + return ret; +} + +int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + gfp_t gfp_flags) +{ + struct ttm_resource_manager *man; + unsigned i; + s64 lret; + + for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) { + man = ttm_manager_type(bdev, i); + if (!man || !man->use_tt) + continue; + + lret = ttm_bo_swapout(bdev, ctx, man, gfp_flags, 1); + /* Can be both positive (num_pages) and negative (error) */ + if (lret) + return lret; + } + return 0; +} +EXPORT_SYMBOL(ttm_device_swapout); + +/** + * ttm_device_init + * + * @bdev: A pointer to a struct ttm_device to initialize. + * @funcs: Function table for the device. + * @dev: The core kernel device pointer for DMA mappings and allocations. + * @mapping: The address space to use for this bo. + * @vma_manager: A pointer to a vma manager. + * @alloc_flags: TTM_ALLOCATION_* flags. + * + * Initializes a struct ttm_device: + * Returns: + * !0: Failure. + */ +int ttm_device_init(struct ttm_device *bdev, const struct ttm_device_funcs *funcs, + struct device *dev, struct address_space *mapping, + struct drm_vma_offset_manager *vma_manager, + unsigned int alloc_flags) +{ + struct ttm_global *glob = &ttm_glob; + int ret, nid; + + if (WARN_ON(vma_manager == NULL)) + return -EINVAL; + + ret = ttm_global_init(); + if (ret) + return ret; + + bdev->wq = alloc_workqueue("ttm", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_UNBOUND, 16); + if (!bdev->wq) { + ttm_global_release(); + return -ENOMEM; + } + + bdev->alloc_flags = alloc_flags; + bdev->funcs = funcs; + + ttm_sys_man_init(bdev); + + if (dev) + nid = dev_to_node(dev); + else + nid = NUMA_NO_NODE; + + ttm_pool_init(&bdev->pool, dev, nid, alloc_flags); + + bdev->vma_manager = vma_manager; + spin_lock_init(&bdev->lru_lock); + INIT_LIST_HEAD(&bdev->unevictable); + bdev->dev_mapping = mapping; + mutex_lock(&ttm_global_mutex); + list_add_tail(&bdev->device_list, &glob->device_list); + mutex_unlock(&ttm_global_mutex); + + return 0; +} +EXPORT_SYMBOL(ttm_device_init); + +void ttm_device_fini(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man; + unsigned i; + + mutex_lock(&ttm_global_mutex); + list_del(&bdev->device_list); + mutex_unlock(&ttm_global_mutex); + + drain_workqueue(bdev->wq); + destroy_workqueue(bdev->wq); + + man = ttm_manager_type(bdev, TTM_PL_SYSTEM); + ttm_resource_manager_set_used(man, false); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL); + + spin_lock(&bdev->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + if (list_empty(&man->lru[0])) + pr_debug("Swap list %d was clean\n", i); + spin_unlock(&bdev->lru_lock); + + ttm_pool_fini(&bdev->pool); + ttm_global_release(); +} +EXPORT_SYMBOL(ttm_device_fini); + +static void ttm_device_clear_lru_dma_mappings(struct ttm_device *bdev, + struct list_head *list) +{ + struct ttm_resource *res; + + spin_lock(&bdev->lru_lock); + while ((res = ttm_lru_first_res_or_null(list))) { + struct ttm_buffer_object *bo = res->bo; + + /* Take ref against racing releases once lru_lock is unlocked */ + if (!ttm_bo_get_unless_zero(bo)) + continue; + + list_del_init(&bo->resource->lru.link); + spin_unlock(&bdev->lru_lock); + + if (bo->ttm) + ttm_tt_unpopulate(bo->bdev, bo->ttm); + + ttm_bo_put(bo); + spin_lock(&bdev->lru_lock); + } + spin_unlock(&bdev->lru_lock); +} + +void ttm_device_clear_dma_mappings(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man; + unsigned int i, j; + + ttm_device_clear_lru_dma_mappings(bdev, &bdev->unevictable); + + for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) { + man = ttm_manager_type(bdev, i); + if (!man || !man->use_tt) + continue; + + for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) + ttm_device_clear_lru_dma_mappings(bdev, &man->lru[j]); + } +} +EXPORT_SYMBOL(ttm_device_clear_dma_mappings); diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c new file mode 100644 index 000000000..bc7a83a9f --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#include + +#include +#include + +static void ttm_eu_backoff_reservation_reverse(struct list_head *list, + struct ttm_validate_buffer *entry) +{ + list_for_each_entry_continue_reverse(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + dma_resv_unlock(bo->base.resv); + } +} + +void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list) +{ + struct ttm_validate_buffer *entry; + + if (list_empty(list)) + return; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + ttm_bo_move_to_lru_tail_unlocked(bo); + dma_resv_unlock(bo->base.resv); + } + + if (ticket) + ww_acquire_fini(ticket); +} +EXPORT_SYMBOL(ttm_eu_backoff_reservation); + +/* + * Reserve buffers for validation. + * + * If a buffer in the list is marked for CPU access, we back off and + * wait for that buffer to become free for GPU access. + * + * If a buffer is reserved for another validation, the validator with + * the highest validation sequence backs off and waits for that buffer + * to become unreserved. This prevents deadlocks when validating multiple + * buffers in different orders. + */ + +int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list, bool intr, + struct list_head *dups) +{ + struct ttm_validate_buffer *entry; + int ret; + + if (list_empty(list)) + return 0; + + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + unsigned int num_fences; + + ret = ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); + if (ret == -EALREADY && dups) { + struct ttm_validate_buffer *safe = entry; + entry = list_prev_entry(entry, head); + list_del(&safe->head); + list_add(&safe->head, dups); + continue; + } + + num_fences = max(entry->num_shared, 1u); + if (!ret) { + ret = dma_resv_reserve_fences(bo->base.resv, + num_fences); + if (!ret) + continue; + } + + /* uh oh, we lost out, drop every reservation and try + * to only reserve this buffer, then start over if + * this succeeds. + */ + ttm_eu_backoff_reservation_reverse(list, entry); + + if (ret == -EDEADLK) { + ret = ttm_bo_reserve_slowpath(bo, intr, ticket); + } + + if (!ret) + ret = dma_resv_reserve_fences(bo->base.resv, + num_fences); + + if (unlikely(ret != 0)) { + if (ticket) { + ww_acquire_done(ticket); + ww_acquire_fini(ticket); + } + return ret; + } + + /* move this item to the front of the list, + * forces correct iteration of the loop without keeping track + */ + list_del(&entry->head); + list_add(&entry->head, list); + } + + return 0; +} +EXPORT_SYMBOL(ttm_eu_reserve_buffers); + +void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, + struct dma_fence *fence) +{ + struct ttm_validate_buffer *entry; + + if (list_empty(list)) + return; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + dma_resv_add_fence(bo->base.resv, fence, entry->num_shared ? + DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE); + ttm_bo_move_to_lru_tail_unlocked(bo); + dma_resv_unlock(bo->base.resv); + } + if (ticket) + ww_acquire_fini(ticket); +} +EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c new file mode 100644 index 000000000..aa137ead5 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + * Jerome Glisse + */ +#include +#include +#include +#include +#include +#include +#include + +#include "ttm_module.h" + +/** + * DOC: TTM + * + * TTM is a memory manager for accelerator devices with dedicated memory. + * + * The basic idea is that resources are grouped together in buffer objects of + * certain size and TTM handles lifetime, movement and CPU mappings of those + * objects. + * + * TODO: Add more design background and information here. + */ + +/** + * ttm_prot_from_caching - Modify the page protection according to the + * ttm cacing mode + * @caching: The ttm caching mode + * @tmp: The original page protection + * + * Return: The modified page protection + */ +pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp) +{ + /* Cached mappings need no adjustment */ + if (caching == ttm_cached) + return tmp; + +#if defined(__i386__) || defined(__x86_64__) + if (caching == ttm_write_combined) + tmp = pgprot_writecombine(tmp); +#ifndef CONFIG_UML + else if (boot_cpu_data.x86 > 3) + tmp = pgprot_noncached(tmp); +#endif /* CONFIG_UML */ +#endif /* __i386__ || __x86_64__ */ +#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ + defined(__powerpc__) || defined(__mips__) || defined(__loongarch__) || \ + defined(__riscv) + if (caching == ttm_write_combined) + tmp = pgprot_writecombine(tmp); + else + tmp = pgprot_noncached(tmp); +#endif +#if defined(__sparc__) + tmp = pgprot_noncached(tmp); +#endif + return tmp; +} + +MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse"); +MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/ttm_module.h b/drivers/gpu/drm/ttm/ttm_module.h new file mode 100644 index 000000000..767fe22ae --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_module.h @@ -0,0 +1,43 @@ +/************************************************************************** + * + * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifndef _TTM_MODULE_H_ +#define _TTM_MODULE_H_ + +#define TTM_PFX "[TTM] " + +struct dentry; +struct ttm_device; + +extern struct dentry *ttm_debugfs_root; + +void ttm_sys_man_init(struct ttm_device *bdev); + +#endif /* _TTM_MODULE_H_ */ diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c new file mode 100644 index 000000000..1bf37023f --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -0,0 +1,1564 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +/* Pooling of allocated pages is necessary because changing the caching + * attributes on x86 of the linear mapping requires a costly cross CPU TLB + * invalidate for those addresses. + * + * Additional to that allocations from the DMA coherent API are pooled as well + * cause they are rather slow compared to alloc_pages+map. + */ + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_X86 +#include +#endif + +#include +#include +#include +#include + +#include "ttm_module.h" +#include "ttm_pool_internal.h" + +#ifdef CONFIG_FAULT_INJECTION +#include +static DECLARE_FAULT_ATTR(backup_fault_inject); + +/* + * Exposed to ttm_backup.c so a mid-compound subpage can be made to fail + * with -ENOMEM, exercising the reactive split-and-retry fallback in + * ttm_pool_backup() for high-order backups. + */ +bool ttm_backup_fault_inject_folio(void) +{ + return should_fail(&backup_fault_inject, 1); +} +#else +#define should_fail(...) false + +bool ttm_backup_fault_inject_folio(void) +{ + return false; +} +#endif + +/** + * struct ttm_pool_dma - Helper object for coherent DMA mappings + * + * @addr: original DMA address returned for the mapping + * @vaddr: original vaddr return for the mapping and order in the lower bits + */ +struct ttm_pool_dma { + dma_addr_t addr; + unsigned long vaddr; +}; + +/** + * struct ttm_pool_alloc_state - Current state of the tt page allocation process + * @pages: Pointer to the next tt page pointer to populate. + * @caching_divide: Pointer to the first page pointer whose page has a staged but + * not committed caching transition from write-back to @tt_caching. + * @dma_addr: Pointer to the next tt dma_address entry to populate if any. + * @remaining_pages: Remaining pages to populate. + * @tt_caching: The requested cpu-caching for the pages allocated. + */ +struct ttm_pool_alloc_state { + struct page **pages; + struct page **caching_divide; + dma_addr_t *dma_addr; + pgoff_t remaining_pages; + enum ttm_caching tt_caching; +}; + +/** + * struct ttm_pool_tt_restore - State representing restore from backup + * @pool: The pool used for page allocation while restoring. + * @snapshot_alloc: A snapshot of the most recent struct ttm_pool_alloc_state. + * @alloced_page: Pointer to the page most recently allocated from a pool or system. + * @first_dma: The dma address corresponding to @alloced_page if dma_mapping + * is requested. + * @alloced_pages: The number of allocated pages present in the struct ttm_tt + * page vector from this restore session. + * @restored_pages: The number of 4K pages restored for @alloced_page (which + * is typically a multi-order page). + * @page_caching: The struct ttm_tt requested caching + * @order: The order of @alloced_page. + * + * Recovery from backup might fail when we've recovered less than the + * full ttm_tt. In order not to loose any data (yet), keep information + * around that allows us to restart a failed ttm backup recovery. + */ +struct ttm_pool_tt_restore { + struct ttm_pool *pool; + struct ttm_pool_alloc_state snapshot_alloc; + struct page *alloced_page; + dma_addr_t first_dma; + pgoff_t alloced_pages; + pgoff_t restored_pages; + enum ttm_caching page_caching; + unsigned int order; +}; + +static unsigned long page_pool_size; + +MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool per NUMA node"); +module_param(page_pool_size, ulong, 0644); + +static unsigned long pool_node_limit[MAX_NUMNODES]; +static atomic_long_t allocated_pages[MAX_NUMNODES]; + +static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS]; +static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS]; + +static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS]; +static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS]; + +static spinlock_t shrinker_lock; +static struct list_head shrinker_list; +static struct shrinker *mm_shrinker; +static DECLARE_RWSEM(pool_shrink_rwsem); + +static int ttm_pool_nid(struct ttm_pool *pool) +{ + int nid = NUMA_NO_NODE; + if (pool) + nid = pool->nid; + if (nid == NUMA_NO_NODE) + nid = numa_node_id(); + return nid; +} + +/* Allocate pages of size 1 << order with the given gfp_flags */ +static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags, + unsigned int order) +{ + const unsigned int beneficial_order = ttm_pool_beneficial_order(pool); + unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; + struct ttm_pool_dma *dma; + struct page *p; + void *vaddr; + + /* Don't set the __GFP_COMP flag for higher order allocations. + * Mapping pages directly into an userspace process and calling + * put_page() on a TTM allocated page is illegal. + */ + if (order) + gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | + __GFP_THISNODE; + + /* + * Do not add latency to the allocation path for allocations orders + * device tolds us do not bring them additional performance gains. + */ + if (order && beneficial_order && order != beneficial_order) + gfp_flags &= ~__GFP_RECLAIM; + + if (beneficial_order && order == beneficial_order) { + gfp_flags &= ~__GFP_NORETRY; + gfp_flags |= __GFP_RETRY_MAYFAIL; + } + + if (!ttm_pool_uses_dma_alloc(pool)) { + p = alloc_pages_node(pool->nid, gfp_flags, order); + if (p) { + p->private = order; + mod_lruvec_page_state(p, NR_GPU_ACTIVE, 1 << order); + } + return p; + } + + dma = kmalloc_obj(*dma); + if (!dma) + return NULL; + + if (order) + attr |= DMA_ATTR_NO_WARN; + + vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE, + &dma->addr, gfp_flags, attr); + if (!vaddr) + goto error_free; + + /* TODO: This is an illegal abuse of the DMA API, but we need to rework + * TTM page fault handling and extend the DMA API to clean this up. + */ + if (is_vmalloc_addr(vaddr)) + p = vmalloc_to_page(vaddr); + else + p = virt_to_page(vaddr); + + dma->vaddr = (unsigned long)vaddr | order; + p->private = (unsigned long)dma; + return p; + +error_free: + kfree(dma); + return NULL; +} + +static void __free_pages_gpu_account(struct page *p, unsigned int order, + bool reclaim) +{ + mod_lruvec_page_state(p, reclaim ? NR_GPU_RECLAIM : NR_GPU_ACTIVE, + -(1 << order)); + __free_pages(p, order); +} + +/* Reset the caching and pages of size 1 << order */ +static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, + unsigned int order, struct page *p, bool reclaim) +{ + unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; + struct ttm_pool_dma *dma; + void *vaddr; + +#ifdef CONFIG_X86 + /* We don't care that set_pages_wb is inefficient here. This is only + * used when we have to shrink and CPU overhead is irrelevant then. + */ + if (caching != ttm_cached && !PageHighMem(p)) + set_pages_wb(p, 1 << order); +#endif + + if (!pool || !ttm_pool_uses_dma_alloc(pool)) { + __free_pages_gpu_account(p, order, reclaim); + return; + } + + if (order) + attr |= DMA_ATTR_NO_WARN; + + dma = (void *)p->private; + vaddr = (void *)(dma->vaddr & PAGE_MASK); + dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr, + attr); + kfree(dma); +} + +/* Apply any cpu-caching deferred during page allocation */ +static int ttm_pool_apply_caching(struct ttm_pool_alloc_state *alloc) +{ +#ifdef CONFIG_X86 + unsigned int num_pages = alloc->pages - alloc->caching_divide; + + if (!num_pages) + return 0; + + switch (alloc->tt_caching) { + case ttm_cached: + break; + case ttm_write_combined: + return set_pages_array_wc(alloc->caching_divide, num_pages); + case ttm_uncached: + return set_pages_array_uc(alloc->caching_divide, num_pages); + } +#endif + alloc->caching_divide = alloc->pages; + return 0; +} + +/* DMA Map pages of 1 << order size and return the resulting dma_address. */ +static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, + struct page *p, dma_addr_t *dma_addr) +{ + dma_addr_t addr; + + if (ttm_pool_uses_dma_alloc(pool)) { + struct ttm_pool_dma *dma = (void *)p->private; + + addr = dma->addr; + } else { + size_t size = (1ULL << order) * PAGE_SIZE; + + addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(pool->dev, addr)) + return -EFAULT; + } + + *dma_addr = addr; + + return 0; +} + +/* Unmap pages of 1 << order size */ +static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, + unsigned int num_pages) +{ + /* Unmapped while freeing the page */ + if (ttm_pool_uses_dma_alloc(pool)) + return; + + dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT, + DMA_BIDIRECTIONAL); +} + +/* Give pages into a specific pool_type */ +static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) +{ + unsigned int i, num_pages = 1 << pt->order; + int nid = page_to_nid(p); + + for (i = 0; i < num_pages; ++i) { + if (PageHighMem(p)) + clear_highpage(p + i); + else + clear_page(page_address(p + i)); + } + + INIT_LIST_HEAD(&p->lru); + rcu_read_lock(); + list_lru_add(&pt->pages, &p->lru, nid, NULL); + rcu_read_unlock(); + + atomic_long_add(num_pages, &allocated_pages[nid]); + mod_lruvec_page_state(p, NR_GPU_ACTIVE, -num_pages); + mod_lruvec_page_state(p, NR_GPU_RECLAIM, num_pages); +} + +static enum lru_status take_one_from_lru(struct list_head *item, + struct list_lru_one *list, + void *cb_arg) +{ + struct page **out_page = cb_arg; + struct page *p = container_of(item, struct page, lru); + list_lru_isolate(list, item); + + *out_page = p; + return LRU_REMOVED; +} + +/* Take pages from a specific pool_type, return NULL when nothing available */ +static struct page *ttm_pool_type_take(struct ttm_pool_type *pt, int nid) +{ + int ret; + struct page *p = NULL; + unsigned long nr_to_walk = 1; + + ret = list_lru_walk_node(&pt->pages, nid, take_one_from_lru, (void *)&p, &nr_to_walk); + if (ret == 1 && p) { + atomic_long_sub(1 << pt->order, &allocated_pages[nid]); + mod_lruvec_page_state(p, NR_GPU_ACTIVE, (1 << pt->order)); + mod_lruvec_page_state(p, NR_GPU_RECLAIM, -(1 << pt->order)); + } + return p; +} + +/* Initialize and add a pool type to the global shrinker list */ +static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool, + enum ttm_caching caching, unsigned int order) +{ + pt->pool = pool; + pt->caching = caching; + pt->order = order; + list_lru_init(&pt->pages); + + spin_lock(&shrinker_lock); + list_add_tail(&pt->shrinker_list, &shrinker_list); + spin_unlock(&shrinker_lock); +} + +static enum lru_status pool_move_to_dispose_list(struct list_head *item, + struct list_lru_one *list, + void *cb_arg) +{ + struct list_head *dispose = cb_arg; + + list_lru_isolate_move(list, item, dispose); + + return LRU_REMOVED; +} + +static void ttm_pool_dispose_list(struct ttm_pool_type *pt, + struct list_head *dispose) +{ + while (!list_empty(dispose)) { + struct page *p; + p = list_first_entry(dispose, struct page, lru); + list_del_init(&p->lru); + atomic_long_sub(1 << pt->order, &allocated_pages[page_to_nid(p)]); + ttm_pool_free_page(pt->pool, pt->caching, pt->order, p, true); + } +} + +/* Remove a pool_type from the global shrinker list and free all pages */ +static void ttm_pool_type_fini(struct ttm_pool_type *pt) +{ + LIST_HEAD(dispose); + + spin_lock(&shrinker_lock); + list_del(&pt->shrinker_list); + spin_unlock(&shrinker_lock); + + list_lru_walk(&pt->pages, pool_move_to_dispose_list, &dispose, LONG_MAX); + ttm_pool_dispose_list(pt, &dispose); +} + +/* Return the pool_type to use for the given caching and order */ +static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool, + enum ttm_caching caching, + unsigned int order) +{ + if (ttm_pool_uses_dma_alloc(pool)) + return &pool->caching[caching].orders[order]; + +#ifdef CONFIG_X86 + switch (caching) { + case ttm_write_combined: + if (ttm_pool_uses_dma32(pool)) + return &global_dma32_write_combined[order]; + + return &global_write_combined[order]; + case ttm_uncached: + if (ttm_pool_uses_dma32(pool)) + return &global_dma32_uncached[order]; + + return &global_uncached[order]; + default: + break; + } +#endif + + return NULL; +} + +/* Free pages using the per-node shrinker list */ +static unsigned int ttm_pool_shrink(int nid, unsigned long num_to_free) +{ + LIST_HEAD(dispose); + struct ttm_pool_type *pt; + unsigned int num_pages; + + down_read(&pool_shrink_rwsem); + spin_lock(&shrinker_lock); + pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list); + list_move_tail(&pt->shrinker_list, &shrinker_list); + spin_unlock(&shrinker_lock); + + num_pages = list_lru_walk_node(&pt->pages, nid, pool_move_to_dispose_list, &dispose, &num_to_free); + num_pages *= 1 << pt->order; + + ttm_pool_dispose_list(pt, &dispose); + up_read(&pool_shrink_rwsem); + + return num_pages; +} + +/* Return the allocation order based for a page */ +static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) +{ + if (ttm_pool_uses_dma_alloc(pool)) { + struct ttm_pool_dma *dma = (void *)p->private; + + return dma->vaddr & ~PAGE_MASK; + } + + return p->private; +} + +/* + * Split larger pages so that we can free each PAGE_SIZE page as soon + * as it has been backed up, in order to avoid memory pressure during + * reclaim. + */ +static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p) +{ + unsigned int order = ttm_pool_page_order(pool, p); + pgoff_t nr; + + if (!order) + return; + + split_page(p, order); + nr = 1UL << order; + while (nr--) + (p++)->private = 0; +} + +/** + * DOC: Partial backup and restoration of a struct ttm_tt. + * + * Swapout using ttm_backup_backup_folio() and swapin using + * ttm_backup_copy_page() may fail. + * The former most likely due to lack of swap-space or memory, the latter due + * to lack of memory or because of signal interruption during waits. + * + * Backup failure is easily handled by using a ttm_tt pages vector that holds + * both backup handles and page pointers. This has to be taken into account when + * restoring such a ttm_tt from backup, and when freeing it while backed up. + * When restoring, for simplicity, new pages are actually allocated from the + * pool and the contents of any old pages are copied in and then the old pages + * are released. + * + * For restoration failures, the struct ttm_pool_tt_restore holds sufficient state + * to be able to resume an interrupted restore, and that structure is freed once + * the restoration is complete. If the struct ttm_tt is destroyed while there + * is a valid struct ttm_pool_tt_restore attached, that is also properly taken + * care of. + */ + +/* Is restore ongoing for the currently allocated page? */ +static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore) +{ + return restore && restore->restored_pages < (1 << restore->order); +} + +/* DMA unmap and free a multi-order page, either to the relevant pool or to system. */ +static pgoff_t ttm_pool_unmap_and_free(struct ttm_pool *pool, struct page *page, + const dma_addr_t *dma_addr, enum ttm_caching caching) +{ + struct ttm_pool_type *pt = NULL; + unsigned int order; + pgoff_t nr; + + if (pool) { + order = ttm_pool_page_order(pool, page); + nr = (1UL << order); + if (dma_addr) + ttm_pool_unmap(pool, *dma_addr, nr); + + pt = ttm_pool_select_type(pool, caching, order); + } else { + order = page->private; + nr = (1UL << order); + } + + if (pt) + ttm_pool_type_give(pt, page); + else + ttm_pool_free_page(pool, caching, order, page, false); + + return nr; +} + +/* Populate the page-array using the most recent allocated multi-order page. */ +static void ttm_pool_allocated_page_commit(struct page *allocated, + dma_addr_t first_dma, + struct ttm_pool_alloc_state *alloc, + pgoff_t nr) +{ + pgoff_t i; + + for (i = 0; i < nr; ++i) + *alloc->pages++ = allocated++; + + alloc->remaining_pages -= nr; + + if (!alloc->dma_addr) + return; + + for (i = 0; i < nr; ++i) { + *alloc->dma_addr++ = first_dma; + first_dma += PAGE_SIZE; + } +} + +/* + * When restoring, restore backed-up content to the newly allocated page and + * if successful, populate the page-table and dma-address arrays. + */ +static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore, + struct file *backup, + const struct ttm_operation_ctx *ctx, + struct ttm_pool_alloc_state *alloc) + +{ + pgoff_t i, nr = 1UL << restore->order; + struct page **first_page = alloc->pages; + struct page *p; + int ret = 0; + + for (i = restore->restored_pages; i < nr; ++i) { + p = first_page[i]; + if (ttm_backup_page_ptr_is_handle(p)) { + unsigned long handle = ttm_backup_page_ptr_to_handle(p); + gfp_t additional_gfp = ctx->gfp_retry_mayfail ? + __GFP_RETRY_MAYFAIL | __GFP_NOWARN : 0; + + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible && + should_fail(&backup_fault_inject, 1)) { + ret = -EINTR; + break; + } + + if (handle == 0) { + restore->restored_pages++; + continue; + } + + ret = ttm_backup_copy_page(backup, restore->alloced_page + i, + handle, ctx->interruptible, + additional_gfp); + if (ret) + break; + + ttm_backup_drop(backup, handle); + } else if (p) { + /* + * We could probably avoid splitting the old page + * using clever logic, but ATM we don't care, as + * we prioritize releasing memory ASAP. Note that + * here, the old retained page is always write-back + * cached. + */ + ttm_pool_split_for_swap(restore->pool, p); + copy_highpage(restore->alloced_page + i, p); + __free_pages_gpu_account(p, 0, false); + } + + restore->restored_pages++; + first_page[i] = ttm_backup_handle_to_page_ptr(0); + } + + if (ret) { + if (!restore->restored_pages) { + dma_addr_t *dma_addr = alloc->dma_addr ? &restore->first_dma : NULL; + + ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, + dma_addr, restore->page_caching); + restore->restored_pages = nr; + } + return ret; + } + + ttm_pool_allocated_page_commit(restore->alloced_page, restore->first_dma, + alloc, nr); + if (restore->page_caching == alloc->tt_caching || PageHighMem(restore->alloced_page)) + alloc->caching_divide = alloc->pages; + restore->snapshot_alloc = *alloc; + restore->alloced_pages += nr; + + return 0; +} + +/* If restoring, save information needed for ttm_pool_restore_commit(). */ +static void +ttm_pool_page_allocated_restore(struct ttm_pool *pool, unsigned int order, + struct page *p, + enum ttm_caching page_caching, + dma_addr_t first_dma, + struct ttm_pool_tt_restore *restore, + const struct ttm_pool_alloc_state *alloc) +{ + restore->pool = pool; + restore->order = order; + restore->restored_pages = 0; + restore->page_caching = page_caching; + restore->first_dma = first_dma; + restore->alloced_page = p; + restore->snapshot_alloc = *alloc; +} + +/* + * Called when we got a page, either from a pool or newly allocated. + * if needed, dma map the page and populate the dma address array. + * Populate the page address array. + * If the caching is consistent, update any deferred caching. Otherwise + * stage this page for an upcoming deferred caching update. + */ +static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, + struct page *p, enum ttm_caching page_caching, + struct ttm_pool_alloc_state *alloc, + struct ttm_pool_tt_restore *restore) +{ + bool caching_consistent; + dma_addr_t first_dma; + int r = 0; + + caching_consistent = (page_caching == alloc->tt_caching) || PageHighMem(p); + + if (caching_consistent) { + r = ttm_pool_apply_caching(alloc); + if (r) + return r; + } + + if (alloc->dma_addr) { + r = ttm_pool_map(pool, order, p, &first_dma); + if (r) + return r; + } + + if (restore) { + ttm_pool_page_allocated_restore(pool, order, p, page_caching, + first_dma, restore, alloc); + } else { + ttm_pool_allocated_page_commit(p, first_dma, alloc, 1UL << order); + + if (caching_consistent) + alloc->caching_divide = alloc->pages; + } + + return 0; +} + +/** + * ttm_pool_free_range() - Free a range of TTM pages + * @pool: The pool used for allocating. + * @tt: The struct ttm_tt holding the page pointers. + * @caching: The page caching mode used by the range. + * @start_page: index for first page to free. + * @end_page: index for last page to free + 1. + * + * During allocation the ttm_tt page-vector may be populated with ranges of + * pages with different attributes if allocation hit an error without being + * able to completely fulfill the allocation. This function can be used + * to free these individual ranges. + */ +static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, + enum ttm_caching caching, + pgoff_t start_page, pgoff_t end_page) +{ + struct page **pages = &tt->pages[start_page]; + struct file *backup = tt->backup; + pgoff_t i, nr; + + for (i = start_page; i < end_page; i += nr, pages += nr) { + struct page *p = *pages; + + nr = 1; + if (ttm_backup_page_ptr_is_handle(p)) { + unsigned long handle = ttm_backup_page_ptr_to_handle(p); + + if (handle != 0) + ttm_backup_drop(backup, handle); + } else if (p) { + dma_addr_t *dma_addr = tt->dma_address ? + tt->dma_address + i : NULL; + + nr = ttm_pool_unmap_and_free(pool, p, dma_addr, caching); + } + } +} + +static void ttm_pool_alloc_state_init(const struct ttm_tt *tt, + struct ttm_pool_alloc_state *alloc) +{ + alloc->pages = tt->pages; + alloc->caching_divide = tt->pages; + alloc->dma_addr = tt->dma_address; + alloc->remaining_pages = tt->num_pages; + alloc->tt_caching = tt->caching; +} + +/* + * Find a suitable allocation order based on highest desired order + * and number of remaining pages + */ +static unsigned int ttm_pool_alloc_find_order(unsigned int highest, + const struct ttm_pool_alloc_state *alloc) +{ + return min_t(unsigned int, highest, __fls(alloc->remaining_pages)); +} + +static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx, + struct ttm_pool_alloc_state *alloc, + struct ttm_pool_tt_restore *restore) +{ + enum ttm_caching page_caching; + gfp_t gfp_flags = GFP_USER; + pgoff_t caching_divide; + unsigned int order; + bool allow_pools; + struct page *p; + int r; + + WARN_ON(!alloc->remaining_pages || ttm_tt_is_populated(tt)); + WARN_ON(alloc->dma_addr && !pool->dev); + + if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (ctx->gfp_retry_mayfail) + gfp_flags |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + if (ttm_pool_uses_dma32(pool)) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= GFP_HIGHUSER; + + page_caching = tt->caching; + allow_pools = true; + for (order = ttm_pool_alloc_find_order(MAX_PAGE_ORDER, alloc); + alloc->remaining_pages; + order = ttm_pool_alloc_find_order(order, alloc)) { + struct ttm_pool_type *pt; + + /* First, try to allocate a page from a pool if one exists. */ + p = NULL; + pt = ttm_pool_select_type(pool, page_caching, order); + if (pt && allow_pools) + p = ttm_pool_type_take(pt, ttm_pool_nid(pool)); + + /* + * If that fails or previously failed, allocate from system. + * Note that this also disallows additional pool allocations using + * write-back cached pools of the same order. Consider removing + * that behaviour. + */ + if (!p) { + page_caching = ttm_cached; + allow_pools = false; + p = ttm_pool_alloc_page(pool, gfp_flags, order); + } + /* If that fails, lower the order if possible and retry. */ + if (!p) { + if (order) { + --order; + page_caching = tt->caching; + allow_pools = true; + continue; + } + r = -ENOMEM; + goto error_free_all; + } + r = ttm_pool_page_allocated(pool, order, p, page_caching, alloc, + restore); + if (r) + goto error_free_page; + + if (ttm_pool_restore_valid(restore)) { + r = ttm_pool_restore_commit(restore, tt->backup, ctx, alloc); + if (r) + goto error_free_all; + } + } + + r = ttm_pool_apply_caching(alloc); + if (r) + goto error_free_all; + + kfree(tt->restore); + tt->restore = NULL; + + return 0; + +error_free_page: + ttm_pool_free_page(pool, page_caching, order, p, false); + +error_free_all: + if (tt->restore) + return r; + + caching_divide = alloc->caching_divide - tt->pages; + ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide); + ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, + tt->num_pages - alloc->remaining_pages); + + return r; +} + +/** + * ttm_pool_alloc - Fill a ttm_tt object + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx) +{ + struct ttm_pool_alloc_state alloc; + + if (WARN_ON(ttm_tt_is_backed_up(tt))) + return -EINVAL; + + ttm_pool_alloc_state_init(tt, &alloc); + + return __ttm_pool_alloc(pool, tt, ctx, &alloc, NULL); +} +EXPORT_SYMBOL(ttm_pool_alloc); + +/** + * ttm_pool_restore_and_alloc - Fill a ttm_tt, restoring previously backed-up + * content. + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. Read in backed-up content. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx) +{ + struct ttm_pool_tt_restore *restore = tt->restore; + struct ttm_pool_alloc_state alloc; + int ret; + + if (WARN_ON(!ttm_tt_is_backed_up(tt))) + return -EINVAL; + + if (!restore) { + gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; + + ttm_pool_alloc_state_init(tt, &alloc); + if (ctx->gfp_retry_mayfail) + gfp |= __GFP_RETRY_MAYFAIL; + + restore = kzalloc_obj(*restore, gfp); + if (!restore) + return -ENOMEM; + + restore->snapshot_alloc = alloc; + restore->pool = pool; + restore->restored_pages = 1; + + tt->restore = restore; + } else { + alloc = restore->snapshot_alloc; + if (ttm_pool_restore_valid(restore)) { + ret = ttm_pool_restore_commit(restore, tt->backup, + ctx, &alloc); + + if (ret) + return ret; + } + if (!alloc.remaining_pages) { + ret = ttm_pool_apply_caching(&alloc); + if (ret) + return ret; + + kfree(tt->restore); + tt->restore = NULL; + + return 0; + } + } + + return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore); +} + +/** + * ttm_pool_free - Free the backing pages from a ttm_tt object + * + * @pool: Pool to give pages back to. + * @tt: ttm_tt object to unpopulate + * + * Give the packing pages back to a pool or free them + */ +void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) +{ + int nid = ttm_pool_nid(pool); + + ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages); + + while (atomic_long_read(&allocated_pages[nid]) > pool_node_limit[nid]) { + unsigned long diff = atomic_long_read(&allocated_pages[nid]) - pool_node_limit[nid]; + ttm_pool_shrink(nid, diff); + } +} +EXPORT_SYMBOL(ttm_pool_free); + +/** + * ttm_pool_drop_backed_up() - Release content of a swapped-out struct ttm_tt + * @tt: The struct ttm_tt. + * + * Release handles with associated content or any remaining pages of + * a backed-up struct ttm_tt. + */ +void ttm_pool_drop_backed_up(struct ttm_tt *tt) +{ + struct ttm_pool_tt_restore *restore; + pgoff_t start_page = 0; + + WARN_ON(!ttm_tt_is_backed_up(tt)); + + restore = tt->restore; + + /* + * Unmap and free any uncommitted restore page. + * any tt page-array backup entries already read back has + * been cleared already + */ + if (ttm_pool_restore_valid(restore)) { + dma_addr_t *dma_addr = tt->dma_address ? &restore->first_dma : NULL; + + ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, + dma_addr, restore->page_caching); + restore->restored_pages = 1UL << restore->order; + } + + /* + * If a restore is ongoing, part of the tt pages may have a + * caching different than writeback. + */ + if (restore) { + pgoff_t mid = restore->snapshot_alloc.caching_divide - tt->pages; + + start_page = restore->alloced_pages; + WARN_ON(mid > start_page); + /* Pages that might be dma-mapped and non-cached */ + ttm_pool_free_range(restore->pool, tt, tt->caching, + 0, mid); + /* Pages that might be dma-mapped but cached */ + ttm_pool_free_range(restore->pool, tt, ttm_cached, + mid, restore->alloced_pages); + kfree(restore); + tt->restore = NULL; + } + + ttm_pool_free_range(NULL, tt, ttm_cached, start_page, tt->num_pages); +} + +/** + * ttm_pool_backup() - Back up or purge a struct ttm_tt + * @pool: The pool used when allocating the struct ttm_tt. + * @tt: The struct ttm_tt. + * @flags: Flags to govern the backup behaviour. + * + * Back up or purge a struct ttm_tt. If @purge is true, then + * all pages will be freed directly to the system rather than to the pool + * they were allocated from, making the function behave similarly to + * ttm_pool_free(). If @purge is false the pages will be backed up instead, + * exchanged for handles. + * A subsequent call to ttm_pool_restore_and_alloc() will then read back the content and + * a subsequent call to ttm_pool_drop_backed_up() will drop it. + * If backup of a page fails for whatever reason, @ttm will still be + * partially backed up, retaining those pages for which backup fails. + * In that case, this function can be retried, possibly after freeing up + * memory resources. + * + * Return: Number of pages actually backed up or freed, or negative + * error code on error. + */ +long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_backup_flags *flags) +{ + struct file *backup = tt->backup; + struct page *page; + gfp_t alloc_gfp; + gfp_t gfp; + int ret = 0; + pgoff_t shrunken = 0; + pgoff_t i, j, num_pages, npages; + pgoff_t nr_backed; + + if (WARN_ON(ttm_tt_is_backed_up(tt))) + return -EINVAL; + + if ((!ttm_backup_bytes_avail() && !flags->purge) || + ttm_pool_uses_dma_alloc(pool) || ttm_tt_is_backed_up(tt)) + return -EBUSY; + +#ifdef CONFIG_X86 + /* Anything returned to the system needs to be cached. Walk allocations + * skipping NULL pages and issue set_pages_array_wb() per contiguous run. + */ + if (tt->caching != ttm_cached) { + pgoff_t run_start = 0, run_count = 0; + + for (i = 0; i < tt->num_pages; i += num_pages) { + page = tt->pages[i]; + if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) { + if (run_count) { + set_pages_array_wb(&tt->pages[run_start], + run_count); + run_count = 0; + } + num_pages = 1; + continue; + } + num_pages = 1UL << ttm_pool_page_order(pool, page); + if (!run_count) + run_start = i; + run_count += num_pages; + } + if (run_count) + set_pages_array_wb(&tt->pages[run_start], run_count); + } +#endif + + if (tt->dma_address || flags->purge) { + for (i = 0; i < tt->num_pages; i += num_pages) { + unsigned int order; + + page = tt->pages[i]; + if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) { + num_pages = 1; + continue; + } + + order = ttm_pool_page_order(pool, page); + num_pages = 1UL << order; + if (tt->dma_address) + ttm_pool_unmap(pool, tt->dma_address[i], + num_pages); + if (flags->purge) { + shrunken += num_pages; + page->private = 0; + __free_pages_gpu_account(page, order, false); + memset(tt->pages + i, 0, + num_pages * sizeof(*tt->pages)); + } + } + } + + if (flags->purge) + return shrunken; + + if (ttm_pool_uses_dma32(pool)) + gfp = GFP_DMA32; + else + gfp = GFP_HIGHUSER; + + alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL; + + num_pages = tt->num_pages; + + /* Pretend doing fault injection by shrinking only half of the pages. */ + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(&backup_fault_inject, 1)) + num_pages = DIV_ROUND_UP(num_pages, 2); + + for (i = 0; i < num_pages; i += npages) { + unsigned int order; + s64 handle; + + npages = 1; + page = tt->pages[i]; + if (unlikely(!page)) + continue; + + /* Already-handled entry from a previous attempt. */ + if (unlikely(ttm_backup_page_ptr_is_handle(page))) + continue; + + order = ttm_pool_page_order(pool, page); + npages = 1UL << order; + + /* + * We don't allow dipping kernel reserves for high order backup + */ + if (order) + alloc_gfp |= __GFP_NOMEMALLOC; + else + alloc_gfp &= ~__GFP_NOMEMALLOC; + + /* + * Back up the compound atomically at its native order. If + * fault injection truncated num_pages mid-compound, skip + * the partial tail rather than splitting. + */ + if (unlikely(i + npages > num_pages)) + break; + + handle = ttm_backup_backup_folio(backup, page_folio(page), + order, flags->writeback, i, + gfp, alloc_gfp, + &nr_backed); + /* + * Zero progress on this compound (whether order 0 or a + * high-order compound that failed before backing up even + * its first subpage) is unrecoverable: bail out rather than + * looping forever with npages == nr_backed == 0 below. + */ + if (unlikely(handle < 0 && !nr_backed)) { + ret = handle; + break; + } + + for (j = 0; j < nr_backed; j++) + tt->pages[i + j] = ttm_backup_handle_to_page_ptr(handle + j); + + shrunken += nr_backed; + + if (unlikely(nr_backed < npages)) { + /* + * Partial OOM backup: split the compound and free the + * subpages whose content is now in shmem. Continue the + * loop from the first un-backed order-0 page. + */ + ttm_pool_split_for_swap(pool, page); + for (j = 0; j < nr_backed; j++) + __free_pages_gpu_account(page + j, 0, false); + npages = nr_backed; + continue; + } + + /* Fully backed up: free at native order. */ + page->private = 0; + __free_pages_gpu_account(page, order, false); + } + + return shrunken ? shrunken : ret; +} + +/** + * ttm_pool_init - Initialize a pool + * + * @pool: the pool to initialize + * @dev: device for DMA allocations and mappings + * @nid: NUMA node to use for allocations + * @alloc_flags: TTM_ALLOCATION_POOL_* flags + * + * Initialize the pool and its pool types. + */ +void ttm_pool_init(struct ttm_pool *pool, struct device *dev, + int nid, unsigned int alloc_flags) +{ + unsigned int i, j; + + WARN_ON(!dev && ttm_pool_uses_dma_alloc(pool)); + + pool->dev = dev; + pool->nid = nid; + pool->alloc_flags = alloc_flags; + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + for (j = 0; j < NR_PAGE_ORDERS; ++j) { + struct ttm_pool_type *pt; + + /* Initialize only pool types which are actually used */ + pt = ttm_pool_select_type(pool, i, j); + if (pt != &pool->caching[i].orders[j]) + continue; + + ttm_pool_type_init(pt, pool, i, j); + } + } +} +EXPORT_SYMBOL(ttm_pool_init); + +/** + * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete. + * + * This is useful to guarantee that all shrinker invocations have seen an + * update, before freeing memory, similar to rcu. + */ +static void ttm_pool_synchronize_shrinkers(void) +{ + down_write(&pool_shrink_rwsem); + up_write(&pool_shrink_rwsem); +} + +/** + * ttm_pool_fini - Cleanup a pool + * + * @pool: the pool to clean up + * + * Free all pages in the pool and unregister the types from the global + * shrinker. + */ +void ttm_pool_fini(struct ttm_pool *pool) +{ + unsigned int i, j; + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + for (j = 0; j < NR_PAGE_ORDERS; ++j) { + struct ttm_pool_type *pt; + + pt = ttm_pool_select_type(pool, i, j); + if (pt != &pool->caching[i].orders[j]) + continue; + + ttm_pool_type_fini(pt); + } + } + + /* We removed the pool types from the LRU, but we need to also make sure + * that no shrinker is concurrently freeing pages from the pool. + */ + ttm_pool_synchronize_shrinkers(); +} +EXPORT_SYMBOL(ttm_pool_fini); + +/* Free average pool number of pages. */ +#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS) + +static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_freed = 0; + + do + num_freed += ttm_pool_shrink(sc->nid, sc->nr_to_scan); + while (num_freed < sc->nr_to_scan && + atomic_long_read(&allocated_pages[sc->nid])); + + sc->nr_scanned = num_freed; + + return num_freed ?: SHRINK_STOP; +} + +/* Return the number of pages available or SHRINK_EMPTY if we have none */ +static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_pages = atomic_long_read(&allocated_pages[sc->nid]); + + return num_pages ? num_pages : SHRINK_EMPTY; +} + +#ifdef CONFIG_DEBUG_FS +/* Count the number of pages available in a pool_type */ +static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) +{ + return list_lru_count(&pt->pages); +} + +/* Print a nice header for the order */ +static void ttm_pool_debugfs_header(struct seq_file *m) +{ + unsigned int i; + + seq_puts(m, "\t "); + for (i = 0; i < NR_PAGE_ORDERS; ++i) + seq_printf(m, " ---%2u---", i); + seq_puts(m, "\n"); +} + +/* Dump information about the different pool types */ +static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, + struct seq_file *m) +{ + unsigned int i; + + for (i = 0; i < NR_PAGE_ORDERS; ++i) + seq_printf(m, " %8u", ttm_pool_type_count(&pt[i])); + seq_puts(m, "\n"); +} + +/* Dump the total amount of allocated pages */ +static void ttm_pool_debugfs_footer(struct seq_file *m) +{ + int nid; + + for_each_node(nid) { + seq_printf(m, "\ntotal node%d\t: %8lu of %8lu\n", nid, + atomic_long_read(&allocated_pages[nid]), pool_node_limit[nid]); + } +} + +/* Dump the information for the global pools */ +static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data) +{ + ttm_pool_debugfs_header(m); + + spin_lock(&shrinker_lock); + seq_puts(m, "wc\t:"); + ttm_pool_debugfs_orders(global_write_combined, m); + seq_puts(m, "uc\t:"); + ttm_pool_debugfs_orders(global_uncached, m); + seq_puts(m, "wc 32\t:"); + ttm_pool_debugfs_orders(global_dma32_write_combined, m); + seq_puts(m, "uc 32\t:"); + ttm_pool_debugfs_orders(global_dma32_uncached, m); + spin_unlock(&shrinker_lock); + + ttm_pool_debugfs_footer(m); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals); + +/** + * ttm_pool_debugfs - Debugfs dump function for a pool + * + * @pool: the pool to dump the information for + * @m: seq_file to dump to + * + * Make a debugfs dump with the per pool and global information. + */ +int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) +{ + unsigned int i; + + if (!ttm_pool_uses_dma_alloc(pool)) { + seq_puts(m, "unused\n"); + return 0; + } + + ttm_pool_debugfs_header(m); + + spin_lock(&shrinker_lock); + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + if (!ttm_pool_select_type(pool, i, 0)) + continue; + seq_puts(m, "DMA "); + switch (i) { + case ttm_cached: + seq_puts(m, "\t:"); + break; + case ttm_write_combined: + seq_puts(m, "wc\t:"); + break; + case ttm_uncached: + seq_puts(m, "uc\t:"); + break; + } + ttm_pool_debugfs_orders(pool->caching[i].orders, m); + } + spin_unlock(&shrinker_lock); + + ttm_pool_debugfs_footer(m); + return 0; +} +EXPORT_SYMBOL(ttm_pool_debugfs); + +/* Test the shrinker functions and dump the result */ +static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) +{ + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = TTM_SHRINKER_BATCH, + }; + unsigned long count, scanned; + int nid; + + fs_reclaim_acquire(GFP_KERNEL); + for_each_node(nid) { + sc.nid = nid; + count = ttm_pool_shrinker_count(mm_shrinker, &sc); + scanned = ttm_pool_shrinker_scan(mm_shrinker, &sc); + + /* Convert shrinker API sentinel values to 0 for debugfs output */ + if (count == SHRINK_EMPTY) + count = 0; + if (scanned == SHRINK_STOP) + scanned = 0; + + seq_printf(m, "%d: %lu/%lu\n", nid, count, scanned); + } + fs_reclaim_release(GFP_KERNEL); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); + +#endif + +static inline u64 ttm_get_node_memory_size(int nid) +{ + /* + * This is directly using si_meminfo_node implementation as the + * function is not exported. + */ + int zone_type; + u64 managed_pages = 0; + + pg_data_t *pgdat = NODE_DATA(nid); + + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) + managed_pages += + zone_managed_pages(&pgdat->node_zones[zone_type]); + return managed_pages * PAGE_SIZE; +} + +/** + * ttm_pool_mgr_init - Initialize globals + * + * @num_pages: default number of pages + * + * Initialize the global locks and lists for the MM shrinker. + */ +int ttm_pool_mgr_init(unsigned long num_pages) +{ + unsigned int i; + + int nid; + for_each_node(nid) { + if (!page_pool_size) { + u64 node_size = ttm_get_node_memory_size(nid); + pool_node_limit[nid] = (node_size >> PAGE_SHIFT) / 2; + } else { + pool_node_limit[nid] = page_pool_size; + } + } + + spin_lock_init(&shrinker_lock); + INIT_LIST_HEAD(&shrinker_list); + + for (i = 0; i < NR_PAGE_ORDERS; ++i) { + ttm_pool_type_init(&global_write_combined[i], NULL, + ttm_write_combined, i); + ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i); + + ttm_pool_type_init(&global_dma32_write_combined[i], NULL, + ttm_write_combined, i); + ttm_pool_type_init(&global_dma32_uncached[i], NULL, + ttm_uncached, i); + } + +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL, + &ttm_pool_debugfs_globals_fops); + debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL, + &ttm_pool_debugfs_shrink_fops); +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("backup_fault_inject", ttm_debugfs_root, + &backup_fault_inject); +#endif +#endif + + mm_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE, "drm-ttm_pool"); + if (!mm_shrinker) + return -ENOMEM; + + mm_shrinker->count_objects = ttm_pool_shrinker_count; + mm_shrinker->scan_objects = ttm_pool_shrinker_scan; + mm_shrinker->batch = TTM_SHRINKER_BATCH; + mm_shrinker->seeks = 1; + + shrinker_register(mm_shrinker); + + return 0; +} + +/** + * ttm_pool_mgr_fini - Finalize globals + * + * Cleanup the global pools and unregister the MM shrinker. + */ +void ttm_pool_mgr_fini(void) +{ + unsigned int i; + + for (i = 0; i < NR_PAGE_ORDERS; ++i) { + ttm_pool_type_fini(&global_write_combined[i]); + ttm_pool_type_fini(&global_uncached[i]); + + ttm_pool_type_fini(&global_dma32_write_combined[i]); + ttm_pool_type_fini(&global_dma32_uncached[i]); + } + + shrinker_free(mm_shrinker); + WARN_ON(!list_empty(&shrinker_list)); +} diff --git a/drivers/gpu/drm/ttm/ttm_pool_internal.h b/drivers/gpu/drm/ttm/ttm_pool_internal.h new file mode 100644 index 000000000..cbb17a212 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_pool_internal.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* Copyright (c) 2025 Valve Corporation */ + +#ifndef _TTM_POOL_INTERNAL_H_ +#define _TTM_POOL_INTERNAL_H_ + +#include +#include + +static inline bool ttm_pool_uses_dma_alloc(struct ttm_pool *pool) +{ + return pool->alloc_flags & TTM_ALLOCATION_POOL_USE_DMA_ALLOC; +} + +static inline bool ttm_pool_uses_dma32(struct ttm_pool *pool) +{ + return pool->alloc_flags & TTM_ALLOCATION_POOL_USE_DMA32; +} + +static inline unsigned int ttm_pool_beneficial_order(struct ttm_pool *pool) +{ + return pool->alloc_flags & 0xff; +} + +/* + * Implemented in ttm_pool.c, used by ttm_backup.c. Returns true if a fault + * should be injected mid-compound to test the reactive split-and-retry + * fallback in ttm_pool_backup(). Always returns false when + * CONFIG_FAULT_INJECTION is disabled. + */ +bool ttm_backup_fault_inject_folio(void); + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c new file mode 100644 index 000000000..b818836f0 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -0,0 +1,242 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Currently we use a spinlock for the lock, but a mutex *may* be + * more appropriate to reduce scheduling latency if the range manager + * ends up with very fragmented allocation patterns. + */ + +struct ttm_range_manager { + struct ttm_resource_manager manager; + struct drm_mm mm; + spinlock_t lock; +}; + +static inline struct ttm_range_manager * +to_range_manager(struct ttm_resource_manager *man) +{ + return container_of(man, struct ttm_range_manager, manager); +} + +static int ttm_range_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct ttm_range_manager *rman = to_range_manager(man); + struct ttm_range_mgr_node *node; + struct drm_mm *mm = &rman->mm; + enum drm_mm_insert_mode mode; + unsigned long lpfn; + int ret; + + lpfn = place->lpfn; + if (!lpfn) + lpfn = man->size; + + node = kzalloc_flex(*node, mm_nodes, 1); + if (!node) + return -ENOMEM; + + mode = DRM_MM_INSERT_BEST; + if (place->flags & TTM_PL_FLAG_TOPDOWN) + mode = DRM_MM_INSERT_HIGH; + + ttm_resource_init(bo, place, &node->base); + + spin_lock(&rman->lock); + ret = drm_mm_insert_node_in_range(mm, &node->mm_nodes[0], + PFN_UP(node->base.size), + bo->page_alignment, 0, + place->fpfn, lpfn, mode); + spin_unlock(&rman->lock); + + if (unlikely(ret)) { + ttm_resource_fini(man, &node->base); + kfree(node); + return ret; + } + + node->base.start = node->mm_nodes[0].start; + *res = &node->base; + return 0; +} + +static void ttm_range_man_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res); + struct ttm_range_manager *rman = to_range_manager(man); + + spin_lock(&rman->lock); + drm_mm_remove_node(&node->mm_nodes[0]); + spin_unlock(&rman->lock); + + ttm_resource_fini(man, res); + kfree(node); +} + +static bool ttm_range_man_intersects(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct drm_mm_node *node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + u32 num_pages = PFN_UP(size); + + /* Don't evict BOs outside of the requested placement range */ + if (place->fpfn >= (node->start + num_pages) || + (place->lpfn && place->lpfn <= node->start)) + return false; + + return true; +} + +static bool ttm_range_man_compatible(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct drm_mm_node *node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + u32 num_pages = PFN_UP(size); + + if (node->start < place->fpfn || + (place->lpfn && (node->start + num_pages) > place->lpfn)) + return false; + + return true; +} + +static void ttm_range_man_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct ttm_range_manager *rman = to_range_manager(man); + + spin_lock(&rman->lock); + drm_mm_print(&rman->mm, printer); + spin_unlock(&rman->lock); +} + +static const struct ttm_resource_manager_func ttm_range_manager_func = { + .alloc = ttm_range_man_alloc, + .free = ttm_range_man_free, + .intersects = ttm_range_man_intersects, + .compatible = ttm_range_man_compatible, + .debug = ttm_range_man_debug +}; + +/** + * ttm_range_man_init_nocheck - Initialise a generic range manager for the + * selected memory type. + * + * @bdev: ttm device + * @type: memory manager type + * @use_tt: if the memory manager uses tt + * @p_size: size of area to be managed in pages. + * + * The range manager is installed for this device in the type slot. + * + * Return: %0 on success or a negative error code on failure + */ +int ttm_range_man_init_nocheck(struct ttm_device *bdev, + unsigned type, bool use_tt, + unsigned long p_size) +{ + struct ttm_resource_manager *man; + struct ttm_range_manager *rman; + + rman = kzalloc_obj(*rman); + if (!rman) + return -ENOMEM; + + man = &rman->manager; + man->use_tt = use_tt; + + man->func = &ttm_range_manager_func; + + ttm_resource_manager_init(man, bdev, p_size); + + drm_mm_init(&rman->mm, 0, p_size); + spin_lock_init(&rman->lock); + + ttm_set_driver_manager(bdev, type, &rman->manager); + ttm_resource_manager_set_used(man, true); + return 0; +} +EXPORT_SYMBOL(ttm_range_man_init_nocheck); + +/** + * ttm_range_man_fini_nocheck - Remove the generic range manager from a slot + * and tear it down. + * + * @bdev: ttm device + * @type: memory manager type + * + * Return: %0 on success or a negative error code on failure + */ +int ttm_range_man_fini_nocheck(struct ttm_device *bdev, + unsigned type) +{ + struct ttm_resource_manager *man = ttm_manager_type(bdev, type); + struct ttm_range_manager *rman = to_range_manager(man); + struct drm_mm *mm = &rman->mm; + int ret; + + if (!man) + return 0; + + ttm_resource_manager_set_used(man, false); + + ret = ttm_resource_manager_evict_all(bdev, man); + if (ret) + return ret; + + spin_lock(&rman->lock); + drm_mm_takedown(mm); + spin_unlock(&rman->lock); + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(bdev, type, NULL); + kfree(rman); + return 0; +} +EXPORT_SYMBOL(ttm_range_man_fini_nocheck); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c new file mode 100644 index 000000000..4e6d3d658 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -0,0 +1,1026 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +/* Detach the cursor from the bulk move list */ +static void +ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor) +{ + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + cursor->bulk = NULL; + list_del_init(&cursor->bulk_link); +} + +/* Move the cursor to the end of the bulk move list it's in */ +static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk, + struct ttm_resource_cursor *cursor) +{ + struct ttm_lru_bulk_move_pos *pos; + + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + if (WARN_ON_ONCE(bulk != cursor->bulk)) { + list_del_init(&cursor->bulk_link); + return; + } + + pos = &bulk->pos[cursor->mem_type][cursor->priority]; + if (pos->last) + list_move(&cursor->hitch.link, &pos->last->lru.link); + ttm_resource_cursor_clear_bulk(cursor); +} + +/* Move all cursors attached to a bulk move to its end */ +static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk) +{ + struct ttm_resource_cursor *cursor, *next; + + list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link) + ttm_resource_cursor_move_bulk_tail(bulk, cursor); +} + +/* Remove a cursor from an empty bulk move list */ +static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk) +{ + struct ttm_resource_cursor *cursor, *next; + + list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link) + ttm_resource_cursor_clear_bulk(cursor); +} + +/** + * ttm_resource_cursor_init() - Initialize a struct ttm_resource_cursor + * @cursor: The cursor to initialize. + * @man: The resource manager. + * + * Initialize the cursor before using it for iteration. + */ +void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, + struct ttm_resource_manager *man) +{ + cursor->priority = 0; + cursor->man = man; + ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH); + INIT_LIST_HEAD(&cursor->bulk_link); + INIT_LIST_HEAD(&cursor->hitch.link); +} + +/** + * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage + * @cursor: The struct ttm_resource_cursor to finalize. + * + * The function pulls the LRU list cursor off any lists it was previously + * attached to. Needs to be called with the LRU lock held. The function + * can be called multiple times after each other. + */ +void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor) +{ + lockdep_assert_held(&cursor->man->bdev->lru_lock); + list_del_init(&cursor->hitch.link); + ttm_resource_cursor_clear_bulk(cursor); +} + +/** + * ttm_lru_bulk_move_init - initialize a bulk move structure + * @bulk: the structure to init + * + * For now just memset the structure to zero. + */ +void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk) +{ + memset(bulk, 0, sizeof(*bulk)); + INIT_LIST_HEAD(&bulk->cursor_list); +} +EXPORT_SYMBOL(ttm_lru_bulk_move_init); + +/** + * ttm_lru_bulk_move_fini - finalize a bulk move structure + * @bdev: The struct ttm_device + * @bulk: the structure to finalize + * + * Sanity checks that bulk moves don't have any + * resources left and hence no cursors attached. + */ +void ttm_lru_bulk_move_fini(struct ttm_device *bdev, + struct ttm_lru_bulk_move *bulk) +{ + spin_lock(&bdev->lru_lock); + ttm_bulk_move_drop_cursors(bulk); + spin_unlock(&bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_lru_bulk_move_fini); + +/** + * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail. + * + * @bulk: bulk move structure + * + * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that + * resource order never changes. Should be called with &ttm_device.lru_lock held. + */ +void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk) +{ + unsigned i, j; + + ttm_bulk_move_adjust_cursors(bulk); + for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) { + for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) { + struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j]; + struct ttm_resource_manager *man; + + if (!pos->first) + continue; + + lockdep_assert_held(&pos->first->bo->bdev->lru_lock); + dma_resv_assert_held(pos->first->bo->base.resv); + dma_resv_assert_held(pos->last->bo->base.resv); + + man = ttm_manager_type(pos->first->bo->bdev, i); + list_bulk_move_tail(&man->lru[j], &pos->first->lru.link, + &pos->last->lru.link); + } + } +} +EXPORT_SYMBOL(ttm_lru_bulk_move_tail); + +/* Return the bulk move pos object for this resource */ +static struct ttm_lru_bulk_move_pos * +ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res) +{ + return &bulk->pos[res->mem_type][res->bo->priority]; +} + +/* Return the previous resource on the list (skip over non-resource list items) */ +static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur) +{ + struct ttm_lru_item *lru = &cur->lru; + + do { + lru = list_prev_entry(lru, link); + } while (!ttm_lru_item_is_res(lru)); + + return ttm_lru_item_to_res(lru); +} + +/* Return the next resource on the list (skip over non-resource list items) */ +static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur) +{ + struct ttm_lru_item *lru = &cur->lru; + + do { + lru = list_next_entry(lru, link); + } while (!ttm_lru_item_is_res(lru)); + + return ttm_lru_item_to_res(lru); +} + +/* Move the resource to the tail of the bulk move range */ +static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos, + struct ttm_resource *res) +{ + if (pos->last != res) { + if (pos->first == res) + pos->first = ttm_lru_next_res(res); + list_move(&res->lru.link, &pos->last->lru.link); + pos->last = res; + } +} + +/* Add the resource to a bulk_move cursor */ +static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk, + struct ttm_resource *res) +{ + struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); + + if (!pos->first) { + pos->first = res; + pos->last = res; + } else { + WARN_ON(pos->first->bo->base.resv != res->bo->base.resv); + ttm_lru_bulk_move_pos_tail(pos, res); + } +} + +/* Remove the resource from a bulk_move range */ +static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk, + struct ttm_resource *res) +{ + struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); + + if (unlikely(WARN_ON(!pos->first || !pos->last) || + (pos->first == res && pos->last == res))) { + pos->first = NULL; + pos->last = NULL; + } else if (pos->first == res) { + pos->first = ttm_lru_next_res(res); + } else if (pos->last == res) { + pos->last = ttm_lru_prev_res(res); + } else { + list_move(&res->lru.link, &pos->last->lru.link); + } +} + +static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_object *bo) +{ + /* + * Take care when creating a new resource for a bo, that it is not considered + * swapped if it's not the current resource for the bo and is thus logically + * associated with the ttm_tt. Think a VRAM resource created to move a + * swapped-out bo to VRAM. + */ + if (bo->resource != res || !bo->ttm) + return false; + + dma_resv_assert_held(bo->base.resv); + return ttm_tt_is_swapped(bo->ttm); +} + +static bool ttm_resource_unevictable(struct ttm_resource *res, struct ttm_buffer_object *bo) +{ + return bo->pin_count || ttm_resource_is_swapped(res, bo); +} + +/* Add the resource to a bulk move if the BO is configured for it */ +void ttm_resource_add_bulk_move(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + if (bo->bulk_move && !ttm_resource_unevictable(res, bo)) + ttm_lru_bulk_move_add(bo->bulk_move, res); +} + +/* Remove the resource from a bulk move if the BO is configured for it */ +void ttm_resource_del_bulk_move(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + if (bo->bulk_move && !ttm_resource_unevictable(res, bo)) + ttm_lru_bulk_move_del(bo->bulk_move, res); +} + +/* + * Remove a resource from its bulk_move, bypassing the unevictable check. + * Use only when the resource is known to still be tracked in the range despite + * the BO having just become unevictable; asserts that this is the case. + */ +void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + WARN_ON_ONCE(!ttm_resource_unevictable(res, bo)); + if (bo->bulk_move) + ttm_lru_bulk_move_del(bo->bulk_move, res); +} + +/* Move a resource to the LRU or bulk tail */ +void ttm_resource_move_to_lru_tail(struct ttm_resource *res) +{ + struct ttm_buffer_object *bo = res->bo; + struct ttm_device *bdev = bo->bdev; + + lockdep_assert_held(&bo->bdev->lru_lock); + + if (ttm_resource_unevictable(res, bo)) { + list_move_tail(&res->lru.link, &bdev->unevictable); + + } else if (bo->bulk_move) { + struct ttm_lru_bulk_move_pos *pos = + ttm_lru_bulk_move_pos(bo->bulk_move, res); + + ttm_lru_bulk_move_pos_tail(pos, res); + } else { + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, res->mem_type); + list_move_tail(&res->lru.link, &man->lru[bo->priority]); + } +} + +/** + * ttm_resource_init - resource object constructor + * @bo: buffer object this resource is allocated for + * @place: placement of the resource + * @res: the resource object to initialize + * + * Initialize a new resource object. Counterpart of ttm_resource_fini(). + */ +void ttm_resource_init(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *res) +{ + struct ttm_resource_manager *man; + + res->start = 0; + res->size = bo->base.size; + res->mem_type = place->mem_type; + res->placement = place->flags; + res->bus.addr = NULL; + res->bus.offset = 0; + res->bus.is_iomem = false; + res->bus.caching = ttm_cached; + res->bo = bo; + + man = ttm_manager_type(bo->bdev, place->mem_type); + spin_lock(&bo->bdev->lru_lock); + if (ttm_resource_unevictable(res, bo)) + list_add_tail(&res->lru.link, &bo->bdev->unevictable); + else + list_add_tail(&res->lru.link, &man->lru[bo->priority]); + man->usage += res->size; + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_resource_init); + +/** + * ttm_resource_fini - resource destructor + * @man: the resource manager this resource belongs to + * @res: the resource to clean up + * + * Should be used by resource manager backends to clean up the TTM resource + * objects before freeing the underlying structure. Makes sure the resource is + * removed from the LRU before destruction. + * Counterpart of ttm_resource_init(). + */ +void ttm_resource_fini(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct ttm_device *bdev = man->bdev; + + spin_lock(&bdev->lru_lock); + list_del_init(&res->lru.link); + man->usage -= res->size; + spin_unlock(&bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_resource_fini); + +/** + * ttm_resource_try_charge - charge a resource manager's cgroup pool + * @bo: buffer for which an allocation should be charged + * @place: where the allocation is attempted to be placed + * @ret_pool: on charge success, the pool that was charged + * @ret_limit_pool: on charge failure, the pool responsible for the failure + * + * Should be used to charge cgroups before attempting resource allocation. + * When charging succeeds, the value of ret_pool should be passed to + * ttm_resource_alloc. + * + * Returns: 0 on charge success, negative errno on failure. + */ +int ttm_resource_try_charge(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct dmem_cgroup_pool_state **ret_pool, + struct dmem_cgroup_pool_state **ret_limit_pool) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, place->mem_type); + + if (!man->cg) { + *ret_pool = NULL; + if (ret_limit_pool) + *ret_limit_pool = NULL; + return 0; + } + + return dmem_cgroup_try_charge(man->cg, bo->base.size, ret_pool, + ret_limit_pool); +} + +int ttm_resource_alloc(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res_ptr, + struct dmem_cgroup_pool_state *charge_pool) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, place->mem_type); + int ret; + + ret = man->func->alloc(man, bo, place, res_ptr); + if (ret) + return ret; + + (*res_ptr)->css = charge_pool; + + spin_lock(&bo->bdev->lru_lock); + ttm_resource_add_bulk_move(*res_ptr, bo); + spin_unlock(&bo->bdev->lru_lock); + return 0; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc); + +void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) +{ + struct ttm_resource_manager *man; + struct dmem_cgroup_pool_state *pool; + + if (!*res) + return; + + spin_lock(&bo->bdev->lru_lock); + ttm_resource_del_bulk_move(*res, bo); + spin_unlock(&bo->bdev->lru_lock); + + pool = (*res)->css; + man = ttm_manager_type(bo->bdev, (*res)->mem_type); + man->func->free(man, *res); + *res = NULL; + if (pool) + dmem_cgroup_uncharge(pool, bo->base.size); +} +EXPORT_SYMBOL(ttm_resource_free); + +/** + * ttm_resource_intersects - test for intersection + * + * @bdev: TTM device structure + * @res: The resource to test + * @place: The placement to test + * @size: How many bytes the new allocation needs. + * + * Test if @res intersects with @place and @size. Used for testing if evictions + * are valuable or not. + * + * Returns true if the res placement intersects with @place and @size. + */ +bool ttm_resource_intersects(struct ttm_device *bdev, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, res->mem_type); + if (!place || !man->func->intersects) + return true; + + return man->func->intersects(man, res, place, size); +} + +/** + * ttm_resource_compatible - check if resource is compatible with placement + * + * @res: the resource to check + * @placement: the placement to check against + * @evicting: true if the caller is doing evictions + * + * Returns true if the placement is compatible. + */ +bool ttm_resource_compatible(struct ttm_resource *res, + struct ttm_placement *placement, + bool evicting) +{ + struct ttm_buffer_object *bo = res->bo; + struct ttm_device *bdev = bo->bdev; + unsigned i; + + if (res->placement & TTM_PL_FLAG_TEMPORARY) + return false; + + for (i = 0; i < placement->num_placement; i++) { + const struct ttm_place *place = &placement->placement[i]; + struct ttm_resource_manager *man; + + if (res->mem_type != place->mem_type) + continue; + + if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) + continue; + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS && + !(res->placement & TTM_PL_FLAG_CONTIGUOUS)) + continue; + + man = ttm_manager_type(bdev, res->mem_type); + if (man->func->compatible && + !man->func->compatible(man, res, place, bo->base.size)) + continue; + + return true; + } + return false; +} + +void ttm_resource_set_bo(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + spin_lock(&bo->bdev->lru_lock); + res->bo = bo; + spin_unlock(&bo->bdev->lru_lock); +} + +/** + * ttm_resource_manager_init + * + * @man: memory manager object to init + * @bdev: ttm device this manager belongs to + * @size: size of managed resources in arbitrary units + * + * Initialize core parts of a manager object. + */ +void ttm_resource_manager_init(struct ttm_resource_manager *man, + struct ttm_device *bdev, + uint64_t size) +{ + unsigned i; + + man->bdev = bdev; + man->size = size; + man->usage = 0; + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&man->lru[i]); + spin_lock_init(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) + man->eviction_fences[i] = NULL; +} +EXPORT_SYMBOL(ttm_resource_manager_init); + +/* + * ttm_resource_manager_evict_all + * + * @bdev: device to use + * @man: manager to use + * + * Evict all the objects out of a memory manager until it is empty. + * Part of memory manager cleanup sequence. + */ +int ttm_resource_manager_evict_all(struct ttm_device *bdev, + struct ttm_resource_manager *man) +{ + struct ttm_operation_ctx ctx = { }; + struct dma_fence *fence; + int ret, i; + + do { + ret = ttm_bo_evict_first(bdev, man, &ctx); + cond_resched(); + } while (!ret); + + if (ret && ret != -ENOENT) + return ret; + + ret = 0; + + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (fence && !dma_fence_is_signaled(fence)) { + dma_fence_get(fence); + spin_unlock(&man->eviction_lock); + ret = dma_fence_wait(fence, false); + dma_fence_put(fence); + if (ret) + return ret; + spin_lock(&man->eviction_lock); + } + } + spin_unlock(&man->eviction_lock); + + return ret; +} +EXPORT_SYMBOL(ttm_resource_manager_evict_all); + +/** + * ttm_resource_manager_usage + * + * @man: A memory manager object. + * + * Return how many resources are currently used. + */ +uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man) +{ + uint64_t usage; + + if (WARN_ON_ONCE(!man->bdev)) + return 0; + + spin_lock(&man->bdev->lru_lock); + usage = man->usage; + spin_unlock(&man->bdev->lru_lock); + return usage; +} +EXPORT_SYMBOL(ttm_resource_manager_usage); + +/** + * ttm_resource_manager_debug + * + * @man: manager type to dump. + * @p: printer to use for debug. + */ +void ttm_resource_manager_debug(struct ttm_resource_manager *man, + struct drm_printer *p) +{ + drm_printf(p, " use_type: %d\n", man->use_type); + drm_printf(p, " use_tt: %d\n", man->use_tt); + drm_printf(p, " size: %llu\n", man->size); + drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man)); + if (man->func->debug) + man->func->debug(man, p); +} +EXPORT_SYMBOL(ttm_resource_manager_debug); + +static void +ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, + struct ttm_lru_item *next_lru) +{ + struct ttm_resource *next = ttm_lru_item_to_res(next_lru); + struct ttm_lru_bulk_move *bulk; + + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + bulk = next->bo->bulk_move; + + if (cursor->bulk != bulk) { + if (bulk) { + list_move_tail(&cursor->bulk_link, &bulk->cursor_list); + cursor->mem_type = next->mem_type; + } else { + list_del_init(&cursor->bulk_link); + } + cursor->bulk = bulk; + } +} + +/** + * ttm_resource_manager_first() - Start iterating over the resources + * of a resource manager + * @cursor: cursor to record the position + * + * Initializes the cursor and starts iterating. When done iterating, + * the caller must explicitly call ttm_resource_cursor_fini(). + * + * Return: The first resource from the resource manager. + */ +struct ttm_resource * +ttm_resource_manager_first(struct ttm_resource_cursor *cursor) +{ + struct ttm_resource_manager *man = cursor->man; + + if (WARN_ON_ONCE(!man)) + return NULL; + + lockdep_assert_held(&man->bdev->lru_lock); + + list_move(&cursor->hitch.link, &man->lru[cursor->priority]); + return ttm_resource_manager_next(cursor); +} + +/** + * ttm_resource_manager_next() - Continue iterating over the resource manager + * resources + * @cursor: cursor to record the position + * + * Return: the next resource from the resource manager. + */ +struct ttm_resource * +ttm_resource_manager_next(struct ttm_resource_cursor *cursor) +{ + struct ttm_resource_manager *man = cursor->man; + struct ttm_lru_item *lru; + + lockdep_assert_held(&man->bdev->lru_lock); + + for (;;) { + lru = &cursor->hitch; + list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) { + if (ttm_lru_item_is_res(lru)) { + ttm_resource_cursor_check_bulk(cursor, lru); + list_move(&cursor->hitch.link, &lru->link); + return ttm_lru_item_to_res(lru); + } + } + + if (++cursor->priority >= TTM_MAX_BO_PRIORITY) + break; + + list_move(&cursor->hitch.link, &man->lru[cursor->priority]); + ttm_resource_cursor_clear_bulk(cursor); + } + + return NULL; +} + +/** + * ttm_lru_first_res_or_null() - Return the first resource on an lru list + * @head: The list head of the lru list. + * + * Return: Pointer to the first resource on the lru list or NULL if + * there is none. + */ +struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head) +{ + struct ttm_lru_item *lru; + + list_for_each_entry(lru, head, link) { + if (ttm_lru_item_is_res(lru)) + return ttm_lru_item_to_res(lru); + } + + return NULL; +} + +static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter, + struct iosys_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_iomap *iter_io = + container_of(iter, typeof(*iter_io), base); + void __iomem *addr; + +retry: + while (i >= iter_io->cache.end) { + iter_io->cache.sg = iter_io->cache.sg ? + sg_next(iter_io->cache.sg) : iter_io->st->sgl; + iter_io->cache.i = iter_io->cache.end; + iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >> + PAGE_SHIFT; + iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) - + iter_io->start; + } + + if (i < iter_io->cache.i) { + iter_io->cache.end = 0; + iter_io->cache.sg = NULL; + goto retry; + } + + addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs + + (((resource_size_t)i - iter_io->cache.i) + << PAGE_SHIFT)); + iosys_map_set_vaddr_iomem(dmap, addr); +} + +static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter, + struct iosys_map *map) +{ + io_mapping_unmap_local(map->vaddr_iomem); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = { + .map_local = ttm_kmap_iter_iomap_map_local, + .unmap_local = ttm_kmap_iter_iomap_unmap_local, + .maps_tt = false, +}; + +/** + * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap + * @iter_io: The struct ttm_kmap_iter_iomap to initialize. + * @iomap: The struct io_mapping representing the underlying linear io_memory. + * @st: sg_table into @iomap, representing the memory of the struct + * ttm_resource. + * @start: Offset that needs to be subtracted from @st to make + * sg_dma_address(st->sgl) - @start == 0 for @iomap start. + * + * Return: Pointer to the embedded struct ttm_kmap_iter. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, + struct io_mapping *iomap, + struct sg_table *st, + resource_size_t start) +{ + iter_io->base.ops = &ttm_kmap_iter_io_ops; + iter_io->iomap = iomap; + iter_io->st = st; + iter_io->start = start; + memset(&iter_io->cache, 0, sizeof(iter_io->cache)); + + return &iter_io->base; +} +EXPORT_SYMBOL(ttm_kmap_iter_iomap_init); + +/** + * DOC: Linear io iterator + * + * This code should die in the not too near future. Best would be if we could + * make io-mapping use memremap for all io memory, and have memremap + * implement a kmap_local functionality. We could then strip a huge amount of + * code. These linear io iterators are implemented to mimic old functionality, + * and they don't use kmap_local semantics at all internally. Rather ioremap or + * friends, and at least on 32-bit they add global TLB flushes and points + * of failure. + */ + +static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter, + struct iosys_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_linear_io *iter_io = + container_of(iter, typeof(*iter_io), base); + + *dmap = iter_io->dmap; + iosys_map_incr(dmap, i * PAGE_SIZE); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = { + .map_local = ttm_kmap_iter_linear_io_map_local, + .maps_tt = false, +}; + +/** + * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory + * @iter_io: The iterator to initialize + * @bdev: The TTM device + * @mem: The ttm resource representing the iomap. + * + * This function is for internal TTM use only. It sets up a memcpy kmap iterator + * pointing at a linear chunk of io memory. + * + * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on + * failure. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, + struct ttm_device *bdev, + struct ttm_resource *mem) +{ + int ret; + + ret = ttm_mem_io_reserve(bdev, mem); + if (ret) + goto out_err; + if (!mem->bus.is_iomem) { + ret = -EINVAL; + goto out_io_free; + } + + if (mem->bus.addr) { + iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); + iter_io->needs_unmap = false; + } else { + iter_io->needs_unmap = true; + memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); + if (mem->bus.caching == ttm_write_combined) + iosys_map_set_vaddr_iomem(&iter_io->dmap, + ioremap_wc(mem->bus.offset, + mem->size)); + else if (mem->bus.caching == ttm_cached) + iosys_map_set_vaddr(&iter_io->dmap, + memremap(mem->bus.offset, mem->size, + MEMREMAP_WB | + MEMREMAP_WT | + MEMREMAP_WC)); + + /* If uncached requested or if mapping cached or wc failed */ + if (iosys_map_is_null(&iter_io->dmap)) + iosys_map_set_vaddr_iomem(&iter_io->dmap, + ioremap(mem->bus.offset, + mem->size)); + + if (iosys_map_is_null(&iter_io->dmap)) { + ret = -ENOMEM; + goto out_io_free; + } + } + + iter_io->base.ops = &ttm_kmap_iter_linear_io_ops; + return &iter_io->base; + +out_io_free: + ttm_mem_io_free(bdev, mem); +out_err: + return ERR_PTR(ret); +} + +/** + * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory + * @iter_io: The iterator to finalize + * @bdev: The TTM device + * @mem: The ttm resource representing the iomap. + * + * This function is for internal TTM use only. It cleans up a memcpy kmap + * iterator initialized by ttm_kmap_iter_linear_io_init. + */ +void +ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, + struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) { + if (iter_io->dmap.is_iomem) + iounmap(iter_io->dmap.vaddr_iomem); + else + memunmap(iter_io->dmap.vaddr); + } + + ttm_mem_io_free(bdev, mem); +} + +#if defined(CONFIG_DEBUG_FS) + +static int ttm_resource_manager_show(struct seq_file *m, void *unused) +{ + struct ttm_resource_manager *man = + (struct ttm_resource_manager *)m->private; + struct drm_printer p = drm_seq_file_printer(m); + ttm_resource_manager_debug(man, &p); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager); + +#endif + +/** + * ttm_resource_manager_create_debugfs - Create debugfs entry for specified + * resource manager. + * @man: The TTM resource manager for which the debugfs stats file to be created + * @parent: debugfs directory in which the file will reside + * @name: The filename to create. + * + * This function sets up a debugfs file that can be used to look + * at debug statistics of the specified ttm_resource_manager. + */ +void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, + struct dentry *parent, + const char *name) +{ +#if defined(CONFIG_DEBUG_FS) + debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops); +#endif +} +EXPORT_SYMBOL(ttm_resource_manager_create_debugfs); + +/** + * ttm_resource_manager_dmem_reclaim() - dmem cgroup reclaim callback for TTM + * resource managers. + * @pool: The dmem cgroup pool state for the cgroup being reclaimed. + * @target_bytes: Number of bytes to try to free. + * @priv: The &ttm_resource_manager pointer, passed as @init.reclaim_priv to + * dmem_cgroup_register_region(). + * + * Drivers should use this as the @reclaim member of their own + * &struct dmem_cgroup_ops, with the &ttm_resource_manager pointer as + * @init.reclaim_priv. + * + * Return: 0 if some memory was freed, -ENOSPC if nothing was freed, or + * another negative error code on fatal failure. + */ +int ttm_resource_manager_dmem_reclaim(struct dmem_cgroup_pool_state *pool, + u64 target_bytes, void *priv) +{ + struct ttm_resource_manager *man = priv; + struct ttm_operation_ctx ctx = { .interruptible = true }; + s64 freed; + + freed = ttm_bo_evict_cgroup(man->bdev, man, pool, target_bytes, &ctx); + if (freed < 0) + return freed; + + return freed > 0 ? 0 : -ENOSPC; +} +EXPORT_SYMBOL(ttm_resource_manager_dmem_reclaim); + +/** + * ttm_resource_manager_set_dmem_region() - Associate a dmem cgroup region with a + * resource manager. + * @man: The resource manager. + * @region: The dmem cgroup region to associate, may be NULL or IS_ERR(). + * + * When @region is valid, stores it in @man->cg so that TTM can look up the + * associated pool during charging and eviction-target selection. When + * @region is %NULL, clears @man->cg to detach the region before teardown. + * An IS_ERR() @region is ignored, leaving @man->cg unchanged. + * The reclaim callback must be wired up using ttm_resource_manager_dmem_reclaim() + * in the driver's own &struct dmem_cgroup_ops, with the manager pointer as + * @init.reclaim_priv. + */ +void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man, + struct dmem_cgroup_region *region) +{ + if (!IS_ERR(region)) + man->cg = region; +} +EXPORT_SYMBOL(ttm_resource_manager_set_dmem_region); diff --git a/drivers/gpu/drm/ttm/ttm_sys_manager.c b/drivers/gpu/drm/ttm/ttm_sys_manager.c new file mode 100644 index 000000000..04398bb9c --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_sys_manager.c @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +#include +#include +#include +#include + +#include "ttm_module.h" + +static int ttm_sys_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + *res = kzalloc_obj(**res); + if (!*res) + return -ENOMEM; + + ttm_resource_init(bo, place, *res); + return 0; +} + +static void ttm_sys_man_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + ttm_resource_fini(man, res); + kfree(res); +} + +static const struct ttm_resource_manager_func ttm_sys_manager_func = { + .alloc = ttm_sys_man_alloc, + .free = ttm_sys_man_free, +}; + +void ttm_sys_man_init(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man = &bdev->sysman; + + /* + * Initialize the system memory buffer type. + * Other types need to be driver / IOCTL initialized. + */ + man->use_tt = true; + man->func = &ttm_sys_manager_func; + + ttm_resource_manager_init(man, bdev, 0); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); + ttm_resource_manager_set_used(man, true); +} diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c new file mode 100644 index 000000000..b645a1818 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -0,0 +1,565 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ttm_module.h" +#include "ttm_pool_internal.h" + +static unsigned long ttm_pages_limit; + +MODULE_PARM_DESC(pages_limit, "Limit for the allocated pages"); +module_param_named(pages_limit, ttm_pages_limit, ulong, 0644); + +static unsigned long ttm_dma32_pages_limit; + +MODULE_PARM_DESC(dma32_pages_limit, "Limit for the allocated DMA32 pages"); +module_param_named(dma32_pages_limit, ttm_dma32_pages_limit, ulong, 0644); + +static atomic_long_t ttm_pages_allocated; +static atomic_long_t ttm_dma32_pages_allocated; + +/* + * Allocates a ttm structure for the given BO. + */ +int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) +{ + struct ttm_device *bdev = bo->bdev; + struct drm_device *ddev = bo->base.dev; + uint32_t page_flags = 0; + + dma_resv_assert_held(bo->base.resv); + + if (bo->ttm) + return 0; + + switch (bo->type) { + case ttm_bo_type_device: + if (zero_alloc) + page_flags |= TTM_TT_FLAG_ZERO_ALLOC; + break; + case ttm_bo_type_kernel: + break; + case ttm_bo_type_sg: + page_flags |= TTM_TT_FLAG_EXTERNAL; + break; + default: + pr_err("Illegal buffer object type\n"); + return -EINVAL; + } + /* + * When using dma_alloc_coherent with memory encryption the + * mapped TT pages need to be decrypted or otherwise the drivers + * will end up sending encrypted mem to the gpu. + */ + if (ttm_pool_uses_dma_alloc(&bdev->pool) && + cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { + page_flags |= TTM_TT_FLAG_DECRYPTED; + drm_info_once(ddev, "TT memory decryption enabled."); + } + + bo->ttm = bdev->funcs->ttm_tt_create(bo, page_flags); + if (unlikely(bo->ttm == NULL)) + return -ENOMEM; + + WARN_ON(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE && + !(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)); + + return 0; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_create); + +/* + * Allocates storage for pointers to the pages that back the ttm. + */ +static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->pages = kvcalloc(ttm->num_pages, sizeof(void*), GFP_KERNEL); + if (!ttm->pages) + return -ENOMEM; + + return 0; +} + +static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->pages = kvcalloc(ttm->num_pages, sizeof(*ttm->pages) + + sizeof(*ttm->dma_address), GFP_KERNEL); + if (!ttm->pages) + return -ENOMEM; + + ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); + return 0; +} + +static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->dma_address = kvzalloc_objs(*ttm->dma_address, ttm->num_pages); + if (!ttm->dma_address) + return -ENOMEM; + + return 0; +} + +void ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) +{ + bdev->funcs->ttm_tt_destroy(bdev, ttm); +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_destroy); + +static void ttm_tt_init_fields(struct ttm_tt *ttm, + struct ttm_buffer_object *bo, + uint32_t page_flags, + enum ttm_caching caching, + unsigned long extra_pages) +{ + ttm->num_pages = (PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT) + extra_pages; + ttm->page_flags = page_flags; + ttm->dma_address = NULL; + ttm->swap_storage = NULL; + ttm->sg = bo->sg; + ttm->caching = caching; + ttm->restore = NULL; + ttm->backup = NULL; +} + +int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + uint32_t page_flags, enum ttm_caching caching, + unsigned long extra_pages) +{ + ttm_tt_init_fields(ttm, bo, page_flags, caching, extra_pages); + + if (ttm_tt_alloc_page_directory(ttm)) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_tt_init); + +void ttm_tt_fini(struct ttm_tt *ttm) +{ + WARN_ON(ttm->page_flags & TTM_TT_FLAG_PRIV_POPULATED); + + if (ttm->swap_storage) + fput(ttm->swap_storage); + ttm->swap_storage = NULL; + + if (ttm_tt_is_backed_up(ttm)) + ttm_pool_drop_backed_up(ttm); + if (ttm->backup) { + ttm_backup_fini(ttm->backup); + ttm->backup = NULL; + } + + if (ttm->pages) + kvfree(ttm->pages); + else + kvfree(ttm->dma_address); + ttm->pages = NULL; + ttm->dma_address = NULL; +} +EXPORT_SYMBOL(ttm_tt_fini); + +int ttm_sg_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + uint32_t page_flags, enum ttm_caching caching) +{ + int ret; + + ttm_tt_init_fields(ttm, bo, page_flags, caching, 0); + + if (page_flags & TTM_TT_FLAG_EXTERNAL) + ret = ttm_sg_tt_alloc_page_directory(ttm); + else + ret = ttm_dma_tt_alloc_page_directory(ttm); + if (ret) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_sg_tt_init); + +int ttm_tt_swapin(struct ttm_tt *ttm) +{ + struct address_space *swap_space; + struct file *swap_storage; + struct page *from_page; + struct page *to_page; + gfp_t gfp_mask; + int i, ret; + + swap_storage = ttm->swap_storage; + BUG_ON(swap_storage == NULL); + + swap_space = swap_storage->f_mapping; + gfp_mask = mapping_gfp_mask(swap_space); + + for (i = 0; i < ttm->num_pages; ++i) { + from_page = shmem_read_mapping_page_gfp(swap_space, i, + gfp_mask); + if (IS_ERR(from_page)) { + ret = PTR_ERR(from_page); + goto out_err; + } + to_page = ttm->pages[i]; + if (unlikely(to_page == NULL)) { + ret = -ENOMEM; + goto out_err; + } + + copy_highpage(to_page, from_page); + put_page(from_page); + } + + fput(swap_storage); + ttm->swap_storage = NULL; + ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; + + return 0; + +out_err: + return ret; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_swapin); + +/** + * ttm_tt_backup() - Helper to back up a struct ttm_tt. + * @bdev: The TTM device. + * @tt: The struct ttm_tt. + * @flags: Flags that govern the backup behaviour. + * + * Update the page accounting and call ttm_pool_shrink_tt to free pages + * or back them up. + * + * Return: Number of pages freed or swapped out, or negative error code on + * error. + */ +long ttm_tt_backup(struct ttm_device *bdev, struct ttm_tt *tt, + const struct ttm_backup_flags flags) +{ + long ret; + + if (WARN_ON(IS_ERR_OR_NULL(tt->backup))) + return 0; + + ret = ttm_pool_backup(&bdev->pool, tt, &flags); + if (ret > 0) { + tt->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED; + tt->page_flags |= TTM_TT_FLAG_BACKED_UP; + } + + return ret; +} + +int ttm_tt_restore(struct ttm_device *bdev, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx) +{ + int ret = ttm_pool_restore_and_alloc(&bdev->pool, tt, ctx); + + if (ret) + return ret; + + tt->page_flags &= ~TTM_TT_FLAG_BACKED_UP; + + return 0; +} +EXPORT_SYMBOL(ttm_tt_restore); + +/** + * ttm_tt_swapout - swap out tt object + * + * @bdev: TTM device structure. + * @ttm: The struct ttm_tt. + * @gfp_flags: Flags to use for memory allocation. + * + * Swapout a TT object to a shmem_file, return number of pages swapped out or + * negative error code. + */ +int ttm_tt_swapout(struct ttm_device *bdev, struct ttm_tt *ttm, + gfp_t gfp_flags) +{ + loff_t size = (loff_t)ttm->num_pages << PAGE_SHIFT; + struct address_space *swap_space; + struct file *swap_storage; + struct page *from_page; + struct page *to_page; + int i, ret; + + swap_storage = shmem_file_setup("ttm swap", size, EMPTY_VMA_FLAGS); + if (IS_ERR(swap_storage)) { + pr_err("Failed allocating swap storage\n"); + return PTR_ERR(swap_storage); + } + + swap_space = swap_storage->f_mapping; + gfp_flags &= mapping_gfp_mask(swap_space); + + for (i = 0; i < ttm->num_pages; ++i) { + from_page = ttm->pages[i]; + if (unlikely(from_page == NULL)) + continue; + + to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_flags); + if (IS_ERR(to_page)) { + ret = PTR_ERR(to_page); + goto out_err; + } + copy_highpage(to_page, from_page); + set_page_dirty(to_page); + mark_page_accessed(to_page); + put_page(to_page); + } + + ttm_tt_unpopulate(bdev, ttm); + ttm->swap_storage = swap_storage; + ttm->page_flags |= TTM_TT_FLAG_SWAPPED; + + return ttm->num_pages; + +out_err: + fput(swap_storage); + + return ret; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_swapout); + +int ttm_tt_populate(struct ttm_device *bdev, + struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) +{ + int ret; + + if (!ttm) + return -EINVAL; + + if (ttm_tt_is_populated(ttm)) + return 0; + + if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + atomic_long_add(ttm->num_pages, &ttm_pages_allocated); + if (ttm_pool_uses_dma32(&bdev->pool)) + atomic_long_add(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + + while (atomic_long_read(&ttm_pages_allocated) > ttm_pages_limit || + atomic_long_read(&ttm_dma32_pages_allocated) > + ttm_dma32_pages_limit) { + + ret = ttm_global_swapout(ctx, GFP_KERNEL); + if (ret == 0) + break; + if (ret < 0) + goto error; + } + + if (bdev->funcs->ttm_tt_populate) + ret = bdev->funcs->ttm_tt_populate(bdev, ttm, ctx); + else + ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); + if (ret) + goto error; + + ttm->page_flags |= TTM_TT_FLAG_PRIV_POPULATED; + ttm->page_flags &= ~TTM_TT_FLAG_BACKED_UP; + if (unlikely(ttm->page_flags & TTM_TT_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_tt_unpopulate(bdev, ttm); + return ret; + } + } + + return 0; + +error: + if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); + if (ttm_pool_uses_dma32(&bdev->pool)) + atomic_long_sub(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + return ret; +} + +#if IS_ENABLED(CONFIG_DRM_TTM_KUNIT_TEST) +EXPORT_SYMBOL(ttm_tt_populate); +#endif + +void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) +{ + if (!ttm_tt_is_populated(ttm)) + return; + + if (bdev->funcs->ttm_tt_unpopulate) + bdev->funcs->ttm_tt_unpopulate(bdev, ttm); + else + ttm_pool_free(&bdev->pool, ttm); + + if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); + if (ttm_pool_uses_dma32(&bdev->pool)) + atomic_long_sub(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + + ttm->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_unpopulate); + +#ifdef CONFIG_DEBUG_FS + +/* Test the shrinker functions and dump the result */ +static int ttm_tt_debugfs_shrink_show(struct seq_file *m, void *data) +{ + struct ttm_operation_ctx ctx = { }; + + seq_printf(m, "%d\n", ttm_global_swapout(&ctx, GFP_KERNEL)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_tt_debugfs_shrink); + +#endif + + +/* + * ttm_tt_mgr_init - register with the MM shrinker + * + * Register with the MM shrinker for swapping out BOs. + */ +void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages) +{ +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("tt_shrink", 0400, ttm_debugfs_root, NULL, + &ttm_tt_debugfs_shrink_fops); +#endif + + if (!ttm_pages_limit) + ttm_pages_limit = num_pages; + + if (!ttm_dma32_pages_limit) + ttm_dma32_pages_limit = num_dma32_pages; +} + +static void ttm_kmap_iter_tt_map_local(struct ttm_kmap_iter *iter, + struct iosys_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_tt *iter_tt = + container_of(iter, typeof(*iter_tt), base); + + iosys_map_set_vaddr(dmap, kmap_local_page_prot(iter_tt->tt->pages[i], + iter_tt->prot)); +} + +static void ttm_kmap_iter_tt_unmap_local(struct ttm_kmap_iter *iter, + struct iosys_map *map) +{ + kunmap_local(map->vaddr); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_tt_ops = { + .map_local = ttm_kmap_iter_tt_map_local, + .unmap_local = ttm_kmap_iter_tt_unmap_local, + .maps_tt = true, +}; + +/** + * ttm_kmap_iter_tt_init - Initialize a struct ttm_kmap_iter_tt + * @iter_tt: The struct ttm_kmap_iter_tt to initialize. + * @tt: Struct ttm_tt holding page pointers of the struct ttm_resource. + * + * Return: Pointer to the embedded struct ttm_kmap_iter. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt, + struct ttm_tt *tt) +{ + iter_tt->base.ops = &ttm_kmap_iter_tt_ops; + iter_tt->tt = tt; + if (tt) + iter_tt->prot = ttm_prot_from_caching(tt->caching, PAGE_KERNEL); + else + iter_tt->prot = PAGE_KERNEL; + + return &iter_tt->base; +} +EXPORT_SYMBOL(ttm_kmap_iter_tt_init); + +unsigned long ttm_tt_pages_limit(void) +{ + return ttm_pages_limit; +} +EXPORT_SYMBOL(ttm_tt_pages_limit); + +/** + * ttm_tt_setup_backup() - Allocate and assign a backup structure for a ttm_tt + * @tt: The ttm_tt for wich to allocate and assign a backup structure. + * + * Assign a backup structure to be used for tt backup. This should + * typically be done at bo creation, to avoid allocations at shrinking + * time. + * + * Return: 0 on success, negative error code on failure. + */ +int ttm_tt_setup_backup(struct ttm_tt *tt) +{ + struct file *backup = + ttm_backup_shmem_create(((loff_t)tt->num_pages) << PAGE_SHIFT); + + if (WARN_ON_ONCE(!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))) + return -EINVAL; + + if (IS_ERR(backup)) + return PTR_ERR(backup); + + if (tt->backup) + ttm_backup_fini(tt->backup); + + tt->backup = backup; + return 0; +} +EXPORT_SYMBOL(ttm_tt_setup_backup); -- cgit v1.3.1