diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
30 files changed, 11186 insertions, 0 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile new file mode 100644 index 000000000..40d07a352 --- /dev/null +++ b/drivers/gpu/drm/ttm/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the drm device driver. This driver provides support for the + +ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ + ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \ + ttm_device.o ttm_sys_manager.o ttm_backup.o +ttm-$(CONFIG_AGP) += ttm_agp_backend.o + +obj-$(CONFIG_DRM_TTM) += ttm.o +obj-$(CONFIG_DRM_TTM_KUNIT_TEST) += tests/ diff --git a/drivers/gpu/drm/ttm/tests/.kunitconfig b/drivers/gpu/drm/ttm/tests/.kunitconfig new file mode 100644 index 000000000..1ae1ffabd --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_DRM=y +CONFIG_DRM_TTM_KUNIT_TEST=y diff --git a/drivers/gpu/drm/ttm/tests/Makefile b/drivers/gpu/drm/ttm/tests/Makefile new file mode 100644 index 000000000..f3149de77 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 AND MIT + +obj-$(CONFIG_DRM_TTM_KUNIT_TEST) += \ + ttm_device_test.o \ + ttm_pool_test.o \ + ttm_resource_test.o \ + ttm_tt_test.o \ + ttm_bo_test.o \ + ttm_bo_validate_test.o \ + ttm_mock_manager.o \ + ttm_kunit_helpers.o diff --git a/drivers/gpu/drm/ttm/tests/TODO b/drivers/gpu/drm/ttm/tests/TODO new file mode 100644 index 000000000..45b03d184 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/TODO @@ -0,0 +1,27 @@ +TODO +===== + +- Add a test case where the only evictable BO is busy +- Update eviction tests so they use parametrized "from" memory type +- Improve mock manager's implementation, e.g. allocate a block of + dummy memory that can be used when testing page mapping functions +- Suggestion: Add test cases with external BOs +- Suggestion: randomize the number and size of tested buffers in + ttm_bo_validate() +- Agree on the naming convention +- Rewrite the mock manager: drop use_tt and manage mock memory using + drm_mm manager + +Notes and gotchas +================= + +- These tests are built and run with a UML kernel, because + 1) We are interested in hardware-independent testing + 2) We don't want to have actual DRM devices interacting with TTM + at the same time as the test one. Getting these to work in + parallel would require some time (...and that's a "todo" in itself!) +- Triggering ttm_bo_vm_ops callbacks from KUnit (i.e. kernel) might be + a challenge, but is worth trying. Look at selftests like + i915/gem/selftests/i915_gem_mman.c for inspiration +- The test suite uses UML where ioremap() call returns NULL, meaning that + ttm_bo_ioremap() can't be tested, unless we find a way to stub it diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c new file mode 100644 index 000000000..f3103307b --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -0,0 +1,637 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include <linux/dma-resv.h> +#include <linux/kthread.h> +#include <linux/delay.h> +#include <linux/timer.h> +#include <linux/jiffies.h> +#include <linux/mutex.h> +#include <linux/ww_mutex.h> + +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include "ttm_kunit_helpers.h" + +#define BO_SIZE SZ_8K + +#ifdef CONFIG_PREEMPT_RT +#define ww_mutex_base_lock(b) rt_mutex_lock(b) +#else +#define ww_mutex_base_lock(b) mutex_lock(b) +#endif + +struct ttm_bo_test_case { + const char *description; + bool interruptible; + bool no_wait; +}; + +static const struct ttm_bo_test_case ttm_bo_reserved_cases[] = { + { + .description = "Cannot be interrupted and sleeps", + .interruptible = false, + .no_wait = false, + }, + { + .description = "Cannot be interrupted, locks straight away", + .interruptible = false, + .no_wait = true, + }, + { + .description = "Can be interrupted, sleeps", + .interruptible = true, + .no_wait = false, + }, +}; + +static void ttm_bo_init_case_desc(const struct ttm_bo_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_bo_reserve, ttm_bo_reserved_cases, ttm_bo_init_case_desc); + +static void ttm_bo_reserve_optimistic_no_ticket(struct kunit *test) +{ + const struct ttm_bo_test_case *params = test->param_value; + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_bo_reserve(bo, params->interruptible, params->no_wait, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + + dma_resv_unlock(bo->base.resv); +} + +static void ttm_bo_reserve_locked_no_sleep(struct kunit *test) +{ + struct ttm_buffer_object *bo; + bool interruptible = false; + bool no_wait = true; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + /* Let's lock it beforehand */ + dma_resv_lock(bo->base.resv, NULL); + + err = ttm_bo_reserve(bo, interruptible, no_wait, NULL); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, -EBUSY); +} + +static void ttm_bo_reserve_no_wait_ticket(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ww_acquire_ctx ctx; + bool interruptible = false; + bool no_wait = true; + int err; + + ww_acquire_init(&ctx, &reservation_ww_class); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + KUNIT_ASSERT_EQ(test, err, -EBUSY); + + ww_acquire_fini(&ctx); +} + +static void ttm_bo_reserve_double_resv(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ww_acquire_ctx ctx; + bool interruptible = false; + bool no_wait = false; + int err; + + ww_acquire_init(&ctx, &reservation_ww_class); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + + dma_resv_unlock(bo->base.resv); + ww_acquire_fini(&ctx); + + KUNIT_ASSERT_EQ(test, err, -EALREADY); +} + +/* + * A test case heavily inspired by ww_test_edeadlk_normal(). It injects + * a deadlock by manipulating the sequence number of the context that holds + * dma_resv lock of bo2 so the other context is "wounded" and has to back off + * (indicated by -EDEADLK). The subtest checks if ttm_bo_reserve() properly + * propagates that error. + */ +static void ttm_bo_reserve_deadlock(struct kunit *test) +{ + struct ttm_buffer_object *bo1, *bo2; + struct ww_acquire_ctx ctx1, ctx2; + bool interruptible = false; + bool no_wait = false; + int err; + + bo1 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo2 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + ww_acquire_init(&ctx1, &reservation_ww_class); + ww_mutex_base_lock(&bo2->base.resv->lock.base); + + /* The deadlock will be caught by WW mutex, don't warn about it */ + lock_release(&bo2->base.resv->lock.base.dep_map, 1); + + bo2->base.resv->lock.ctx = &ctx2; + ctx2 = ctx1; + ctx2.stamp--; /* Make the context holding the lock younger */ + + err = ttm_bo_reserve(bo1, interruptible, no_wait, &ctx1); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_bo_reserve(bo2, interruptible, no_wait, &ctx1); + KUNIT_ASSERT_EQ(test, err, -EDEADLK); + + dma_resv_unlock(bo1->base.resv); + ww_acquire_fini(&ctx1); +} + +#if IS_BUILTIN(CONFIG_DRM_TTM_KUNIT_TEST) +struct signal_timer { + struct timer_list timer; + struct ww_acquire_ctx *ctx; +}; + +static void signal_for_ttm_bo_reserve(struct timer_list *t) +{ + struct signal_timer *s_timer = timer_container_of(s_timer, t, timer); + struct task_struct *task = s_timer->ctx->task; + + do_send_sig_info(SIGTERM, SEND_SIG_PRIV, task, PIDTYPE_PID); +} + +static int threaded_ttm_bo_reserve(void *arg) +{ + struct ttm_buffer_object *bo = arg; + struct signal_timer s_timer; + struct ww_acquire_ctx ctx; + bool interruptible = true; + bool no_wait = false; + int err; + + ww_acquire_init(&ctx, &reservation_ww_class); + + /* Prepare a signal that will interrupt the reservation attempt */ + timer_setup_on_stack(&s_timer.timer, &signal_for_ttm_bo_reserve, 0); + s_timer.ctx = &ctx; + + mod_timer(&s_timer.timer, msecs_to_jiffies(100)); + + err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); + + timer_delete_sync(&s_timer.timer); + timer_destroy_on_stack(&s_timer.timer); + + ww_acquire_fini(&ctx); + + return err; +} + +static void ttm_bo_reserve_interrupted(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct task_struct *task; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + task = kthread_create(threaded_ttm_bo_reserve, bo, "ttm-bo-reserve"); + + if (IS_ERR(task)) + KUNIT_FAIL(test, "Couldn't create ttm bo reserve task\n"); + + /* Take a lock so the threaded reserve has to wait */ + dma_resv_lock(bo->base.resv, NULL); + + wake_up_process(task); + msleep(20); + err = kthread_stop(task); + + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, -ERESTARTSYS); +} +#endif /* IS_BUILTIN(CONFIG_DRM_TTM_KUNIT_TEST) */ + +static void ttm_bo_unreserve_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_device *ttm_dev; + struct ttm_resource *res1, *res2; + struct ttm_place *place; + struct ttm_resource_manager *man; + unsigned int bo_prio = TTM_MAX_BO_PRIORITY - 1; + u32 mem_type = TTM_PL_SYSTEM; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->priority = bo_prio; + + err = ttm_resource_alloc(bo, place, &res1, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + + bo->resource = res1; + + /* Add a dummy resource to populate LRU */ + ttm_resource_alloc(bo, place, &res2, NULL); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unreserve(bo); + + man = ttm_manager_type(priv->ttm_dev, mem_type); + KUNIT_ASSERT_EQ(test, + list_is_last(&res1->lru.link, &man->lru[bo->priority]), 1); + + ttm_resource_free(bo, &res2); + ttm_resource_free(bo, &res1); +} + +static void ttm_bo_unreserve_pinned(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_device *ttm_dev; + struct ttm_resource *res1, *res2; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + place = ttm_place_kunit_init(test, mem_type, 0); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_pin(bo); + + err = ttm_resource_alloc(bo, place, &res1, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res1; + + /* Add a dummy resource to the pinned list */ + err = ttm_resource_alloc(bo, place, &res2, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, + list_is_last(&res2->lru.link, &priv->ttm_dev->unevictable), 1); + + ttm_bo_unreserve(bo); + KUNIT_ASSERT_EQ(test, + list_is_last(&res1->lru.link, &priv->ttm_dev->unevictable), 1); + + ttm_resource_free(bo, &res1); + ttm_resource_free(bo, &res2); +} + +static void ttm_bo_unreserve_bulk(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_lru_bulk_move lru_bulk_move; + struct ttm_lru_bulk_move_pos *pos; + struct ttm_buffer_object *bo1, *bo2; + struct ttm_resource *res1, *res2; + struct ttm_device *ttm_dev; + struct ttm_place *place; + struct dma_resv *resv; + u32 mem_type = TTM_PL_SYSTEM; + unsigned int bo_priority = 0; + int err; + + ttm_lru_bulk_move_init(&lru_bulk_move); + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + resv = kunit_kzalloc(test, sizeof(*resv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, resv); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + dma_resv_init(resv); + + bo1 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, resv); + bo2 = ttm_bo_kunit_init(test, test->priv, BO_SIZE, resv); + + dma_resv_lock(bo1->base.resv, NULL); + ttm_bo_set_bulk_move(bo1, &lru_bulk_move); + dma_resv_unlock(bo1->base.resv); + + err = ttm_resource_alloc(bo1, place, &res1, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo1->resource = res1; + + dma_resv_lock(bo2->base.resv, NULL); + ttm_bo_set_bulk_move(bo2, &lru_bulk_move); + dma_resv_unlock(bo2->base.resv); + + err = ttm_resource_alloc(bo2, place, &res2, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo2->resource = res2; + + ttm_bo_reserve(bo1, false, false, NULL); + ttm_bo_unreserve(bo1); + + pos = &lru_bulk_move.pos[mem_type][bo_priority]; + KUNIT_ASSERT_PTR_EQ(test, res1, pos->last); + + ttm_resource_free(bo1, &res1); + ttm_resource_free(bo2, &res2); + + dma_resv_fini(resv); +} + +static void ttm_bo_fini_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + struct ttm_device *ttm_dev; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_device; + + err = ttm_resource_alloc(bo, place, &res, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_bo_fini(bo); +} + +static const char *mock_name(struct dma_fence *f) +{ + return "kunit-ttm-bo-put"; +} + +static const struct dma_fence_ops mock_fence_ops = { + .get_driver_name = mock_name, + .get_timeline_name = mock_name, +}; + +static void ttm_bo_fini_shared_resv(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct dma_resv *external_resv; + struct dma_fence *fence; + /* A dummy DMA fence lock */ + spinlock_t fence_lock; + struct ttm_device *ttm_dev; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + external_resv = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, external_resv); + + dma_resv_init(external_resv); + + fence = kunit_kzalloc(test, sizeof(*fence), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fence); + + spin_lock_init(&fence_lock); + dma_fence_init(fence, &mock_fence_ops, &fence_lock, 0, 0); + + dma_resv_lock(external_resv, NULL); + dma_resv_reserve_fences(external_resv, 1); + dma_resv_add_fence(external_resv, fence, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_unlock(external_resv); + + dma_fence_signal(fence); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_device; + bo->base.resv = external_resv; + + ttm_bo_fini(bo); +} + +static void ttm_bo_pin_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_device *ttm_dev; + unsigned int no_pins = 3; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + for (int i = 0; i < no_pins; i++) { + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_pin(bo); + dma_resv_unlock(bo->base.resv); + } + + KUNIT_ASSERT_EQ(test, bo->pin_count, no_pins); +} + +static void ttm_bo_pin_unpin_resource(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_lru_bulk_move lru_bulk_move; + struct ttm_lru_bulk_move_pos *pos; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + struct ttm_device *ttm_dev; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + unsigned int bo_priority = 0; + int err; + + ttm_lru_bulk_move_init(&lru_bulk_move); + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_resource_alloc(bo, place, &res, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res; + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_set_bulk_move(bo, &lru_bulk_move); + ttm_bo_pin(bo); + dma_resv_unlock(bo->base.resv); + + pos = &lru_bulk_move.pos[mem_type][bo_priority]; + + KUNIT_ASSERT_EQ(test, bo->pin_count, 1); + KUNIT_ASSERT_NULL(test, pos->first); + KUNIT_ASSERT_NULL(test, pos->last); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_PTR_EQ(test, res, pos->last); + KUNIT_ASSERT_EQ(test, bo->pin_count, 0); + + ttm_resource_free(bo, &res); +} + +static void ttm_bo_multiple_pin_one_unpin(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_lru_bulk_move lru_bulk_move; + struct ttm_lru_bulk_move_pos *pos; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + struct ttm_device *ttm_dev; + struct ttm_place *place; + u32 mem_type = TTM_PL_SYSTEM; + unsigned int bo_priority = 0; + int err; + + ttm_lru_bulk_move_init(&lru_bulk_move); + + place = ttm_place_kunit_init(test, mem_type, 0); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + priv->ttm_dev = ttm_dev; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_resource_alloc(bo, place, &res, NULL); + KUNIT_ASSERT_EQ(test, err, 0); + bo->resource = res; + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_set_bulk_move(bo, &lru_bulk_move); + + /* Multiple pins */ + ttm_bo_pin(bo); + ttm_bo_pin(bo); + + dma_resv_unlock(bo->base.resv); + + pos = &lru_bulk_move.pos[mem_type][bo_priority]; + + KUNIT_ASSERT_EQ(test, bo->pin_count, 2); + KUNIT_ASSERT_NULL(test, pos->first); + KUNIT_ASSERT_NULL(test, pos->last); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, bo->pin_count, 1); + KUNIT_ASSERT_NULL(test, pos->first); + KUNIT_ASSERT_NULL(test, pos->last); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + ttm_resource_free(bo, &res); +} + +static struct kunit_case ttm_bo_test_cases[] = { + KUNIT_CASE_PARAM(ttm_bo_reserve_optimistic_no_ticket, + ttm_bo_reserve_gen_params), + KUNIT_CASE(ttm_bo_reserve_locked_no_sleep), + KUNIT_CASE(ttm_bo_reserve_no_wait_ticket), + KUNIT_CASE(ttm_bo_reserve_double_resv), +#if IS_BUILTIN(CONFIG_DRM_TTM_KUNIT_TEST) + KUNIT_CASE(ttm_bo_reserve_interrupted), +#endif + KUNIT_CASE(ttm_bo_reserve_deadlock), + KUNIT_CASE(ttm_bo_unreserve_basic), + KUNIT_CASE(ttm_bo_unreserve_pinned), + KUNIT_CASE(ttm_bo_unreserve_bulk), + KUNIT_CASE(ttm_bo_fini_basic), + KUNIT_CASE(ttm_bo_fini_shared_resv), + KUNIT_CASE(ttm_bo_pin_basic), + KUNIT_CASE(ttm_bo_pin_unpin_resource), + KUNIT_CASE(ttm_bo_multiple_pin_one_unpin), + {} +}; + +static struct kunit_suite ttm_bo_test_suite = { + .name = "ttm_bo", + .init = ttm_test_devices_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_bo_test_cases, +}; + +kunit_test_suites(&ttm_bo_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_bo APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c new file mode 100644 index 000000000..2db221f6f --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c @@ -0,0 +1,1217 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include <linux/delay.h> +#include <linux/kthread.h> + +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include "ttm_kunit_helpers.h" +#include "ttm_mock_manager.h" + +#define BO_SIZE SZ_4K +#define MANAGER_SIZE SZ_1M + +static struct spinlock fence_lock; + +struct ttm_bo_validate_test_case { + const char *description; + enum ttm_bo_type bo_type; + u32 mem_type; + bool with_ttm; + bool no_gpu_wait; +}; + +static struct ttm_placement *ttm_placement_kunit_init(struct kunit *test, + struct ttm_place *places, + unsigned int num_places) +{ + struct ttm_placement *placement; + + placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, placement); + + placement->num_placement = num_places; + placement->placement = places; + + return placement; +} + +static const char *fence_name(struct dma_fence *f) +{ + return "ttm-bo-validate-fence"; +} + +static const struct dma_fence_ops fence_ops = { + .get_driver_name = fence_name, + .get_timeline_name = fence_name, +}; + +static struct dma_fence *alloc_mock_fence(struct kunit *test) +{ + struct dma_fence *fence; + + fence = kunit_kzalloc(test, sizeof(*fence), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fence); + + dma_fence_init(fence, &fence_ops, &fence_lock, 0, 0); + + return fence; +} + +static void dma_resv_kunit_active_fence_init(struct kunit *test, + struct dma_resv *resv, + enum dma_resv_usage usage) +{ + struct dma_fence *fence; + + fence = alloc_mock_fence(test); + dma_fence_enable_sw_signaling(fence); + + dma_resv_lock(resv, NULL); + dma_resv_reserve_fences(resv, 1); + dma_resv_add_fence(resv, fence, usage); + dma_resv_unlock(resv); +} + +static void ttm_bo_validate_case_desc(const struct ttm_bo_validate_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +static const struct ttm_bo_validate_test_case ttm_bo_type_cases[] = { + { + .description = "Buffer object for userspace", + .bo_type = ttm_bo_type_device, + }, + { + .description = "Kernel buffer object", + .bo_type = ttm_bo_type_kernel, + }, + { + .description = "Shared buffer object", + .bo_type = ttm_bo_type_sg, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_types, ttm_bo_type_cases, + ttm_bo_validate_case_desc); + +static void ttm_bo_init_reserved_sys_man(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_test_devices *priv = test->priv; + enum ttm_bo_type bo_type = params->bo_type; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx, NULL, NULL, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, kref_read(&bo->kref), 1); + KUNIT_EXPECT_PTR_EQ(test, bo->bdev, priv->ttm_dev); + KUNIT_EXPECT_EQ(test, bo->type, bo_type); + KUNIT_EXPECT_EQ(test, bo->page_alignment, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, bo->destroy, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, bo->pin_count, 0); + KUNIT_EXPECT_NULL(test, bo->bulk_move); + KUNIT_EXPECT_NOT_NULL(test, bo->ttm); + KUNIT_EXPECT_FALSE(test, ttm_tt_is_populated(bo->ttm)); + KUNIT_EXPECT_NOT_NULL(test, (void *)bo->base.resv->fences); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, size); + + if (bo_type != ttm_bo_type_kernel) + KUNIT_EXPECT_TRUE(test, + drm_mm_node_allocated(&bo->base.vma_node.vm_node)); + + ttm_resource_free(bo, &bo->resource); + ttm_bo_fini(bo); +} + +static void ttm_bo_init_reserved_mock_man(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + enum ttm_bo_type bo_type = params->bo_type; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + u32 mem_type = TTM_PL_VRAM; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx, NULL, NULL, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, kref_read(&bo->kref), 1); + KUNIT_EXPECT_PTR_EQ(test, bo->bdev, priv->ttm_dev); + KUNIT_EXPECT_EQ(test, bo->type, bo_type); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, size); + + if (bo_type != ttm_bo_type_kernel) + KUNIT_EXPECT_TRUE(test, + drm_mm_node_allocated(&bo->base.vma_node.vm_node)); + + ttm_resource_free(bo, &bo->resource); + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_init_reserved_resv(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct dma_resv resv; + int err; + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + dma_resv_init(&resv); + dma_resv_lock(&resv, NULL); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx, NULL, &resv, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_PTR_EQ(test, bo->base.resv, &resv); + + ttm_resource_free(bo, &bo->resource); + ttm_bo_fini(bo); +} + +static void ttm_bo_validate_basic(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + u32 fst_mem = TTM_PL_SYSTEM, snd_mem = TTM_PL_VRAM; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *fst_placement, *snd_placement; + struct ttm_test_devices *priv = test->priv; + struct ttm_place *fst_place, *snd_place; + u32 size = ALIGN(SZ_8K, PAGE_SIZE); + struct ttm_buffer_object *bo; + int err; + + ttm_mock_manager_init(priv->ttm_dev, snd_mem, MANAGER_SIZE); + + fst_place = ttm_place_kunit_init(test, fst_mem, 0); + fst_placement = ttm_placement_kunit_init(test, fst_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, params->bo_type, + fst_placement, PAGE_SIZE, &ctx_init, NULL, + NULL, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + snd_place = ttm_place_kunit_init(test, snd_mem, GPU_BUDDY_TOPDOWN_ALLOCATION); + snd_placement = ttm_placement_kunit_init(test, snd_place, 1); + + err = ttm_bo_validate(bo, snd_placement, &ctx_val); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, bo->base.size); + KUNIT_EXPECT_NOT_NULL(test, bo->ttm); + KUNIT_EXPECT_TRUE(test, ttm_tt_is_populated(bo->ttm)); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); + KUNIT_EXPECT_EQ(test, bo->resource->placement, + GPU_BUDDY_TOPDOWN_ALLOCATION); + + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, snd_mem); +} + +static void ttm_bo_validate_invalid_placement(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + u32 unknown_mem_type = TTM_PL_PRIV + 1; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, unknown_mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + + ttm_bo_fini(bo); +} + +static void ttm_bo_validate_failed_alloc(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + struct ttm_placement *placement; + u32 mem_type = TTM_PL_VRAM; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + ttm_bad_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + + ttm_bo_fini(bo); + ttm_bad_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_pinned(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + ttm_bo_pin(bo); + err = ttm_bo_validate(bo, placement, &ctx); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -EINVAL); + + ttm_bo_reserve(bo, false, false, NULL); + ttm_bo_unpin(bo); + dma_resv_unlock(bo->base.resv); + + ttm_bo_fini(bo); +} + +static const struct ttm_bo_validate_test_case ttm_mem_type_cases[] = { + { + .description = "System manager", + .mem_type = TTM_PL_SYSTEM, + }, + { + .description = "VRAM manager", + .mem_type = TTM_PL_VRAM, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_validate_mem, ttm_mem_type_cases, + ttm_bo_validate_case_desc); + +static void ttm_bo_validate_same_placement(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, params->mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + if (params->mem_type != TTM_PL_SYSTEM) + ttm_mock_manager_init(priv->ttm_dev, params->mem_type, MANAGER_SIZE); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, params->bo_type, + placement, PAGE_SIZE, &ctx_init, NULL, + NULL, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + err = ttm_bo_validate(bo, placement, &ctx_val); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, 0); + + ttm_bo_fini(bo); + + if (params->mem_type != TTM_PL_SYSTEM) + ttm_mock_manager_fini(priv->ttm_dev, params->mem_type); +} + +static void ttm_bo_validate_busy_placement(struct kunit *test) +{ + u32 fst_mem = TTM_PL_VRAM, snd_mem = TTM_PL_VRAM + 1; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *placement_init, *placement_val; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_place *init_place, places[2]; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + int err; + + ttm_bad_manager_init(priv->ttm_dev, fst_mem, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, snd_mem, MANAGER_SIZE); + + init_place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement_init = ttm_placement_kunit_init(test, init_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement_init, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + places[0] = (struct ttm_place){ .mem_type = fst_mem, .flags = TTM_PL_FLAG_DESIRED }; + places[1] = (struct ttm_place){ .mem_type = snd_mem, .flags = TTM_PL_FLAG_FALLBACK }; + placement_val = ttm_placement_kunit_init(test, places, 2); + + err = ttm_bo_validate(bo, placement_val, &ctx_val); + dma_resv_unlock(bo->base.resv); + + man = ttm_manager_type(priv->ttm_dev, snd_mem); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, bo->base.size); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); + KUNIT_ASSERT_TRUE(test, list_is_singular(&man->lru[bo->priority])); + + ttm_bo_fini(bo); + ttm_bad_manager_fini(priv->ttm_dev, fst_mem); + ttm_mock_manager_fini(priv->ttm_dev, snd_mem); +} + +static void ttm_bo_validate_multihop(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *placement_init, *placement_val; + u32 fst_mem = TTM_PL_VRAM, tmp_mem = TTM_PL_TT, final_mem = TTM_PL_SYSTEM; + struct ttm_test_devices *priv = test->priv; + struct ttm_place *fst_place, *final_place; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_buffer_object *bo; + int err; + + ttm_mock_manager_init(priv->ttm_dev, fst_mem, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, tmp_mem, MANAGER_SIZE); + + fst_place = ttm_place_kunit_init(test, fst_mem, 0); + placement_init = ttm_placement_kunit_init(test, fst_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, params->bo_type, + placement_init, PAGE_SIZE, &ctx_init, NULL, + NULL, &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + final_place = ttm_place_kunit_init(test, final_mem, 0); + placement_val = ttm_placement_kunit_init(test, final_place, 1); + + err = ttm_bo_validate(bo, placement_val, &ctx_val); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, size * 2); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, final_mem); + + ttm_bo_fini(bo); + + ttm_mock_manager_fini(priv->ttm_dev, fst_mem); + ttm_mock_manager_fini(priv->ttm_dev, tmp_mem); +} + +static const struct ttm_bo_validate_test_case ttm_bo_no_placement_cases[] = { + { + .description = "Buffer object in system domain, no page vector", + }, + { + .description = "Buffer object in system domain with an existing page vector", + .with_ttm = true, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_no_placement, ttm_bo_no_placement_cases, + ttm_bo_validate_case_desc); + +static void ttm_bo_validate_no_placement_signaled(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_resource_manager *man; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_tt *old_tt; + u32 flags; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + man = ttm_manager_type(priv->ttm_dev, mem_type); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + if (params->with_ttm) { + old_tt = priv->ttm_dev->funcs->ttm_tt_create(bo, 0); + ttm_pool_alloc(&priv->ttm_dev->pool, old_tt, &ctx); + bo->ttm = old_tt; + } + + placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, placement); + + ttm_bo_reserve(bo, false, false, NULL); + + err = ttm_resource_alloc(bo, place, &bo->resource, NULL); + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, man->usage, size); + + err = ttm_bo_validate(bo, placement, &ctx); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, man->usage, 0); + KUNIT_ASSERT_NOT_NULL(test, bo->ttm); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, 0); + + if (params->with_ttm) { + flags = bo->ttm->page_flags; + + KUNIT_ASSERT_PTR_EQ(test, bo->ttm, old_tt); + KUNIT_ASSERT_FALSE(test, flags & TTM_TT_FLAG_PRIV_POPULATED); + KUNIT_ASSERT_TRUE(test, flags & TTM_TT_FLAG_ZERO_ALLOC); + } + + ttm_bo_fini(bo); +} + +static int threaded_dma_resv_signal(void *arg) +{ + struct ttm_buffer_object *bo = arg; + struct dma_resv *resv = bo->base.resv; + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) { + dma_fence_signal(fence); + } + dma_resv_iter_end(&cursor); + + return 0; +} + +static void ttm_bo_validate_no_placement_not_signaled(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct task_struct *task; + struct ttm_place *place; + int err; + + place = ttm_place_kunit_init(test, mem_type, 0); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = params->bo_type; + + err = ttm_resource_alloc(bo, place, &bo->resource, NULL); + KUNIT_EXPECT_EQ(test, err, 0); + + placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, placement); + + /* Create an active fence to simulate a non-idle resv object */ + spin_lock_init(&fence_lock); + dma_resv_kunit_active_fence_init(test, bo->base.resv, usage); + + task = kthread_create(threaded_dma_resv_signal, bo, "dma-resv-signal"); + if (IS_ERR(task)) + KUNIT_FAIL(test, "Couldn't create dma resv signal task\n"); + + wake_up_process(task); + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, bo->ttm); + KUNIT_ASSERT_NULL(test, bo->resource); + KUNIT_ASSERT_NULL(test, bo->bulk_move); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, 0); + + if (bo->type != ttm_bo_type_sg) + KUNIT_ASSERT_PTR_EQ(test, bo->base.resv, &bo->base._resv); + + /* Make sure we have an idle object at this point */ + dma_resv_wait_timeout(bo->base.resv, usage, false, MAX_SCHEDULE_TIMEOUT); + + ttm_bo_fini(bo); +} + +static void ttm_bo_validate_move_fence_signaled(struct kunit *test) +{ + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_operation_ctx ctx = { }; + u32 mem_type = TTM_PL_SYSTEM; + struct ttm_resource_manager *man; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + + man = ttm_manager_type(priv->ttm_dev, mem_type); + man->eviction_fences[0] = dma_fence_get_stub(); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + bo->type = bo_type; + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, ctx.bytes_moved, size); + + ttm_bo_fini(bo); + dma_fence_put(man->eviction_fences[0]); +} + +static const struct ttm_bo_validate_test_case ttm_bo_validate_wait_cases[] = { + { + .description = "Waits for GPU", + .no_gpu_wait = false, + }, + { + .description = "Tries to lock straight away", + .no_gpu_wait = true, + }, +}; + +KUNIT_ARRAY_PARAM(ttm_bo_validate_wait, ttm_bo_validate_wait_cases, + ttm_bo_validate_case_desc); + +static int threaded_fence_signal(void *arg) +{ + struct dma_fence *fence = arg; + + msleep(20); + + return dma_fence_check_and_signal(fence) ? -EINVAL : 0; +} + +static void ttm_bo_validate_move_fence_not_signaled(struct kunit *test) +{ + const struct ttm_bo_validate_test_case *params = test->param_value; + struct ttm_operation_ctx ctx_init = { }, + ctx_val = { .no_wait_gpu = params->no_gpu_wait }; + u32 fst_mem = TTM_PL_VRAM, snd_mem = TTM_PL_VRAM + 1; + struct ttm_placement *placement_init, *placement_val; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + u32 size = ALIGN(BO_SIZE, PAGE_SIZE); + struct ttm_place *init_place, places[2]; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + struct task_struct *task; + int err; + + init_place = ttm_place_kunit_init(test, TTM_PL_SYSTEM, 0); + placement_init = ttm_placement_kunit_init(test, init_place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, size); + + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement_init, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_mock_manager_init(priv->ttm_dev, fst_mem, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, snd_mem, MANAGER_SIZE); + + places[0] = (struct ttm_place){ .mem_type = fst_mem, .flags = TTM_PL_FLAG_DESIRED }; + places[1] = (struct ttm_place){ .mem_type = snd_mem, .flags = TTM_PL_FLAG_FALLBACK }; + placement_val = ttm_placement_kunit_init(test, places, 2); + + spin_lock_init(&fence_lock); + man = ttm_manager_type(priv->ttm_dev, fst_mem); + man->eviction_fences[0] = alloc_mock_fence(test); + + task = kthread_create(threaded_fence_signal, man->eviction_fences[0], "move-fence-signal"); + if (IS_ERR(task)) + KUNIT_FAIL(test, "Couldn't create move fence signal task\n"); + + wake_up_process(task); + err = ttm_bo_validate(bo, placement_val, &ctx_val); + dma_resv_unlock(bo->base.resv); + + dma_fence_wait_timeout(man->eviction_fences[0], false, MAX_SCHEDULE_TIMEOUT); + man->eviction_fences[0] = NULL; + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, size); + + if (params->no_gpu_wait) + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); + else + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, fst_mem); + + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, fst_mem); + ttm_mock_manager_fini(priv->ttm_dev, snd_mem); +} + +static void ttm_bo_validate_swapout(struct kunit *test) +{ + u32 mem_type = TTM_PL_TT; + struct ttm_test_devices *priv = test->priv; + struct ttm_operation_ctx ctx_init = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_resource_manager *man; + struct ttm_placement *placement; + struct ttm_buffer_object *bo; + struct ttm_place *place; + int err; + s64 swapped; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + man = ttm_manager_type(priv->ttm_dev, mem_type); + KUNIT_ASSERT_NOT_NULL(test, man); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + drm_gem_private_object_init(priv->drm, &bo->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo->base.resv); + + swapped = ttm_bo_swapout(priv->ttm_dev, &ctx_init, man, GFP_KERNEL, + MANAGER_SIZE / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, swapped, MANAGER_SIZE / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, TTM_PL_SYSTEM); + KUNIT_EXPECT_TRUE(test, bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED); + + ttm_bo_fini(bo); + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_happy_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT, + mem_type_evict = TTM_PL_SYSTEM; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + u32 small = SZ_8K, medium = SZ_512K, + big = MANAGER_SIZE - (small + medium); + u32 bo_sizes[] = { small, medium, big }; + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bos, *bo_val; + struct ttm_placement *placement; + struct ttm_place *place; + u32 bo_no = 3; + int i, err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_multihop, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bos = kunit_kmalloc_array(test, bo_no, sizeof(*bos), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bos); + + memset(bos, 0, sizeof(*bos) * bo_no); + for (i = 0; i < bo_no; i++) { + drm_gem_private_object_init(priv->drm, &bos[i].base, bo_sizes[i]); + err = ttm_bo_init_reserved(priv->ttm_dev, &bos[i], bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + dma_resv_unlock(bos[i].base.resv); + } + + bo_val = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_val->type = bo_type; + + ttm_bo_reserve(bo_val, false, false, NULL); + err = ttm_bo_validate(bo_val, placement, &ctx_val); + ttm_bo_unreserve(bo_val); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bos[0].resource->mem_type, mem_type_evict); + KUNIT_EXPECT_TRUE(test, bos[0].ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC); + KUNIT_EXPECT_TRUE(test, bos[0].ttm->page_flags & TTM_TT_FLAG_PRIV_POPULATED); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, small * 2 + BO_SIZE); + KUNIT_EXPECT_EQ(test, bos[1].resource->mem_type, mem_type); + + for (i = 0; i < bo_no; i++) + ttm_bo_fini(&bos[i]); + ttm_bo_fini(bo_val); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_multihop); +} + +static void ttm_bo_validate_all_pinned_evict(struct kunit *test) +{ + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_buffer_object *bo_big, *bo_small; + struct ttm_test_devices *priv = test->priv; + struct ttm_placement *placement; + u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_multihop, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_big = kunit_kzalloc(test, sizeof(*bo_big), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_big); + + drm_gem_private_object_init(priv->drm, &bo_big->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_big, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_bo_pin(bo_big); + dma_resv_unlock(bo_big->base.resv); + + bo_small = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_small->type = bo_type; + + ttm_bo_reserve(bo_small, false, false, NULL); + err = ttm_bo_validate(bo_small, placement, &ctx_val); + ttm_bo_unreserve(bo_small); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + + ttm_bo_fini(bo_small); + + ttm_bo_reserve(bo_big, false, false, NULL); + ttm_bo_unpin(bo_big); + dma_resv_unlock(bo_big->base.resv); + ttm_bo_fini(bo_big); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_multihop); +} + +static void ttm_bo_validate_allowed_only_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT, + mem_type_evict = TTM_PL_SYSTEM; + struct ttm_buffer_object *bo, *bo_evictable, *bo_pinned; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_placement *placement; + struct ttm_place *place; + u32 size = SZ_512K; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_multihop, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_pinned = kunit_kzalloc(test, sizeof(*bo_pinned), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_pinned); + + drm_gem_private_object_init(priv->drm, &bo_pinned->base, size); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_pinned, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + ttm_bo_pin(bo_pinned); + dma_resv_unlock(bo_pinned->base.resv); + + bo_evictable = kunit_kzalloc(test, sizeof(*bo_evictable), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_evictable); + + drm_gem_private_object_init(priv->drm, &bo_evictable->base, size); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_evictable, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_evictable->base.resv); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx_val); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, bo_pinned->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, bo_evictable->resource->mem_type, mem_type_evict); + KUNIT_EXPECT_EQ(test, ctx_val.bytes_moved, size * 2 + BO_SIZE); + + ttm_bo_fini(bo); + ttm_bo_fini(bo_evictable); + + ttm_bo_reserve(bo_pinned, false, false, NULL); + ttm_bo_unpin(bo_pinned); + dma_resv_unlock(bo_pinned->base.resv); + ttm_bo_fini(bo_pinned); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_multihop); +} + +static void ttm_bo_validate_deleted_evict(struct kunit *test) +{ + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + u32 small = SZ_8K, big = MANAGER_SIZE - BO_SIZE; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_buffer_object *bo_big, *bo_small; + struct ttm_test_devices *priv = test->priv; + struct ttm_resource_manager *man; + u32 mem_type = TTM_PL_VRAM; + struct ttm_placement *placement; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + man = ttm_manager_type(priv->ttm_dev, mem_type); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_big = kunit_kzalloc(test, sizeof(*bo_big), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_big); + + drm_gem_private_object_init(priv->drm, &bo_big->base, big); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_big, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, ttm_resource_manager_usage(man), big); + + dma_resv_unlock(bo_big->base.resv); + bo_big->deleted = true; + + bo_small = ttm_bo_kunit_init(test, test->priv, small, NULL); + bo_small->type = bo_type; + + ttm_bo_reserve(bo_small, false, false, NULL); + err = ttm_bo_validate(bo_small, placement, &ctx_val); + ttm_bo_unreserve(bo_small); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo_small->resource->mem_type, mem_type); + KUNIT_EXPECT_EQ(test, ttm_resource_manager_usage(man), small); + KUNIT_EXPECT_NULL(test, bo_big->ttm); + KUNIT_EXPECT_NULL(test, bo_big->resource); + + ttm_bo_fini(bo_small); + ttm_bo_fini(bo_big); + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_busy_domain_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_VRAM, mem_type_evict = TTM_PL_MOCK1; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo_init, *bo_val; + struct ttm_placement *placement; + struct ttm_place *place; + int err; + + /* + * Drop the default device and setup a new one that points to busy + * thus unsuitable eviction domain + */ + ttm_device_fini(priv->ttm_dev); + + err = ttm_device_kunit_init_bad_evict(test->priv, priv->ttm_dev); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_busy_manager_init(priv->ttm_dev, mem_type_evict, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_init = kunit_kzalloc(test, sizeof(*bo_init), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_init); + + drm_gem_private_object_init(priv->drm, &bo_init->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_init, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_init->base.resv); + + bo_val = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_val->type = bo_type; + + ttm_bo_reserve(bo_val, false, false, NULL); + err = ttm_bo_validate(bo_val, placement, &ctx_val); + ttm_bo_unreserve(bo_val); + + KUNIT_EXPECT_EQ(test, err, -ENOMEM); + KUNIT_EXPECT_EQ(test, bo_init->resource->mem_type, mem_type); + KUNIT_EXPECT_NULL(test, bo_val->resource); + + ttm_bo_fini(bo_init); + ttm_bo_fini(bo_val); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_bad_manager_fini(priv->ttm_dev, mem_type_evict); +} + +static void ttm_bo_validate_evict_gutting(struct kunit *test) +{ + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_buffer_object *bo, *bo_evict; + u32 mem_type = TTM_PL_MOCK1; + struct ttm_placement *placement; + struct ttm_place *place; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + + place = ttm_place_kunit_init(test, mem_type, 0); + placement = ttm_placement_kunit_init(test, place, 1); + + bo_evict = kunit_kzalloc(test, sizeof(*bo_evict), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_evict); + + drm_gem_private_object_init(priv->drm, &bo_evict->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_evict, bo_type, placement, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_evict->base.resv); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = bo_type; + + ttm_bo_reserve(bo, false, false, NULL); + err = ttm_bo_validate(bo, placement, &ctx_val); + ttm_bo_unreserve(bo); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, bo->resource->mem_type, mem_type); + KUNIT_ASSERT_NULL(test, bo_evict->resource); + KUNIT_ASSERT_TRUE(test, bo_evict->ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC); + + ttm_bo_fini(bo_evict); + ttm_bo_fini(bo); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); +} + +static void ttm_bo_validate_recrusive_evict(struct kunit *test) +{ + u32 mem_type = TTM_PL_TT, mem_type_evict = TTM_PL_MOCK2; + struct ttm_operation_ctx ctx_init = { }, ctx_val = { }; + struct ttm_placement *placement_tt, *placement_mock; + struct ttm_buffer_object *bo_tt, *bo_mock, *bo_val; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct ttm_test_devices *priv = test->priv; + struct ttm_place *place_tt, *place_mock; + int err; + + ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE); + ttm_mock_manager_init(priv->ttm_dev, mem_type_evict, MANAGER_SIZE); + + place_tt = ttm_place_kunit_init(test, mem_type, 0); + place_mock = ttm_place_kunit_init(test, mem_type_evict, 0); + + placement_tt = ttm_placement_kunit_init(test, place_tt, 1); + placement_mock = ttm_placement_kunit_init(test, place_mock, 1); + + bo_tt = kunit_kzalloc(test, sizeof(*bo_tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_tt); + + bo_mock = kunit_kzalloc(test, sizeof(*bo_mock), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo_mock); + + drm_gem_private_object_init(priv->drm, &bo_tt->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_tt, bo_type, placement_tt, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_tt->base.resv); + + drm_gem_private_object_init(priv->drm, &bo_mock->base, MANAGER_SIZE); + err = ttm_bo_init_reserved(priv->ttm_dev, bo_mock, bo_type, placement_mock, + PAGE_SIZE, &ctx_init, NULL, NULL, + &dummy_ttm_bo_destroy); + KUNIT_EXPECT_EQ(test, err, 0); + dma_resv_unlock(bo_mock->base.resv); + + bo_val = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo_val->type = bo_type; + + ttm_bo_reserve(bo_val, false, false, NULL); + err = ttm_bo_validate(bo_val, placement_tt, &ctx_val); + ttm_bo_unreserve(bo_val); + + KUNIT_EXPECT_EQ(test, err, 0); + + ttm_mock_manager_fini(priv->ttm_dev, mem_type); + ttm_mock_manager_fini(priv->ttm_dev, mem_type_evict); + + ttm_bo_fini(bo_val); + ttm_bo_fini(bo_tt); + ttm_bo_fini(bo_mock); +} + +static struct kunit_case ttm_bo_validate_test_cases[] = { + KUNIT_CASE_PARAM(ttm_bo_init_reserved_sys_man, ttm_bo_types_gen_params), + KUNIT_CASE_PARAM(ttm_bo_init_reserved_mock_man, ttm_bo_types_gen_params), + KUNIT_CASE(ttm_bo_init_reserved_resv), + KUNIT_CASE_PARAM(ttm_bo_validate_basic, ttm_bo_types_gen_params), + KUNIT_CASE(ttm_bo_validate_invalid_placement), + KUNIT_CASE_PARAM(ttm_bo_validate_same_placement, + ttm_bo_validate_mem_gen_params), + KUNIT_CASE(ttm_bo_validate_failed_alloc), + KUNIT_CASE(ttm_bo_validate_pinned), + KUNIT_CASE(ttm_bo_validate_busy_placement), + KUNIT_CASE_PARAM(ttm_bo_validate_multihop, ttm_bo_types_gen_params), + KUNIT_CASE_PARAM(ttm_bo_validate_no_placement_signaled, + ttm_bo_no_placement_gen_params), + KUNIT_CASE_PARAM(ttm_bo_validate_no_placement_not_signaled, + ttm_bo_types_gen_params), + KUNIT_CASE(ttm_bo_validate_move_fence_signaled), + KUNIT_CASE_PARAM(ttm_bo_validate_move_fence_not_signaled, + ttm_bo_validate_wait_gen_params), + KUNIT_CASE(ttm_bo_validate_swapout), + KUNIT_CASE(ttm_bo_validate_happy_evict), + KUNIT_CASE(ttm_bo_validate_all_pinned_evict), + KUNIT_CASE(ttm_bo_validate_allowed_only_evict), + KUNIT_CASE(ttm_bo_validate_deleted_evict), + KUNIT_CASE(ttm_bo_validate_busy_domain_evict), + KUNIT_CASE(ttm_bo_validate_evict_gutting), + KUNIT_CASE(ttm_bo_validate_recrusive_evict), + {} +}; + +static struct kunit_suite ttm_bo_validate_test_suite = { + .name = "ttm_bo_validate", + .init = ttm_test_devices_all_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_bo_validate_test_cases, +}; + +kunit_test_suites(&ttm_bo_validate_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_bo APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_device_test.c b/drivers/gpu/drm/ttm/tests/ttm_device_test.c new file mode 100644 index 000000000..8bcac84e9 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_device_test.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_placement.h> + +#include "ttm_kunit_helpers.h" +#include "../ttm_pool_internal.h" + +struct ttm_device_test_case { + const char *description; + unsigned int alloc_flags; + bool pools_init_expected; +}; + +static void ttm_device_init_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_device *ttm_dev; + struct ttm_resource_manager *ttm_sys_man; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_EXPECT_PTR_EQ(test, ttm_dev->funcs, &ttm_dev_funcs); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev->wq); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev->man_drv[TTM_PL_SYSTEM]); + + ttm_sys_man = &ttm_dev->sysman; + KUNIT_ASSERT_NOT_NULL(test, ttm_sys_man); + KUNIT_EXPECT_TRUE(test, ttm_sys_man->use_tt); + KUNIT_EXPECT_TRUE(test, ttm_sys_man->use_type); + KUNIT_ASSERT_NOT_NULL(test, ttm_sys_man->func); + + KUNIT_EXPECT_PTR_EQ(test, ttm_dev->dev_mapping, + priv->drm->anon_inode->i_mapping); + + ttm_device_fini(ttm_dev); +} + +static void ttm_device_init_multiple(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_device *ttm_devs; + unsigned int i, num_dev = 3; + int err; + + ttm_devs = kunit_kcalloc(test, num_dev, sizeof(*ttm_devs), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_devs); + + for (i = 0; i < num_dev; i++) { + err = ttm_device_kunit_init(priv, &ttm_devs[i], 0); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_EXPECT_PTR_EQ(test, ttm_devs[i].dev_mapping, + priv->drm->anon_inode->i_mapping); + KUNIT_ASSERT_NOT_NULL(test, ttm_devs[i].wq); + KUNIT_EXPECT_PTR_EQ(test, ttm_devs[i].funcs, &ttm_dev_funcs); + KUNIT_ASSERT_NOT_NULL(test, ttm_devs[i].man_drv[TTM_PL_SYSTEM]); + } + + KUNIT_ASSERT_EQ(test, list_count_nodes(&ttm_devs[0].device_list), num_dev); + + for (i = 0; i < num_dev; i++) + ttm_device_fini(&ttm_devs[i]); +} + +static void ttm_device_fini_basic(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct ttm_device *ttm_dev; + struct ttm_resource_manager *man; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + man = ttm_manager_type(ttm_dev, TTM_PL_SYSTEM); + KUNIT_ASSERT_NOT_NULL(test, man); + + ttm_device_fini(ttm_dev); + + KUNIT_ASSERT_FALSE(test, man->use_type); + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[0])); + KUNIT_ASSERT_NULL(test, ttm_dev->man_drv[TTM_PL_SYSTEM]); +} + +static void ttm_device_init_no_vma_man(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + struct drm_device *drm = priv->drm; + struct ttm_device *ttm_dev; + struct drm_vma_offset_manager *vma_man; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + /* Let's pretend there's no VMA manager allocated */ + vma_man = drm->vma_offset_manager; + drm->vma_offset_manager = NULL; + + err = ttm_device_kunit_init(priv, ttm_dev, 0); + KUNIT_EXPECT_EQ(test, err, -EINVAL); + + /* Bring the manager back for a graceful cleanup */ + drm->vma_offset_manager = vma_man; +} + +static const struct ttm_device_test_case ttm_device_cases[] = { + { + .description = "No DMA allocations, no DMA32 required", + .pools_init_expected = false, + }, + { + .description = "DMA allocations, DMA32 required", + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC | + TTM_ALLOCATION_POOL_USE_DMA32, + .pools_init_expected = true, + }, + { + .description = "No DMA allocations, DMA32 required", + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA32, + .pools_init_expected = false, + }, + { + .description = "DMA allocations, no DMA32 required", + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC, + .pools_init_expected = true, + }, +}; + +static void ttm_device_case_desc(const struct ttm_device_test_case *t, char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_device, ttm_device_cases, ttm_device_case_desc); + +static void ttm_device_init_pools(struct kunit *test) +{ + struct ttm_test_devices *priv = test->priv; + const struct ttm_device_test_case *params = test->param_value; + struct ttm_device *ttm_dev; + struct ttm_pool *pool; + struct ttm_pool_type pt; + int err; + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(priv, ttm_dev, params->alloc_flags); + KUNIT_ASSERT_EQ(test, err, 0); + + pool = &ttm_dev->pool; + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_EXPECT_PTR_EQ(test, pool->dev, priv->dev); + KUNIT_EXPECT_EQ(test, pool->alloc_flags, params->alloc_flags); + + if (params->pools_init_expected) { + for (int i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + for (int j = 0; j < NR_PAGE_ORDERS; ++j) { + pt = pool->caching[i].orders[j]; + KUNIT_EXPECT_PTR_EQ(test, pt.pool, pool); + KUNIT_EXPECT_EQ(test, pt.caching, i); + KUNIT_EXPECT_EQ(test, pt.order, j); + + if (ttm_pool_uses_dma_alloc(pool)) + KUNIT_ASSERT_FALSE(test, + list_lru_count(&pt.pages)); + } + } + } + + ttm_device_fini(ttm_dev); +} + +static struct kunit_case ttm_device_test_cases[] = { + KUNIT_CASE(ttm_device_init_basic), + KUNIT_CASE(ttm_device_init_multiple), + KUNIT_CASE(ttm_device_fini_basic), + KUNIT_CASE(ttm_device_init_no_vma_man), + KUNIT_CASE_PARAM(ttm_device_init_pools, ttm_device_gen_params), + {} +}; + +static struct kunit_suite ttm_device_test_suite = { + .name = "ttm_device", + .init = ttm_test_devices_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_device_test_cases, +}; + +kunit_test_suites(&ttm_device_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_device APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c new file mode 100644 index 000000000..5cfe8f3f8 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include <linux/export.h> + +#include <drm/ttm/ttm_tt.h> + +#include "ttm_kunit_helpers.h" + +static const struct ttm_place sys_place = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_SYSTEM, + .flags = TTM_PL_FLAG_FALLBACK, +}; + +static const struct ttm_place mock1_place = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_MOCK1, + .flags = TTM_PL_FLAG_FALLBACK, +}; + +static const struct ttm_place mock2_place = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_MOCK2, + .flags = TTM_PL_FLAG_FALLBACK, +}; + +static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_place, +}; + +static struct ttm_placement bad_placement = { + .num_placement = 1, + .placement = &mock1_place, +}; + +static struct ttm_placement mock_placement = { + .num_placement = 1, + .placement = &mock2_place, +}; + +static struct ttm_tt *ttm_tt_simple_create(struct ttm_buffer_object *bo, u32 page_flags) +{ + struct ttm_tt *tt; + + tt = kzalloc_obj(*tt); + ttm_tt_init(tt, bo, page_flags, ttm_cached, 0); + + return tt; +} + +static void ttm_tt_simple_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) +{ + kfree(ttm); +} + +static int mock_move(struct ttm_buffer_object *bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_resource *new_mem, + struct ttm_place *hop) +{ + struct ttm_resource *old_mem = bo->resource; + + if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm)) { + ttm_bo_move_null(bo, new_mem); + return 0; + } + + if (bo->resource->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM) { + hop->mem_type = TTM_PL_TT; + hop->flags = TTM_PL_FLAG_TEMPORARY; + hop->fpfn = 0; + hop->lpfn = 0; + return -EMULTIHOP; + } + + if ((old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_TT) || + (old_mem->mem_type == TTM_PL_TT && + new_mem->mem_type == TTM_PL_SYSTEM)) { + ttm_bo_move_null(bo, new_mem); + return 0; + } + + return ttm_bo_move_memcpy(bo, ctx, new_mem); +} + +static void mock_evict_flags(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + switch (bo->resource->mem_type) { + case TTM_PL_VRAM: + case TTM_PL_SYSTEM: + *placement = sys_placement; + break; + case TTM_PL_TT: + *placement = mock_placement; + break; + case TTM_PL_MOCK1: + /* Purge objects coming from this domain */ + break; + } +} + +static void bad_evict_flags(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + *placement = bad_placement; +} + +static int ttm_device_kunit_init_with_funcs(struct ttm_test_devices *priv, + struct ttm_device *ttm, + unsigned int alloc_flags, + struct ttm_device_funcs *funcs) +{ + struct drm_device *drm = priv->drm; + int err; + + err = ttm_device_init(ttm, funcs, drm->dev, + drm->anon_inode->i_mapping, + drm->vma_offset_manager, + alloc_flags); + + return err; +} + +struct ttm_device_funcs ttm_dev_funcs = { + .ttm_tt_create = ttm_tt_simple_create, + .ttm_tt_destroy = ttm_tt_simple_destroy, + .move = mock_move, + .eviction_valuable = ttm_bo_eviction_valuable, + .evict_flags = mock_evict_flags, +}; +EXPORT_SYMBOL_GPL(ttm_dev_funcs); + +int ttm_device_kunit_init(struct ttm_test_devices *priv, + struct ttm_device *ttm, + unsigned int alloc_flags) +{ + return ttm_device_kunit_init_with_funcs(priv, ttm, alloc_flags, + &ttm_dev_funcs); +} +EXPORT_SYMBOL_GPL(ttm_device_kunit_init); + +struct ttm_device_funcs ttm_dev_funcs_bad_evict = { + .ttm_tt_create = ttm_tt_simple_create, + .ttm_tt_destroy = ttm_tt_simple_destroy, + .move = mock_move, + .eviction_valuable = ttm_bo_eviction_valuable, + .evict_flags = bad_evict_flags, +}; +EXPORT_SYMBOL_GPL(ttm_dev_funcs_bad_evict); + +int ttm_device_kunit_init_bad_evict(struct ttm_test_devices *priv, + struct ttm_device *ttm) +{ + return ttm_device_kunit_init_with_funcs(priv, ttm, 0, + &ttm_dev_funcs_bad_evict); +} +EXPORT_SYMBOL_GPL(ttm_device_kunit_init_bad_evict); + +struct ttm_buffer_object *ttm_bo_kunit_init(struct kunit *test, + struct ttm_test_devices *devs, + size_t size, + struct dma_resv *obj) +{ + struct drm_gem_object gem_obj = { }; + struct ttm_buffer_object *bo; + int err; + + bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + bo->base = gem_obj; + + if (obj) + bo->base.resv = obj; + + err = drm_gem_object_init(devs->drm, &bo->base, size); + KUNIT_ASSERT_EQ(test, err, 0); + + bo->bdev = devs->ttm_dev; + bo->destroy = dummy_ttm_bo_destroy; + + kref_init(&bo->kref); + + return bo; +} +EXPORT_SYMBOL_GPL(ttm_bo_kunit_init); + +struct ttm_place *ttm_place_kunit_init(struct kunit *test, u32 mem_type, u32 flags) +{ + struct ttm_place *place; + + place = kunit_kzalloc(test, sizeof(*place), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, place); + + place->mem_type = mem_type; + place->flags = flags; + + return place; +} +EXPORT_SYMBOL_GPL(ttm_place_kunit_init); + +void dummy_ttm_bo_destroy(struct ttm_buffer_object *bo) +{ + drm_gem_object_release(&bo->base); +} +EXPORT_SYMBOL_GPL(dummy_ttm_bo_destroy); + +struct ttm_test_devices *ttm_test_devices_basic(struct kunit *test) +{ + struct ttm_test_devices *devs; + + devs = kunit_kzalloc(test, sizeof(*devs), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, devs); + + devs->dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, devs->dev); + + /* Set mask for alloc_coherent mappings to enable ttm_pool_alloc testing */ + devs->dev->coherent_dma_mask = -1; + + devs->drm = __drm_kunit_helper_alloc_drm_device(test, devs->dev, + sizeof(*devs->drm), 0, + DRIVER_GEM); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, devs->drm); + + return devs; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_basic); + +struct ttm_test_devices *ttm_test_devices_all(struct kunit *test) +{ + struct ttm_test_devices *devs; + struct ttm_device *ttm_dev; + int err; + + devs = ttm_test_devices_basic(test); + + ttm_dev = kunit_kzalloc(test, sizeof(*ttm_dev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + + err = ttm_device_kunit_init(devs, ttm_dev, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + devs->ttm_dev = ttm_dev; + + return devs; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_all); + +void ttm_test_devices_put(struct kunit *test, struct ttm_test_devices *devs) +{ + if (devs->ttm_dev) + ttm_device_fini(devs->ttm_dev); + + drm_kunit_helper_free_device(test, devs->dev); +} +EXPORT_SYMBOL_GPL(ttm_test_devices_put); + +int ttm_test_devices_init(struct kunit *test) +{ + struct ttm_test_devices *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv = ttm_test_devices_basic(test); + test->priv = priv; + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_init); + +int ttm_test_devices_all_init(struct kunit *test) +{ + struct ttm_test_devices *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv = ttm_test_devices_all(test); + test->priv = priv; + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_test_devices_all_init); + +void ttm_test_devices_fini(struct kunit *test) +{ + ttm_test_devices_put(test, test->priv); +} +EXPORT_SYMBOL_GPL(ttm_test_devices_fini); + +MODULE_DESCRIPTION("TTM KUnit test helper functions"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h new file mode 100644 index 000000000..f8402b979 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2023 Intel Corporation + */ +#ifndef TTM_KUNIT_HELPERS_H +#define TTM_KUNIT_HELPERS_H + +#include <drm/drm_drv.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_placement.h> + +#include <drm/drm_kunit_helpers.h> +#include <kunit/test.h> + +#define TTM_PL_MOCK1 (TTM_PL_PRIV + 1) +#define TTM_PL_MOCK2 (TTM_PL_PRIV + 2) + +extern struct ttm_device_funcs ttm_dev_funcs; +extern struct ttm_device_funcs ttm_dev_funcs_bad_evict; + +struct ttm_test_devices { + struct drm_device *drm; + struct device *dev; + struct ttm_device *ttm_dev; +}; + +/* Building blocks for test-specific init functions */ +int ttm_device_kunit_init(struct ttm_test_devices *priv, + struct ttm_device *ttm, + unsigned int alloc_flags); +int ttm_device_kunit_init_bad_evict(struct ttm_test_devices *priv, + struct ttm_device *ttm); +struct ttm_buffer_object *ttm_bo_kunit_init(struct kunit *test, + struct ttm_test_devices *devs, + size_t size, + struct dma_resv *obj); +struct ttm_place *ttm_place_kunit_init(struct kunit *test, u32 mem_type, + u32 flags); +void dummy_ttm_bo_destroy(struct ttm_buffer_object *bo); + +struct ttm_test_devices *ttm_test_devices_basic(struct kunit *test); +struct ttm_test_devices *ttm_test_devices_all(struct kunit *test); + +void ttm_test_devices_put(struct kunit *test, struct ttm_test_devices *devs); + +/* Generic init/fini for tests that only need DRM/TTM devices */ +int ttm_test_devices_init(struct kunit *test); +int ttm_test_devices_all_init(struct kunit *test); +void ttm_test_devices_fini(struct kunit *test); + +#endif // TTM_KUNIT_HELPERS_H diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c new file mode 100644 index 000000000..0d91bc51f --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include <linux/export.h> +#include <linux/module.h> + +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_placement.h> + +#include "ttm_mock_manager.h" + +static inline struct ttm_mock_manager * +to_mock_mgr(struct ttm_resource_manager *man) +{ + return container_of(man, struct ttm_mock_manager, man); +} + +static inline struct ttm_mock_resource * +to_mock_mgr_resource(struct ttm_resource *res) +{ + return container_of(res, struct ttm_mock_resource, base); +} + +static int ttm_mock_manager_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct ttm_mock_manager *manager = to_mock_mgr(man); + struct ttm_mock_resource *mock_res; + struct gpu_buddy *mm = &manager->mm; + u64 lpfn, fpfn, alloc_size; + int err; + + mock_res = kzalloc_obj(*mock_res); + + if (!mock_res) + return -ENOMEM; + + fpfn = 0; + lpfn = man->size; + + ttm_resource_init(bo, place, &mock_res->base); + INIT_LIST_HEAD(&mock_res->blocks); + + if (place->flags & TTM_PL_FLAG_TOPDOWN) + mock_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) + mock_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; + + alloc_size = (uint64_t)mock_res->base.size; + mutex_lock(&manager->lock); + err = gpu_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size, + manager->default_page_size, + &mock_res->blocks, + mock_res->flags); + + if (err) + goto error_free_blocks; + mutex_unlock(&manager->lock); + + *res = &mock_res->base; + return 0; + +error_free_blocks: + gpu_buddy_free_list(mm, &mock_res->blocks, 0); + ttm_resource_fini(man, &mock_res->base); + mutex_unlock(&manager->lock); + + return err; +} + +static void ttm_mock_manager_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct ttm_mock_manager *manager = to_mock_mgr(man); + struct ttm_mock_resource *mock_res = to_mock_mgr_resource(res); + struct gpu_buddy *mm = &manager->mm; + + mutex_lock(&manager->lock); + gpu_buddy_free_list(mm, &mock_res->blocks, 0); + mutex_unlock(&manager->lock); + + ttm_resource_fini(man, res); + kfree(mock_res); +} + +static const struct ttm_resource_manager_func ttm_mock_manager_funcs = { + .alloc = ttm_mock_manager_alloc, + .free = ttm_mock_manager_free, +}; + +int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) +{ + struct ttm_mock_manager *manager; + struct ttm_resource_manager *base; + int err; + + manager = kzalloc_obj(*manager); + if (!manager) + return -ENOMEM; + + mutex_init(&manager->lock); + + err = gpu_buddy_init(&manager->mm, size, PAGE_SIZE); + + if (err) { + kfree(manager); + return err; + } + + manager->default_page_size = PAGE_SIZE; + base = &manager->man; + base->func = &ttm_mock_manager_funcs; + base->use_tt = true; + + ttm_resource_manager_init(base, bdev, size); + ttm_set_driver_manager(bdev, mem_type, base); + ttm_resource_manager_set_used(base, true); + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_mock_manager_init); + +void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type) +{ + struct ttm_resource_manager *man; + struct ttm_mock_manager *mock_man; + int err; + + man = ttm_manager_type(bdev, mem_type); + mock_man = to_mock_mgr(man); + + err = ttm_resource_manager_evict_all(bdev, man); + if (err) + return; + + ttm_resource_manager_set_used(man, false); + + mutex_lock(&mock_man->lock); + gpu_buddy_fini(&mock_man->mm); + mutex_unlock(&mock_man->lock); + + ttm_set_driver_manager(bdev, mem_type, NULL); +} +EXPORT_SYMBOL_GPL(ttm_mock_manager_fini); + +static int ttm_bad_manager_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + return -ENOSPC; +} + +static int ttm_busy_manager_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + return -EBUSY; +} + +static void ttm_bad_manager_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ +} + +static bool ttm_bad_manager_compatible(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + return true; +} + +static const struct ttm_resource_manager_func ttm_bad_manager_funcs = { + .alloc = ttm_bad_manager_alloc, + .free = ttm_bad_manager_free, + .compatible = ttm_bad_manager_compatible +}; + +static const struct ttm_resource_manager_func ttm_bad_busy_manager_funcs = { + .alloc = ttm_busy_manager_alloc, + .free = ttm_bad_manager_free, + .compatible = ttm_bad_manager_compatible +}; + +int ttm_bad_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) +{ + struct ttm_resource_manager *man; + + man = kzalloc_obj(*man); + if (!man) + return -ENOMEM; + + man->func = &ttm_bad_manager_funcs; + + ttm_resource_manager_init(man, bdev, size); + ttm_set_driver_manager(bdev, mem_type, man); + ttm_resource_manager_set_used(man, true); + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_bad_manager_init); + +int ttm_busy_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) +{ + struct ttm_resource_manager *man; + + ttm_bad_manager_init(bdev, mem_type, size); + man = ttm_manager_type(bdev, mem_type); + + man->func = &ttm_bad_busy_manager_funcs; + + return 0; +} +EXPORT_SYMBOL_GPL(ttm_busy_manager_init); + +void ttm_bad_manager_fini(struct ttm_device *bdev, uint32_t mem_type) +{ + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, mem_type); + + ttm_resource_manager_set_used(man, false); + ttm_set_driver_manager(bdev, mem_type, NULL); + + kfree(man); +} +EXPORT_SYMBOL_GPL(ttm_bad_manager_fini); + +MODULE_DESCRIPTION("KUnit tests for ttm with mock resource managers"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h new file mode 100644 index 000000000..08710756f --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2023 Intel Corporation + */ +#ifndef TTM_MOCK_MANAGER_H +#define TTM_MOCK_MANAGER_H + +#include <linux/gpu_buddy.h> + +struct ttm_mock_manager { + struct ttm_resource_manager man; + struct gpu_buddy mm; + u64 default_page_size; + /* protects allocations of mock buffer objects */ + struct mutex lock; +}; + +struct ttm_mock_resource { + struct ttm_resource base; + struct list_head blocks; + unsigned long flags; +}; + +int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size); +int ttm_bad_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size); +int ttm_busy_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size); +void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type); +void ttm_bad_manager_fini(struct ttm_device *bdev, u32 mem_type); + +#endif // TTM_MOCK_MANAGER_H diff --git a/drivers/gpu/drm/ttm/tests/ttm_pool_test.c b/drivers/gpu/drm/ttm/tests/ttm_pool_test.c new file mode 100644 index 000000000..be75c8abf --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_pool_test.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include <linux/mm.h> + +#include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_pool.h> + +#include "ttm_kunit_helpers.h" +#include "../ttm_pool_internal.h" + +struct ttm_pool_test_case { + const char *description; + unsigned int order; + unsigned int alloc_flags; +}; + +struct ttm_pool_test_priv { + struct ttm_test_devices *devs; + + /* Used to create mock ttm_tts */ + struct ttm_buffer_object *mock_bo; +}; + +static struct ttm_operation_ctx simple_ctx = { + .interruptible = true, + .no_wait_gpu = false, +}; + +static int ttm_pool_test_init(struct kunit *test) +{ + struct ttm_pool_test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv->devs = ttm_test_devices_basic(test); + test->priv = priv; + + return 0; +} + +static void ttm_pool_test_fini(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + + ttm_test_devices_put(test, priv->devs); +} + +static struct ttm_tt *ttm_tt_kunit_init(struct kunit *test, + u32 page_flags, + enum ttm_caching caching, + size_t size) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err; + + bo = ttm_bo_kunit_init(test, priv->devs, size, NULL); + KUNIT_ASSERT_NOT_NULL(test, bo); + priv->mock_bo = bo; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, priv->mock_bo, page_flags, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + return tt; +} + +static struct ttm_pool *ttm_pool_pre_populated(struct kunit *test, + size_t size, + enum ttm_caching caching) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + struct ttm_pool *pool; + struct ttm_tt *tt; + int err; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, TTM_ALLOCATION_POOL_USE_DMA_ALLOC); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + return pool; +} + +static const struct ttm_pool_test_case ttm_pool_basic_cases[] = { + { + .description = "One page", + .order = 0, + }, + { + .description = "More than one page", + .order = 2, + }, + { + .description = "Above the allocation limit", + .order = MAX_PAGE_ORDER + 1, + }, + { + .description = "One page, with coherent DMA mappings enabled", + .order = 0, + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC, + }, + { + .description = "Above the allocation limit, with coherent DMA mappings enabled", + .order = MAX_PAGE_ORDER + 1, + .alloc_flags = TTM_ALLOCATION_POOL_USE_DMA_ALLOC, + }, +}; + +static void ttm_pool_alloc_case_desc(const struct ttm_pool_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_pool_alloc_basic, ttm_pool_basic_cases, + ttm_pool_alloc_case_desc); + +static void ttm_pool_alloc_basic(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + const struct ttm_pool_test_case *params = test->param_value; + struct ttm_tt *tt; + struct ttm_pool *pool; + struct page *fst_page, *last_page; + enum ttm_caching caching = ttm_uncached; + unsigned int expected_num_pages = 1 << params->order; + size_t size = expected_num_pages * PAGE_SIZE; + int err; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, params->alloc_flags); + + KUNIT_ASSERT_PTR_EQ(test, pool->dev, devs->dev); + KUNIT_ASSERT_EQ(test, pool->nid, NUMA_NO_NODE); + KUNIT_ASSERT_EQ(test, pool->alloc_flags, params->alloc_flags); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, tt->num_pages, expected_num_pages); + + fst_page = tt->pages[0]; + last_page = tt->pages[tt->num_pages - 1]; + + if (params->order <= MAX_PAGE_ORDER) { + if (ttm_pool_uses_dma_alloc(pool)) { + KUNIT_ASSERT_NOT_NULL(test, (void *)fst_page->private); + KUNIT_ASSERT_NOT_NULL(test, (void *)last_page->private); + } else { + KUNIT_ASSERT_EQ(test, fst_page->private, params->order); + } + } else { + if (ttm_pool_uses_dma_alloc(pool)) { + KUNIT_ASSERT_NOT_NULL(test, (void *)fst_page->private); + KUNIT_ASSERT_NULL(test, (void *)last_page->private); + } else { + /* + * We expect to alloc one big block, followed by + * order 0 blocks + */ + KUNIT_ASSERT_EQ(test, fst_page->private, + min_t(unsigned int, MAX_PAGE_ORDER, + params->order)); + KUNIT_ASSERT_EQ(test, last_page->private, 0); + } + } + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_basic_dma_addr(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + const struct ttm_pool_test_case *params = test->param_value; + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_buffer_object *bo; + dma_addr_t dma1, dma2; + enum ttm_caching caching = ttm_uncached; + unsigned int expected_num_pages = 1 << params->order; + size_t size = expected_num_pages * PAGE_SIZE; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, devs, size, NULL); + KUNIT_ASSERT_NOT_NULL(test, bo); + + err = ttm_sg_tt_init(tt, bo, 0, caching); + KUNIT_ASSERT_EQ(test, err, 0); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, TTM_ALLOCATION_POOL_USE_DMA_ALLOC); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, tt->num_pages, expected_num_pages); + + dma1 = tt->dma_address[0]; + dma2 = tt->dma_address[tt->num_pages - 1]; + + KUNIT_ASSERT_NOT_NULL(test, (void *)(uintptr_t)dma1); + KUNIT_ASSERT_NOT_NULL(test, (void *)(uintptr_t)dma2); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_order_caching_match(struct kunit *test) +{ + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 0; + size_t size = PAGE_SIZE; + int err; + + pool = ttm_pool_pre_populated(test, size, caching); + + pt = &pool->caching[caching].orders[order]; + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt->pages)); + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt->pages)); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_caching_mismatch(struct kunit *test) +{ + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt_pool, *pt_tt; + enum ttm_caching tt_caching = ttm_uncached; + enum ttm_caching pool_caching = ttm_cached; + size_t size = PAGE_SIZE; + unsigned int order = 0; + int err; + + pool = ttm_pool_pre_populated(test, size, pool_caching); + + pt_pool = &pool->caching[pool_caching].orders[order]; + pt_tt = &pool->caching[tt_caching].orders[order]; + + tt = ttm_tt_kunit_init(test, 0, tt_caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt_tt->pages)); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_tt->pages)); + + ttm_pool_fini(pool); +} + +static void ttm_pool_alloc_order_mismatch(struct kunit *test) +{ + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt_pool, *pt_tt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 2; + size_t fst_size = (1 << order) * PAGE_SIZE; + size_t snd_size = PAGE_SIZE; + int err; + + pool = ttm_pool_pre_populated(test, fst_size, caching); + + pt_pool = &pool->caching[caching].orders[order]; + pt_tt = &pool->caching[caching].orders[0]; + + tt = ttm_tt_kunit_init(test, 0, caching, snd_size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt_tt->pages)); + + err = ttm_pool_alloc(pool, tt, &simple_ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_pool->pages)); + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt_tt->pages)); + + ttm_pool_fini(pool); +} + +static void ttm_pool_free_dma_alloc(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + struct ttm_tt *tt; + struct ttm_pool *pool; + struct ttm_pool_type *pt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 2; + size_t size = (1 << order) * PAGE_SIZE; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, TTM_ALLOCATION_POOL_USE_DMA_ALLOC); + ttm_pool_alloc(pool, tt, &simple_ctx); + + pt = &pool->caching[caching].orders[order]; + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt->pages)); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt->pages)); + + ttm_pool_fini(pool); +} + +static void ttm_pool_free_no_dma_alloc(struct kunit *test) +{ + struct ttm_pool_test_priv *priv = test->priv; + struct ttm_test_devices *devs = priv->devs; + struct ttm_tt *tt; + struct ttm_pool *pool; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 2; + size_t size = (1 << order) * PAGE_SIZE; + + tt = ttm_tt_kunit_init(test, 0, caching, size); + KUNIT_ASSERT_NOT_NULL(test, tt); + + pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + + ttm_pool_init(pool, devs->dev, NUMA_NO_NODE, 0); + ttm_pool_alloc(pool, tt, &simple_ctx); + + ttm_pool_free(pool, tt); + ttm_tt_fini(tt); + + ttm_pool_fini(pool); +} + +static void ttm_pool_fini_basic(struct kunit *test) +{ + struct ttm_pool *pool; + struct ttm_pool_type *pt; + enum ttm_caching caching = ttm_uncached; + unsigned int order = 0; + size_t size = PAGE_SIZE; + + pool = ttm_pool_pre_populated(test, size, caching); + pt = &pool->caching[caching].orders[order]; + + KUNIT_ASSERT_FALSE(test, !list_lru_count(&pt->pages)); + + ttm_pool_fini(pool); + + KUNIT_ASSERT_TRUE(test, !list_lru_count(&pt->pages)); +} + +static struct kunit_case ttm_pool_test_cases[] = { + KUNIT_CASE_PARAM(ttm_pool_alloc_basic, ttm_pool_alloc_basic_gen_params), + KUNIT_CASE_PARAM(ttm_pool_alloc_basic_dma_addr, + ttm_pool_alloc_basic_gen_params), + KUNIT_CASE(ttm_pool_alloc_order_caching_match), + KUNIT_CASE(ttm_pool_alloc_caching_mismatch), + KUNIT_CASE(ttm_pool_alloc_order_mismatch), + KUNIT_CASE(ttm_pool_free_dma_alloc), + KUNIT_CASE(ttm_pool_free_no_dma_alloc), + KUNIT_CASE(ttm_pool_fini_basic), + {} +}; + +static struct kunit_suite ttm_pool_test_suite = { + .name = "ttm_pool", + .init = ttm_pool_test_init, + .exit = ttm_pool_test_fini, + .test_cases = ttm_pool_test_cases, +}; + +kunit_test_suites(&ttm_pool_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_pool APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_resource_test.c b/drivers/gpu/drm/ttm/tests/ttm_resource_test.c new file mode 100644 index 000000000..c0e4e35e0 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_resource_test.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include <drm/ttm/ttm_resource.h> + +#include "ttm_kunit_helpers.h" + +#define RES_SIZE SZ_4K +#define TTM_PRIV_DUMMY_REG (TTM_NUM_MEM_TYPES - 1) + +struct ttm_resource_test_case { + const char *description; + u32 mem_type; + u32 flags; +}; + +struct ttm_resource_test_priv { + struct ttm_test_devices *devs; + struct ttm_buffer_object *bo; + struct ttm_place *place; +}; + +static const struct ttm_resource_manager_func ttm_resource_manager_mock_funcs = { }; + +static int ttm_resource_test_init(struct kunit *test) +{ + struct ttm_resource_test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + + priv->devs = ttm_test_devices_all(test); + KUNIT_ASSERT_NOT_NULL(test, priv->devs); + + test->priv = priv; + + return 0; +} + +static void ttm_resource_test_fini(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + + ttm_test_devices_put(test, priv->devs); +} + +static void ttm_init_test_mocks(struct kunit *test, + struct ttm_resource_test_priv *priv, + u32 mem_type, u32 flags) +{ + size_t size = RES_SIZE; + + /* Make sure we have what we need for a good BO mock */ + KUNIT_ASSERT_NOT_NULL(test, priv->devs->ttm_dev); + + priv->bo = ttm_bo_kunit_init(test, priv->devs, size, NULL); + priv->place = ttm_place_kunit_init(test, mem_type, flags); +} + +static void ttm_init_test_manager(struct kunit *test, + struct ttm_resource_test_priv *priv, + u32 mem_type) +{ + struct ttm_device *ttm_dev = priv->devs->ttm_dev; + struct ttm_resource_manager *man; + size_t size = SZ_16K; + + man = kunit_kzalloc(test, sizeof(*man), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, man); + + man->use_tt = false; + man->func = &ttm_resource_manager_mock_funcs; + + ttm_resource_manager_init(man, ttm_dev, size); + ttm_set_driver_manager(ttm_dev, mem_type, man); + ttm_resource_manager_set_used(man, true); +} + +static const struct ttm_resource_test_case ttm_resource_cases[] = { + { + .description = "Init resource in TTM_PL_SYSTEM", + .mem_type = TTM_PL_SYSTEM, + }, + { + .description = "Init resource in TTM_PL_VRAM", + .mem_type = TTM_PL_VRAM, + }, + { + .description = "Init resource in a private placement", + .mem_type = TTM_PRIV_DUMMY_REG, + }, + { + .description = "Init resource in TTM_PL_SYSTEM, set placement flags", + .mem_type = TTM_PL_SYSTEM, + .flags = TTM_PL_FLAG_TOPDOWN, + }, +}; + +static void ttm_resource_case_desc(const struct ttm_resource_test_case *t, char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_resource, ttm_resource_cases, ttm_resource_case_desc); + +static void ttm_resource_init_basic(struct kunit *test) +{ + const struct ttm_resource_test_case *params = test->param_value; + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + u64 expected_usage; + + ttm_init_test_mocks(test, priv, params->mem_type, params->flags); + bo = priv->bo; + place = priv->place; + + if (params->mem_type > TTM_PL_SYSTEM) + ttm_init_test_manager(test, priv, params->mem_type); + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + expected_usage = man->usage + RES_SIZE; + + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[bo->priority])); + + ttm_resource_init(bo, place, res); + + KUNIT_ASSERT_EQ(test, res->start, 0); + KUNIT_ASSERT_EQ(test, res->size, RES_SIZE); + KUNIT_ASSERT_EQ(test, res->mem_type, place->mem_type); + KUNIT_ASSERT_EQ(test, res->placement, place->flags); + KUNIT_ASSERT_PTR_EQ(test, res->bo, bo); + + KUNIT_ASSERT_NULL(test, res->bus.addr); + KUNIT_ASSERT_EQ(test, res->bus.offset, 0); + KUNIT_ASSERT_FALSE(test, res->bus.is_iomem); + KUNIT_ASSERT_EQ(test, res->bus.caching, ttm_cached); + KUNIT_ASSERT_EQ(test, man->usage, expected_usage); + + KUNIT_ASSERT_TRUE(test, list_is_singular(&man->lru[bo->priority])); + + ttm_resource_fini(man, res); +} + +static void ttm_resource_init_pinned(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + + ttm_init_test_mocks(test, priv, TTM_PL_SYSTEM, 0); + bo = priv->bo; + place = priv->place; + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + KUNIT_ASSERT_TRUE(test, list_empty(&bo->bdev->unevictable)); + + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_pin(bo); + ttm_resource_init(bo, place, res); + KUNIT_ASSERT_TRUE(test, list_is_singular(&bo->bdev->unevictable)); + + ttm_bo_unpin(bo); + ttm_resource_fini(man, res); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_TRUE(test, list_empty(&bo->bdev->unevictable)); +} + +static void ttm_resource_fini_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + + ttm_init_test_mocks(test, priv, TTM_PL_SYSTEM, 0); + bo = priv->bo; + place = priv->place; + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + ttm_resource_init(bo, place, res); + ttm_resource_fini(man, res); + + KUNIT_ASSERT_TRUE(test, list_empty(&res->lru.link)); + KUNIT_ASSERT_EQ(test, man->usage, 0); +} + +static void ttm_resource_manager_init_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + size_t size = SZ_16K; + int i; + + man = kunit_kzalloc(test, sizeof(*man), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, man); + + ttm_resource_manager_init(man, priv->devs->ttm_dev, size); + + KUNIT_ASSERT_PTR_EQ(test, man->bdev, priv->devs->ttm_dev); + KUNIT_ASSERT_EQ(test, man->size, size); + KUNIT_ASSERT_EQ(test, man->usage, 0); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) + KUNIT_ASSERT_NULL(test, man->eviction_fences[i]); + + for (int i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[i])); +} + +static void ttm_resource_manager_usage_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource *res; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource_manager *man; + u64 actual_usage; + + ttm_init_test_mocks(test, priv, TTM_PL_SYSTEM, TTM_PL_FLAG_TOPDOWN); + bo = priv->bo; + place = priv->place; + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + man = ttm_manager_type(priv->devs->ttm_dev, place->mem_type); + + ttm_resource_init(bo, place, res); + actual_usage = ttm_resource_manager_usage(man); + + KUNIT_ASSERT_EQ(test, actual_usage, RES_SIZE); + + ttm_resource_fini(man, res); +} + +static void ttm_resource_manager_set_used_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + + man = ttm_manager_type(priv->devs->ttm_dev, TTM_PL_SYSTEM); + KUNIT_ASSERT_TRUE(test, man->use_type); + + ttm_resource_manager_set_used(man, false); + KUNIT_ASSERT_FALSE(test, man->use_type); +} + +static void ttm_sys_man_alloc_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource *res; + u32 mem_type = TTM_PL_SYSTEM; + int ret; + + ttm_init_test_mocks(test, priv, mem_type, 0); + bo = priv->bo; + place = priv->place; + + man = ttm_manager_type(priv->devs->ttm_dev, mem_type); + ret = man->func->alloc(man, bo, place, &res); + + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, res->size, RES_SIZE); + KUNIT_ASSERT_EQ(test, res->mem_type, mem_type); + KUNIT_ASSERT_PTR_EQ(test, res->bo, bo); + + ttm_resource_fini(man, res); +} + +static void ttm_sys_man_free_basic(struct kunit *test) +{ + struct ttm_resource_test_priv *priv = test->priv; + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + struct ttm_place *place; + struct ttm_resource *res; + u32 mem_type = TTM_PL_SYSTEM; + + ttm_init_test_mocks(test, priv, mem_type, 0); + bo = priv->bo; + place = priv->place; + + res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, res); + + ttm_resource_alloc(bo, place, &res, NULL); + + man = ttm_manager_type(priv->devs->ttm_dev, mem_type); + man->func->free(man, res); + + KUNIT_ASSERT_TRUE(test, list_empty(&man->lru[bo->priority])); + KUNIT_ASSERT_EQ(test, man->usage, 0); +} + +static struct kunit_case ttm_resource_test_cases[] = { + KUNIT_CASE_PARAM(ttm_resource_init_basic, ttm_resource_gen_params), + KUNIT_CASE(ttm_resource_init_pinned), + KUNIT_CASE(ttm_resource_fini_basic), + KUNIT_CASE(ttm_resource_manager_init_basic), + KUNIT_CASE(ttm_resource_manager_usage_basic), + KUNIT_CASE(ttm_resource_manager_set_used_basic), + KUNIT_CASE(ttm_sys_man_alloc_basic), + KUNIT_CASE(ttm_sys_man_free_basic), + {} +}; + +static struct kunit_suite ttm_resource_test_suite = { + .name = "ttm_resource", + .init = ttm_resource_test_init, + .exit = ttm_resource_test_fini, + .test_cases = ttm_resource_test_cases, +}; + +kunit_test_suites(&ttm_resource_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_resource and ttm_sys_man APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/tests/ttm_tt_test.c b/drivers/gpu/drm/ttm/tests/ttm_tt_test.c new file mode 100644 index 000000000..bd5f7d0b9 --- /dev/null +++ b/drivers/gpu/drm/ttm/tests/ttm_tt_test.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2023 Intel Corporation + */ +#include <linux/shmem_fs.h> +#include <drm/ttm/ttm_tt.h> + +#include "ttm_kunit_helpers.h" + +#define BO_SIZE SZ_4K + +struct ttm_tt_test_case { + const char *description; + u32 size; + u32 extra_pages_num; +}; + +static const struct ttm_tt_test_case ttm_tt_init_basic_cases[] = { + { + .description = "Page-aligned size", + .size = SZ_4K, + }, + { + .description = "Extra pages requested", + .size = SZ_4K, + .extra_pages_num = 1, + }, +}; + +static void ttm_tt_init_case_desc(const struct ttm_tt_test_case *t, + char *desc) +{ + strscpy(desc, t->description, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ttm_tt_init_basic, ttm_tt_init_basic_cases, + ttm_tt_init_case_desc); + +static void ttm_tt_init_basic(struct kunit *test) +{ + const struct ttm_tt_test_case *params = test->param_value; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + u32 page_flags = TTM_TT_FLAG_ZERO_ALLOC; + enum ttm_caching caching = ttm_cached; + u32 extra_pages = params->extra_pages_num; + int num_pages = params->size >> PAGE_SHIFT; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, params->size, NULL); + + err = ttm_tt_init(tt, bo, page_flags, caching, extra_pages); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_ASSERT_EQ(test, tt->num_pages, num_pages + extra_pages); + + KUNIT_ASSERT_EQ(test, tt->page_flags, page_flags); + KUNIT_ASSERT_EQ(test, tt->caching, caching); + + KUNIT_ASSERT_NULL(test, tt->dma_address); + KUNIT_ASSERT_NULL(test, tt->swap_storage); +} + +static void ttm_tt_init_misaligned(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + u32 size = SZ_8K; + int num_pages = (size + SZ_4K) >> PAGE_SHIFT; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, size, NULL); + + /* Make the object size misaligned */ + bo->base.size += 1; + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + KUNIT_ASSERT_EQ(test, tt->num_pages, num_pages); +} + +static void ttm_tt_fini_basic(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, tt->pages); + + ttm_tt_fini(tt); + KUNIT_ASSERT_NULL(test, tt->pages); +} + +static void ttm_tt_fini_sg(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_sg_tt_init(tt, bo, 0, caching); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, tt->dma_address); + + ttm_tt_fini(tt); + KUNIT_ASSERT_NULL(test, tt->dma_address); +} + +static void ttm_tt_fini_shmem(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + struct file *shmem; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + shmem = shmem_file_setup("ttm swap", BO_SIZE, EMPTY_VMA_FLAGS); + tt->swap_storage = shmem; + + ttm_tt_fini(tt); + KUNIT_ASSERT_NULL(test, tt->swap_storage); +} + +static void ttm_tt_create_basic(struct kunit *test) +{ + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_device; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_NOT_NULL(test, bo->ttm); + + /* Free manually, as it was allocated outside of KUnit */ + kfree(bo->ttm); +} + +static void ttm_tt_create_invalid_bo_type(struct kunit *test) +{ + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + bo->type = ttm_bo_type_sg + 1; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_EXPECT_EQ(test, err, -EINVAL); + KUNIT_EXPECT_NULL(test, bo->ttm); +} + +static void ttm_tt_create_ttm_exists(struct kunit *test) +{ + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + enum ttm_caching caching = ttm_cached; + int err; + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + err = ttm_tt_init(tt, bo, 0, caching, 0); + KUNIT_ASSERT_EQ(test, err, 0); + bo->ttm = tt; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + /* Expect to keep the previous TTM */ + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_PTR_EQ(test, tt, bo->ttm); +} + +static struct ttm_tt *ttm_tt_null_create(struct ttm_buffer_object *bo, + u32 page_flags) +{ + return NULL; +} + +static struct ttm_device_funcs ttm_dev_empty_funcs = { + .ttm_tt_create = ttm_tt_null_create, +}; + +static void ttm_tt_create_failed(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + /* Update ttm_device_funcs so we don't alloc ttm_tt */ + devs->ttm_dev->funcs = &ttm_dev_empty_funcs; + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, -ENOMEM); +} + +static void ttm_tt_destroy_basic(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_buffer_object *bo; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + dma_resv_lock(bo->base.resv, NULL); + err = ttm_tt_create(bo, false); + dma_resv_unlock(bo->base.resv); + + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NOT_NULL(test, bo->ttm); + + ttm_tt_destroy(devs->ttm_dev, bo->ttm); +} + +static void ttm_tt_populate_null_ttm(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_operation_ctx ctx = { }; + int err; + + err = ttm_tt_populate(devs->ttm_dev, NULL, &ctx); + KUNIT_ASSERT_EQ(test, err, -EINVAL); +} + +static void ttm_tt_populate_populated_ttm(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_operation_ctx ctx = { }; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + struct page *populated_page; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + populated_page = *tt->pages; + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_PTR_EQ(test, populated_page, *tt->pages); +} + +static void ttm_tt_unpopulate_basic(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_operation_ctx ctx = { }; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_TRUE(test, ttm_tt_is_populated(tt)); + + ttm_tt_unpopulate(devs->ttm_dev, tt); + KUNIT_ASSERT_FALSE(test, ttm_tt_is_populated(tt)); +} + +static void ttm_tt_unpopulate_empty_ttm(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + ttm_tt_unpopulate(devs->ttm_dev, tt); + /* Expect graceful handling of unpopulated TTs */ +} + +static void ttm_tt_swapin_basic(struct kunit *test) +{ + const struct ttm_test_devices *devs = test->priv; + int expected_num_pages = BO_SIZE >> PAGE_SHIFT; + struct ttm_operation_ctx ctx = { }; + struct ttm_buffer_object *bo; + struct ttm_tt *tt; + int err, num_pages; + + bo = ttm_bo_kunit_init(test, test->priv, BO_SIZE, NULL); + + tt = kunit_kzalloc(test, sizeof(*tt), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tt); + + err = ttm_tt_init(tt, bo, 0, ttm_cached, 0); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_populate(devs->ttm_dev, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_TRUE(test, ttm_tt_is_populated(tt)); + + num_pages = ttm_tt_swapout(devs->ttm_dev, tt, GFP_KERNEL); + KUNIT_ASSERT_EQ(test, num_pages, expected_num_pages); + KUNIT_ASSERT_NOT_NULL(test, tt->swap_storage); + KUNIT_ASSERT_TRUE(test, tt->page_flags & TTM_TT_FLAG_SWAPPED); + + /* Swapout depopulates TT, allocate pages and then swap them in */ + err = ttm_pool_alloc(&devs->ttm_dev->pool, tt, &ctx); + KUNIT_ASSERT_EQ(test, err, 0); + + err = ttm_tt_swapin(tt); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_NULL(test, tt->swap_storage); + KUNIT_ASSERT_FALSE(test, tt->page_flags & TTM_TT_FLAG_SWAPPED); +} + +static struct kunit_case ttm_tt_test_cases[] = { + KUNIT_CASE_PARAM(ttm_tt_init_basic, ttm_tt_init_basic_gen_params), + KUNIT_CASE(ttm_tt_init_misaligned), + KUNIT_CASE(ttm_tt_fini_basic), + KUNIT_CASE(ttm_tt_fini_sg), + KUNIT_CASE(ttm_tt_fini_shmem), + KUNIT_CASE(ttm_tt_create_basic), + KUNIT_CASE(ttm_tt_create_invalid_bo_type), + KUNIT_CASE(ttm_tt_create_ttm_exists), + KUNIT_CASE(ttm_tt_create_failed), + KUNIT_CASE(ttm_tt_destroy_basic), + KUNIT_CASE(ttm_tt_populate_null_ttm), + KUNIT_CASE(ttm_tt_populate_populated_ttm), + KUNIT_CASE(ttm_tt_unpopulate_basic), + KUNIT_CASE(ttm_tt_unpopulate_empty_ttm), + KUNIT_CASE(ttm_tt_swapin_basic), + {} +}; + +static struct kunit_suite ttm_tt_test_suite = { + .name = "ttm_tt", + .init = ttm_test_devices_all_init, + .exit = ttm_test_devices_fini, + .test_cases = ttm_tt_test_cases, +}; + +kunit_test_suites(&ttm_tt_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ttm_tt APIs"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c new file mode 100644 index 000000000..147e25ee5 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + * Keith Packard. + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_resource.h> +#include <linux/agp_backend.h> +#include <linux/export.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <asm/agp.h> + +struct ttm_agp_backend { + struct ttm_tt ttm; + struct agp_memory *mem; + struct agp_bridge_data *bridge; +}; + +int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_resource *bo_mem) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + struct page *dummy_read_page = ttm_glob.dummy_read_page; + struct agp_memory *mem; + int ret, cached = ttm->caching == ttm_cached; + unsigned i; + + if (agp_be->mem) + return 0; + + mem = agp_allocate_memory(agp_be->bridge, ttm->num_pages, AGP_USER_MEMORY); + if (unlikely(mem == NULL)) + return -ENOMEM; + + mem->page_count = 0; + for (i = 0; i < ttm->num_pages; i++) { + struct page *page = ttm->pages[i]; + + if (!page) + page = dummy_read_page; + + mem->pages[mem->page_count++] = page; + } + agp_be->mem = mem; + + mem->is_flushed = 1; + mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; + + ret = agp_bind_memory(mem, bo_mem->start); + if (ret) + pr_err("AGP Bind memory failed\n"); + + return ret; +} +EXPORT_SYMBOL(ttm_agp_bind); + +void ttm_agp_unbind(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) { + if (agp_be->mem->is_bound) { + agp_unbind_memory(agp_be->mem); + return; + } + agp_free_memory(agp_be->mem); + agp_be->mem = NULL; + } +} +EXPORT_SYMBOL(ttm_agp_unbind); + +bool ttm_agp_is_bound(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (!ttm) + return false; + + return (agp_be->mem != NULL); +} +EXPORT_SYMBOL(ttm_agp_is_bound); + +void ttm_agp_destroy(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) + ttm_agp_unbind(ttm); + ttm_tt_fini(ttm); + kfree(agp_be); +} +EXPORT_SYMBOL(ttm_agp_destroy); + +struct ttm_tt *ttm_agp_tt_create(struct ttm_buffer_object *bo, + struct agp_bridge_data *bridge, + uint32_t page_flags) +{ + struct ttm_agp_backend *agp_be; + + agp_be = kmalloc_obj(*agp_be); + if (!agp_be) + return NULL; + + agp_be->mem = NULL; + agp_be->bridge = bridge; + + if (ttm_tt_init(&agp_be->ttm, bo, page_flags, ttm_write_combined, 0)) { + kfree(agp_be); + return NULL; + } + + return &agp_be->ttm; +} +EXPORT_SYMBOL(ttm_agp_tt_create); diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c new file mode 100644 index 000000000..81df4cb56 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include <drm/ttm/ttm_backup.h> + +#include <linux/export.h> +#include <linux/page-flags.h> +#include <linux/swap.h> + +/* + * Need to map shmem indices to handle since a handle value + * of 0 means error, following the swp_entry_t convention. + */ +static unsigned long ttm_backup_shmem_idx_to_handle(pgoff_t idx) +{ + return (unsigned long)idx + 1; +} + +static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle) +{ + return handle - 1; +} + +/** + * ttm_backup_drop() - release memory associated with a handle + * @backup: The struct backup pointer used to obtain the handle + * @handle: The handle obtained from the @backup_page function. + */ +void ttm_backup_drop(struct file *backup, pgoff_t handle) +{ + loff_t start = ttm_backup_handle_to_shmem_idx(handle); + + start <<= PAGE_SHIFT; + shmem_truncate_range(file_inode(backup), start, + start + PAGE_SIZE - 1); +} + +/** + * ttm_backup_copy_page() - Copy the contents of a previously backed + * up page + * @backup: The struct backup pointer used to back up the page. + * @dst: The struct page to copy into. + * @handle: The handle returned when the page was backed up. + * @intr: Try to perform waits interruptible or at least killable. + * @additional_gfp: GFP mask to add to the default GFP mask if any. + * + * Return: 0 on success, Negative error code on failure, notably + * -EINTR if @intr was set to true and a signal is pending. + */ +int ttm_backup_copy_page(struct file *backup, struct page *dst, + pgoff_t handle, bool intr, gfp_t additional_gfp) +{ + struct address_space *mapping = backup->f_mapping; + struct folio *from_folio; + pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle); + + from_folio = shmem_read_folio_gfp(mapping, idx, mapping_gfp_mask(mapping) + | additional_gfp); + if (IS_ERR(from_folio)) + return PTR_ERR(from_folio); + + copy_highpage(dst, folio_file_page(from_folio, idx)); + folio_put(from_folio); + + return 0; +} + +/** + * ttm_backup_backup_page() - Backup a page + * @backup: The struct backup pointer to use. + * @page: The page to back up. + * @writeback: Whether to perform immediate writeback of the page. + * This may have performance implications. + * @idx: A unique integer for each page and each struct backup. + * This allows the backup implementation to avoid managing + * its address space separately. + * @page_gfp: The gfp value used when the page was allocated. + * This is used for accounting purposes. + * @alloc_gfp: The gfp to be used when allocating memory. + * + * Context: If called from reclaim context, the caller needs to + * assert that the shrinker gfp has __GFP_FS set, to avoid + * deadlocking on lock_page(). If @writeback is set to true and + * called from reclaim context, the caller also needs to assert + * that the shrinker gfp has __GFP_IO set, since without it, + * we're not allowed to start backup IO. + * + * Return: A handle on success. Negative error code on failure. + * + * Note: This function could be extended to back up a folio and + * implementations would then split the folio internally if needed. + * Drawback is that the caller would then have to keep track of + * the folio size- and usage. + */ +s64 +ttm_backup_backup_page(struct file *backup, struct page *page, + bool writeback, pgoff_t idx, gfp_t page_gfp, + gfp_t alloc_gfp) +{ + struct address_space *mapping = backup->f_mapping; + unsigned long handle = 0; + struct folio *to_folio; + int ret; + + to_folio = shmem_read_folio_gfp(mapping, idx, alloc_gfp); + if (IS_ERR(to_folio)) + return PTR_ERR(to_folio); + + folio_mark_accessed(to_folio); + folio_lock(to_folio); + folio_mark_dirty(to_folio); + copy_highpage(folio_file_page(to_folio, idx), page); + handle = ttm_backup_shmem_idx_to_handle(idx); + + if (writeback && !folio_mapped(to_folio) && + folio_clear_dirty_for_io(to_folio)) { + folio_set_reclaim(to_folio); + ret = shmem_writeout(to_folio, NULL, NULL); + if (!folio_test_writeback(to_folio)) + folio_clear_reclaim(to_folio); + /* + * If writeout succeeds, it unlocks the folio. errors + * are otherwise dropped, since writeout is only best + * effort here. + */ + if (ret) + folio_unlock(to_folio); + } else { + folio_unlock(to_folio); + } + + folio_put(to_folio); + + return handle; +} + +/** + * ttm_backup_fini() - Free the struct backup resources after last use. + * @backup: Pointer to the struct backup whose resources to free. + * + * After a call to this function, it's illegal to use the @backup pointer. + */ +void ttm_backup_fini(struct file *backup) +{ + fput(backup); +} + +/** + * ttm_backup_bytes_avail() - Report the approximate number of bytes of backup space + * left for backup. + * + * This function is intended also for driver use to indicate whether a + * backup attempt is meaningful. + * + * Return: An approximate size of backup space available. + */ +u64 ttm_backup_bytes_avail(void) +{ + /* + * The idea behind backing up to shmem is that shmem objects may + * eventually be swapped out. So no point swapping out if there + * is no or low swap-space available. But the accuracy of this + * number also depends on shmem actually swapping out backed-up + * shmem objects without too much buffering. + */ + return (u64)get_nr_swap_pages() << PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail); + +/** + * ttm_backup_shmem_create() - Create a shmem-based struct backup. + * @size: The maximum size (in bytes) to back up. + * + * Create a backup utilizing shmem objects. + * + * Return: A pointer to a struct file on success, + * an error pointer on error. + */ +struct file *ttm_backup_shmem_create(loff_t size) +{ + return shmem_file_setup("ttm shmem backup", size, + EMPTY_VMA_FLAGS); +} diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c new file mode 100644 index 000000000..bcd76f6bb --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -0,0 +1,1300 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/drm_print.h> +#include <drm/drm_util.h> +#include <drm/ttm/ttm_allocation.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include <linux/export.h> +#include <linux/jiffies.h> +#include <linux/slab.h> +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/file.h> +#include <linux/module.h> +#include <linux/atomic.h> +#include <linux/cgroup_dmem.h> +#include <linux/dma-resv.h> + +#include "ttm_module.h" +#include "ttm_bo_internal.h" + +static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX); + struct ttm_resource_manager *man; + int i, mem_type; + + for (i = 0; i < placement->num_placement; i++) { + mem_type = placement->placement[i].mem_type; + drm_printf(&p, " placement[%d]=0x%08X (%d)\n", + i, placement->placement[i].flags, mem_type); + man = ttm_manager_type(bo->bdev, mem_type); + ttm_resource_manager_debug(man, &p); + } +} + +/** + * ttm_bo_move_to_lru_tail + * + * @bo: The buffer object. + * + * Move this BO to the tail of all lru lists used to lookup and reserve an + * object. This function must be called with struct ttm_global::lru_lock + * held, and is used to make a BO less likely to be considered for eviction. + */ +void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + + if (bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); +} +EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); + +/** + * ttm_bo_set_bulk_move - update BOs bulk move object + * + * @bo: The buffer object. + * @bulk: bulk move structure + * + * Update the BOs bulk move object, making sure that resources are added/removed + * as well. A bulk move allows to move many resource on the LRU at once, + * resulting in much less overhead of maintaining the LRU. + * The only requirement is that the resources stay together on the LRU and are + * never separated. This is enforces by setting the bulk_move structure on a BO. + * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of + * their LRU list. + */ +void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, + struct ttm_lru_bulk_move *bulk) +{ + dma_resv_assert_held(bo->base.resv); + + if (bo->bulk_move == bulk) + return; + + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + bo->bulk_move = bulk; + if (bo->resource) + ttm_resource_add_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_set_bulk_move); + +static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + struct ttm_resource *mem, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) +{ + struct ttm_device *bdev = bo->bdev; + bool old_use_tt, new_use_tt; + int ret; + + old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; + new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; + + ttm_bo_unmap_virtual(bo); + + /* + * Create and bind a ttm if required. + */ + + if (new_use_tt) { + /* Zero init the new TTM structure if the old location should + * have used one as well. + */ + ret = ttm_tt_create(bo, old_use_tt); + if (ret) + goto out_err; + + if (mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_bo_populate(bo, ctx); + if (ret) + goto out_err; + } + } + + ret = dma_resv_reserve_fences(bo->base.resv, 1); + if (ret) + goto out_err; + + ret = bdev->funcs->move(bo, evict, ctx, mem, hop); + if (ret) { + if (ret == -EMULTIHOP) + return ret; + goto out_err; + } + + ctx->bytes_moved += bo->base.size; + return 0; + +out_err: + if (!old_use_tt) + ttm_bo_tt_destroy(bo); + + return ret; +} + +/* + * Call bo::reserved. + * Will release GPU memory type usage on destruction. + * This is the place to put in driver specific hooks to release + * driver private resources. + * Will release the bo::reserved lock. + */ + +static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) +{ + if (bo->bdev->funcs->delete_mem_notify) + bo->bdev->funcs->delete_mem_notify(bo); + + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->resource); +} + +static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) +{ + int r; + + if (bo->base.resv == &bo->base._resv) + return 0; + + BUG_ON(!dma_resv_trylock(&bo->base._resv)); + + r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); + dma_resv_unlock(&bo->base._resv); + if (r) + return r; + + if (bo->type != ttm_bo_type_sg) { + /* This works because the BO is about to be destroyed and nobody + * reference it any more. The only tricky case is the trylock on + * the resv object while holding the lru_lock. + */ + spin_lock(&bo->bdev->lru_lock); + bo->base.resv = &bo->base._resv; + spin_unlock(&bo->bdev->lru_lock); + } + + return r; +} + +static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) +{ + struct dma_resv *resv = &bo->base._resv; + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) + dma_fence_enable_sw_signaling(fence); + dma_resv_iter_end(&cursor); +} + +/* + * Block for the dma_resv object to become idle, lock the buffer and clean up + * the resource and tt object. + */ +static void ttm_bo_delayed_delete(struct work_struct *work) +{ + struct ttm_buffer_object *bo; + + bo = container_of(work, typeof(*bo), delayed_delete); + + dma_resv_wait_timeout(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); +} + +static void ttm_bo_release(struct kref *kref) +{ + struct ttm_buffer_object *bo = + container_of(kref, struct ttm_buffer_object, kref); + struct ttm_device *bdev = bo->bdev; + int ret; + + WARN_ON_ONCE(bo->pin_count); + WARN_ON_ONCE(bo->bulk_move); + + if (!bo->deleted) { + ret = ttm_bo_individualize_resv(bo); + if (ret) { + /* Last resort, if we fail to allocate memory for the + * fences block for the BO to become idle + */ + dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, false, + 30 * HZ); + } + + if (bdev->funcs->release_notify) + bdev->funcs->release_notify(bo); + + drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); + ttm_mem_io_free(bdev, bo->resource); + + if (!dma_resv_test_signaled(&bo->base._resv, + DMA_RESV_USAGE_BOOKKEEP) || + (want_init_on_free() && (bo->ttm != NULL)) || + bo->type == ttm_bo_type_sg || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; + + spin_lock(&bdev->lru_lock); + + /* + * Make pinned bos immediately available to + * shrinkers, now that they are queued for + * destruction. + * + * FIXME: QXL is triggering this. Can be removed when the + * driver is fixed. + */ + if (bo->pin_count) { + bo->pin_count = 0; + ttm_resource_move_to_lru_tail(bo->resource); + } + + kref_init(&bo->kref); + spin_unlock(&bdev->lru_lock); + + INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); + + /* Schedule the worker on the closest NUMA node. This + * improves performance since system memory might be + * cleared on free and that is best done on a CPU core + * close to it. + */ + queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete); + return; + } + + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + } + + atomic_dec(&ttm_glob.bo_count); + bo->destroy(bo); +} + +/* TODO: remove! */ +void ttm_bo_put(struct ttm_buffer_object *bo) +{ + kref_put(&bo->kref, ttm_bo_release); +} + +void ttm_bo_fini(struct ttm_buffer_object *bo) +{ + ttm_bo_put(bo); +} +EXPORT_SYMBOL(ttm_bo_fini); + +static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) +{ + struct ttm_placement hop_placement; + struct ttm_resource *hop_mem; + int ret; + + hop_placement.num_placement = 1; + hop_placement.placement = hop; + + /* find space in the bounce domain */ + ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); + if (ret) + return ret; + /* move to the bounce domain */ + ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); + if (ret) { + ttm_resource_free(bo, &hop_mem); + return ret; + } + return 0; +} + +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + struct ttm_resource *evict_mem; + struct ttm_placement placement; + struct ttm_place hop; + int ret = 0; + + memset(&hop, 0, sizeof(hop)); + + dma_resv_assert_held(bo->base.resv); + + placement.num_placement = 0; + bo->bdev->funcs->evict_flags(bo, &placement); + + if (!placement.num_placement) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + return ret; + + /* + * Since we've already synced, this frees backing store + * immediately. + */ + return ttm_bo_pipeline_gutting(bo); + } + + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); + if (ret) { + if (ret != -ERESTARTSYS) { + pr_err("Failed to find memory space for buffer 0x%p eviction\n", + bo); + ttm_bo_mem_space_debug(bo, &placement); + } + goto out; + } + + do { + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret != -EMULTIHOP) + break; + + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + } while (!ret); + + if (ret) { + ttm_resource_free(bo, &evict_mem); + if (ret != -ERESTARTSYS && ret != -EINTR) + pr_err("Buffer eviction failed\n"); + } +out: + return ret; +} + +/** + * ttm_bo_eviction_valuable + * + * @bo: The buffer object to evict + * @place: the placement we need to make room for + * + * Check if it is valuable to evict the BO to make room for the given placement. + */ +bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, + const struct ttm_place *place) +{ + struct ttm_resource *res = bo->resource; + + dma_resv_assert_held(bo->base.resv); + + if (res->mem_type == TTM_PL_SYSTEM) + return true; + + /* Don't evict this BO if it's outside of the + * requested placement range + */ + return ttm_resource_intersects(bo->bdev, res, place, bo->base.size); +} +EXPORT_SYMBOL(ttm_bo_eviction_valuable); + +/** + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. + * @bdev: The ttm device. + * @man: The manager whose bo to evict. + * @ctx: The TTM operation ctx governing the eviction. + * + * Return: 0 if successful or the resource disappeared. Negative error code on error. + */ +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx) +{ + struct ttm_resource_cursor cursor; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + unsigned int mem_type; + int ret = 0; + + spin_lock(&bdev->lru_lock); + ttm_resource_cursor_init(&cursor, man); + res = ttm_resource_manager_first(&cursor); + ttm_resource_cursor_fini(&cursor); + if (!res) { + ret = -ENOENT; + goto out_no_ref; + } + bo = res->bo; + if (!ttm_bo_get_unless_zero(bo)) + goto out_no_ref; + mem_type = res->mem_type; + spin_unlock(&bdev->lru_lock); + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); + if (ret) + goto out_no_lock; + if (!bo->resource || bo->resource->mem_type != mem_type) + goto out_bo_moved; + + if (bo->deleted) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (!ret) + ttm_bo_cleanup_memtype_use(bo); + } else { + ret = ttm_bo_evict(bo, ctx); + } +out_bo_moved: + dma_resv_unlock(bo->base.resv); +out_no_lock: + ttm_bo_put(bo); + return ret; + +out_no_ref: + spin_unlock(&bdev->lru_lock); + return ret; +} + +/** + * struct ttm_bo_evict_walk - Parameters for the evict walk. + */ +struct ttm_bo_evict_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @place: The place passed to the resource allocation. */ + const struct ttm_place *place; + /** @evictor: The buffer object we're trying to make room for. */ + struct ttm_buffer_object *evictor; + /** @res: The allocated resource if any. */ + struct ttm_resource **res; + /** @evicted: Number of successful evictions. */ + unsigned long evicted; + + /** @limit_pool: Which pool limit we should test against */ + struct dmem_cgroup_pool_state *limit_pool; + /** @try_low: Whether we should attempt to evict BO's with low watermark threshold */ + bool try_low; + /** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */ + bool hit_low; +}; + +static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) +{ + struct ttm_bo_evict_walk *evict_walk = + container_of(walk, typeof(*evict_walk), walk); + s64 lret; + + if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css, + evict_walk->try_low, &evict_walk->hit_low)) + return 0; + + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) + return 0; + + if (bo->deleted) { + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); + if (!lret) + ttm_bo_cleanup_memtype_use(bo); + } else { + lret = ttm_bo_evict(bo, walk->arg.ctx); + } + + if (lret) + goto out; + + evict_walk->evicted++; + if (evict_walk->res) + lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, + evict_walk->res, NULL); + if (lret == 0) + return 1; +out: + /* Errors that should terminate the walk. */ + if (lret == -ENOSPC) + return -EBUSY; + + return lret; +} + +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { + .process_bo = ttm_bo_evict_cb, +}; + +static int ttm_bo_evict_alloc(struct ttm_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_buffer_object *evictor, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket, + struct ttm_resource **res, + struct dmem_cgroup_pool_state *limit_pool) +{ + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { + .ctx = ctx, + .ticket = ticket, + } + }, + .place = place, + .evictor = evictor, + .res = res, + .limit_pool = limit_pool, + }; + s64 lret; + + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + + /* One more attempt if we hit low limit? */ + if (!lret && evict_walk.hit_low) { + evict_walk.try_low = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } + if (lret || !ticket) + goto out; + + /* Reset low limit */ + evict_walk.try_low = evict_walk.hit_low = false; + /* If ticket-locking, repeat while making progress. */ + evict_walk.walk.arg.trylock_only = false; + +retry: + do { + /* The walk may clear the evict_walk.walk.ticket field */ + evict_walk.walk.arg.ticket = ticket; + evict_walk.evicted = 0; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } while (!lret && evict_walk.evicted); + + /* We hit the low limit? Try once more */ + if (!lret && evict_walk.hit_low && !evict_walk.try_low) { + evict_walk.try_low = true; + goto retry; + } +out: + if (lret < 0) + return lret; + if (lret == 0) + return -EBUSY; + return 0; +} + +/** + * ttm_bo_pin - Pin the buffer object. + * @bo: The buffer object to pin + * + * Make sure the buffer is not evicted any more during memory pressure. + * @bo must be unpinned again by calling ttm_bo_unpin(). + */ +void ttm_bo_pin(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + if (!bo->pin_count++ && bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_pin); + +/** + * ttm_bo_unpin - Unpin the buffer object. + * @bo: The buffer object to unpin + * + * Allows the buffer object to be evicted again during memory pressure. + */ +void ttm_bo_unpin(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + if (WARN_ON_ONCE(!bo->pin_count)) + return; + + spin_lock(&bo->bdev->lru_lock); + if (!--bo->pin_count && bo->resource) { + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + } + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_unpin); + +/* + * Add the pipelined eviction fencesto the BO as kernel dependency and reserve new + * fence slots. + */ +static int ttm_bo_add_pipelined_eviction_fences(struct ttm_buffer_object *bo, + struct ttm_resource_manager *man, + bool no_wait_gpu) +{ + struct dma_fence *fence; + int i; + + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (!fence) + continue; + + if (no_wait_gpu) { + if (!dma_fence_is_signaled(fence)) { + spin_unlock(&man->eviction_lock); + return -EBUSY; + } + } else { + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + } + } + spin_unlock(&man->eviction_lock); + + /* TODO: this call should be removed. */ + return dma_resv_reserve_fences(bo->base.resv, 1); +} + +/** + * ttm_bo_alloc_resource - Allocate backing store for a BO + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @ctx: if and how to sleep, lock buffers and alloc memory + * @force_space: If we should evict buffers to force space + * @res: The resulting struct ttm_resource. + * + * Allocates a resource for the buffer object pointed to by @bo, using the + * placement flags in @placement, potentially evicting other buffer objects when + * @force_space is true. + * This function may sleep while waiting for resources to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == true). + * -ENOSPC: Could not allocate space for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. + */ +static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx, + bool force_space, + struct ttm_resource **res) +{ + struct ttm_device *bdev = bo->bdev; + struct ww_acquire_ctx *ticket; + int i, ret; + + ticket = dma_resv_locking_ctx(bo->base.resv); + ret = dma_resv_reserve_fences(bo->base.resv, TTM_NUM_MOVE_FENCES); + if (unlikely(ret)) + return ret; + + for (i = 0; i < placement->num_placement; ++i) { + const struct ttm_place *place = &placement->placement[i]; + struct dmem_cgroup_pool_state *limit_pool = NULL; + struct ttm_resource_manager *man; + bool may_evict; + + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) + continue; + + if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) + continue; + + may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); + ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL); + if (ret) { + if (ret != -ENOSPC) { + dmem_cgroup_pool_state_put(limit_pool); + return ret; + } + if (!may_evict) { + dmem_cgroup_pool_state_put(limit_pool); + continue; + } + + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, + ticket, res, limit_pool); + dmem_cgroup_pool_state_put(limit_pool); + if (ret == -EBUSY) + continue; + if (ret) + return ret; + } + + ret = ttm_bo_add_pipelined_eviction_fences(bo, man, ctx->no_wait_gpu); + if (unlikely(ret)) { + ttm_resource_free(bo, res); + if (ret == -EBUSY) + continue; + + return ret; + } + return 0; + } + + return -ENOSPC; +} + +/* + * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @res: The resulting struct ttm_resource. + * @ctx: if and how to sleep, lock buffers and alloc memory + * + * Tries both idle allocation and forcefully eviction of buffers. See + * ttm_bo_alloc_resource for details. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_resource **res, + struct ttm_operation_ctx *ctx) +{ + bool force_space = false; + int ret; + + do { + ret = ttm_bo_alloc_resource(bo, placement, ctx, + force_space, res); + force_space = !force_space; + } while (ret == -ENOSPC && force_space); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_mem_space); + +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @ctx: validation parameters. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ +int ttm_bo_validate(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) +{ + struct ttm_resource *res; + struct ttm_place hop; + bool force_space; + int ret; + + dma_resv_assert_held(bo->base.resv); + + /* + * Remove the backing store if no placement is given. + */ + if (!placement->num_placement) + return ttm_bo_pipeline_gutting(bo); + + force_space = false; + do { + /* Check whether we need to move buffer. */ + if (bo->resource && + ttm_resource_compatible(bo->resource, placement, + force_space)) + return 0; + + /* Moving of pinned BOs is forbidden */ + if (bo->pin_count) + return -EINVAL; + + /* + * Determine where to move the buffer. + * + * If driver determines move is going to need + * an extra step then it will return -EMULTIHOP + * and the buffer will be moved to the temporary + * stop and the driver will be called to make + * the second hop. + */ + ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space, + &res); + force_space = !force_space; + if (ret == -ENOSPC) + continue; + if (ret) + return ret; + +bounce: + ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop); + if (ret == -EMULTIHOP) { + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + /* try and move to final place now. */ + if (!ret) + goto bounce; + } + if (ret) { + ttm_resource_free(bo, &res); + return ret; + } + + } while (ret && force_space); + + /* For backward compatibility with userspace */ + if (ret == -ENOSPC) + return bo->bdev->alloc_flags & TTM_ALLOCATION_PROPAGATE_ENOSPC ? + ret : -ENOMEM; + + /* + * We might need to add a TTM. + */ + if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { + ret = ttm_tt_create(bo, true); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL(ttm_bo_validate); + +/** + * ttm_bo_init_reserved + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @ctx: TTM operation context for memory allocation. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, enables driver-specific objects + * derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * Furthermore, if resv == NULL, the buffer's reservation lock will be held, + * and it is the caller's responsibility to call ttm_bo_unreserve. + * + * If a failure occurs, the function will call the @destroy function. Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, struct ttm_operation_ctx *ctx, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) +{ + int ret; + + kref_init(&bo->kref); + bo->bdev = bdev; + bo->type = type; + bo->page_alignment = alignment; + bo->destroy = destroy; + bo->pin_count = 0; + bo->sg = sg; + bo->bulk_move = NULL; + if (resv) + bo->base.resv = resv; + else + bo->base.resv = &bo->base._resv; + atomic_inc(&ttm_glob.bo_count); + + /* + * For ttm_bo_type_device buffers, allocate + * address space from the device. + */ + if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { + ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, + PFN_UP(bo->base.size)); + if (ret) + goto err_put; + } + + /* passed reservation objects should already be locked, + * since otherwise lockdep will be angered in radeon. + */ + if (!resv) + WARN_ON(!dma_resv_trylock(bo->base.resv)); + else + dma_resv_assert_held(resv); + + ret = ttm_bo_validate(bo, placement, ctx); + if (unlikely(ret)) + goto err_unlock; + + return 0; + +err_unlock: + if (!resv) + dma_resv_unlock(bo->base.resv); + +err_put: + ttm_bo_put(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_init_reserved); + +/** + * ttm_bo_init_validate + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @interruptible: If needing to sleep to wait for GPU resources, + * sleep interruptible. + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, + * enables driver-specific objects derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * + * If a failure occurs, the function will call the @destroy function, Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, bool interruptible, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) +{ + struct ttm_operation_ctx ctx = { .interruptible = interruptible }; + int ret; + + ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, + sg, resv, destroy); + if (ret) + return ret; + + if (!resv) + ttm_bo_unreserve(bo); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_init_validate); + +/* + * buffer object vm functions. + */ + +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ +void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) +{ + struct ttm_device *bdev = bo->bdev; + + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); + ttm_mem_io_free(bdev, bo->resource); +} +EXPORT_SYMBOL(ttm_bo_unmap_virtual); + +/** + * ttm_bo_wait_ctx - wait for buffer idle. + * + * @bo: The buffer object. + * @ctx: defines how to wait + * + * Waits for the buffer to be idle. Used timeout depends on the context. + * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or + * zero on success. + */ +int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) +{ + long ret; + + if (ctx->no_wait_gpu) { + if (dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + return 0; + else + return -EBUSY; + } + + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, 15 * HZ); + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret == 0)) + return -EBUSY; + return 0; +} +EXPORT_SYMBOL(ttm_bo_wait_ctx); + +/** + * struct ttm_bo_swapout_walk - Parameters for the swapout walk + */ +struct ttm_bo_swapout_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ + gfp_t gfp_flags; + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ + bool hit_low, evict_low; +}; + +static s64 +ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) +{ + struct ttm_place place = { .mem_type = bo->resource->mem_type }; + struct ttm_bo_swapout_walk *swapout_walk = + container_of(walk, typeof(*swapout_walk), walk); + struct ttm_operation_ctx *ctx = walk->arg.ctx; + struct ttm_device *bdev = bo->bdev; + struct ttm_tt *tt = bo->ttm; + s64 ret; + + /* + * While the bo may already reside in SYSTEM placement, set + * SYSTEM as new placement to cover also the move further below. + * The driver may use the fact that we're moving from SYSTEM + * as an indication that we're about to swap out. + */ + if (bo->pin_count || !bdev->funcs->eviction_valuable(bo, &place)) { + ret = -EBUSY; + goto out; + } + + if (!tt || !ttm_tt_is_populated(tt) || + tt->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_SWAPPED)) { + ret = -EBUSY; + goto out; + } + + if (bo->deleted) { + pgoff_t num_pages = tt->num_pages; + + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; + + ttm_bo_cleanup_memtype_use(bo); + ret = num_pages; + goto out; + } + + /* + * Move to system cached + */ + if (bo->resource->mem_type != TTM_PL_SYSTEM) { + struct ttm_resource *evict_mem; + struct ttm_place hop; + + memset(&hop, 0, sizeof(hop)); + place.mem_type = TTM_PL_SYSTEM; + ret = ttm_resource_alloc(bo, &place, &evict_mem, NULL); + if (ret) + goto out; + + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret) { + WARN(ret == -EMULTIHOP, + "Unexpected multihop in swapout - likely driver bug.\n"); + ttm_resource_free(bo, &evict_mem); + goto out; + } + } + + /* + * Make sure BO is idle. + */ + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; + + ttm_bo_unmap_virtual(bo); + if (bdev->funcs->swap_notify) + bdev->funcs->swap_notify(bo); + + if (ttm_tt_is_populated(tt)) { + ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags); + if (!ret) { + spin_lock(&bdev->lru_lock); + ttm_resource_del_bulk_move_unevictable(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } + } + +out: + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; + + return ret; +} + +const struct ttm_lru_walk_ops ttm_swap_ops = { + .process_bo = ttm_bo_swapout_cb, +}; + +/** + * ttm_bo_swapout() - Swap out buffer objects on the LRU list to shmem. + * @bdev: The ttm device. + * @ctx: The ttm_operation_ctx governing the swapout operation. + * @man: The resource manager whose resources / buffer objects are + * goint to be swapped out. + * @gfp_flags: The gfp flags used for shmem page allocations. + * @target: The desired number of pages to swap out. + * + * Return: The number of pages actually swapped out, or negative error code + * on error. + */ +s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + struct ttm_resource_manager *man, gfp_t gfp_flags, + s64 target) +{ + struct ttm_bo_swapout_walk swapout_walk = { + .walk = { + .ops = &ttm_swap_ops, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, + }, + .gfp_flags = gfp_flags, + }; + + return ttm_lru_walk_for_evict(&swapout_walk.walk, bdev, man, target); +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_bo_swapout); + +void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) +{ + if (bo->ttm == NULL) + return; + + ttm_tt_unpopulate(bo->bdev, bo->ttm); + ttm_tt_destroy(bo->bdev, bo->ttm); + bo->ttm = NULL; +} + +/** + * ttm_bo_populate() - Ensure that a buffer object has backing pages + * @bo: The buffer object + * @ctx: The ttm_operation_ctx governing the operation. + * + * For buffer objects in a memory type whose manager uses + * struct ttm_tt for backing pages, ensure those backing pages + * are present and with valid content. The bo's resource is also + * placed on the correct LRU list if it was previously swapped + * out. + * + * Return: 0 if successful, negative error code on failure. + * Note: May return -EINTR or -ERESTARTSYS if @ctx::interruptible + * is set to true. + */ +int ttm_bo_populate(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + struct ttm_device *bdev = bo->bdev; + struct ttm_tt *tt = bo->ttm; + bool swapped; + int ret; + + dma_resv_assert_held(bo->base.resv); + + if (!tt) + return 0; + + swapped = ttm_tt_is_swapped(tt); + ret = ttm_tt_populate(bdev, tt, ctx); + if (ret) + return ret; + + if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count && + bo->resource) { + spin_lock(&bdev->lru_lock); + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_populate); + +int ttm_bo_setup_export(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + int ret; + + ret = ttm_bo_reserve(bo, false, false, NULL); + if (ret != 0) + return ret; + + ret = ttm_bo_populate(bo, ctx); + ttm_bo_unreserve(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_setup_export); diff --git a/drivers/gpu/drm/ttm/ttm_bo_internal.h b/drivers/gpu/drm/ttm/ttm_bo_internal.h new file mode 100644 index 000000000..e0d48eac7 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_internal.h @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ + +#ifndef _TTM_BO_INTERNAL_H_ +#define _TTM_BO_INTERNAL_H_ + +#include <drm/ttm/ttm_bo.h> + +/** + * ttm_bo_get - reference a struct ttm_buffer_object + * + * @bo: The buffer object. + */ +static inline void ttm_bo_get(struct ttm_buffer_object *bo) +{ + kref_get(&bo->kref); +} + +/** + * ttm_bo_get_unless_zero - reference a struct ttm_buffer_object unless + * its refcount has already reached zero. + * @bo: The buffer object. + * + * Used to reference a TTM buffer object in lookups where the object is removed + * from the lookup structure during the destructor and for RCU lookups. + * + * Returns: @bo if the referencing was successful, NULL otherwise. + */ +static inline __must_check struct ttm_buffer_object * +ttm_bo_get_unless_zero(struct ttm_buffer_object *bo) +{ + if (!kref_get_unless_zero(&bo->kref)) + return NULL; + return bo; +} + +void ttm_bo_put(struct ttm_buffer_object *bo); + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c new file mode 100644 index 000000000..3e3c201a0 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -0,0 +1,1166 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#include <linux/export.h> +#include <linux/swap.h> +#include <linux/vmalloc.h> + +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include <drm/drm_cache.h> + +#include "ttm_bo_internal.h" + +struct ttm_transfer_obj { + struct ttm_buffer_object base; + struct ttm_buffer_object *bo; +}; + +int ttm_mem_io_reserve(struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (mem->bus.offset || mem->bus.addr) + return 0; + + mem->bus.is_iomem = false; + if (!bdev->funcs->io_mem_reserve) + return 0; + + return bdev->funcs->io_mem_reserve(bdev, mem); +} + +void ttm_mem_io_free(struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (!mem) + return; + + if (!mem->bus.offset && !mem->bus.addr) + return; + + if (bdev->funcs->io_mem_free) + bdev->funcs->io_mem_free(bdev, mem); + + mem->bus.offset = 0; + mem->bus.addr = NULL; +} + +/** + * ttm_move_memcpy - Helper to perform a memcpy ttm move operation. + * @clear: Whether to clear rather than copy. + * @num_pages: Number of pages of the operation. + * @dst_iter: A struct ttm_kmap_iter representing the destination resource. + * @src_iter: A struct ttm_kmap_iter representing the source resource. + * + * This function is intended to be able to move out async under a + * dma-fence if desired. + */ +void ttm_move_memcpy(bool clear, + u32 num_pages, + struct ttm_kmap_iter *dst_iter, + struct ttm_kmap_iter *src_iter) +{ + const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops; + const struct ttm_kmap_iter_ops *src_ops = src_iter->ops; + struct iosys_map src_map, dst_map; + pgoff_t i; + + /* Single TTM move. NOP */ + if (dst_ops->maps_tt && src_ops->maps_tt) + return; + + /* Don't move nonexistent data. Clear destination instead. */ + if (clear) { + for (i = 0; i < num_pages; ++i) { + dst_ops->map_local(dst_iter, &dst_map, i); + if (dst_map.is_iomem) + memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE); + else + memset(dst_map.vaddr, 0, PAGE_SIZE); + if (dst_ops->unmap_local) + dst_ops->unmap_local(dst_iter, &dst_map); + } + return; + } + + for (i = 0; i < num_pages; ++i) { + dst_ops->map_local(dst_iter, &dst_map, i); + src_ops->map_local(src_iter, &src_map, i); + + drm_memcpy_from_wc(&dst_map, &src_map, PAGE_SIZE); + + if (src_ops->unmap_local) + src_ops->unmap_local(src_iter, &src_map); + if (dst_ops->unmap_local) + dst_ops->unmap_local(dst_iter, &dst_map); + } +} +EXPORT_SYMBOL(ttm_move_memcpy); + +/** + * ttm_bo_move_memcpy + * + * @bo: A pointer to a struct ttm_buffer_object. + * @ctx: operation context + * @dst_mem: struct ttm_resource indicating where to move. + * + * Fallback move function for a mappable buffer object in mappable memory. + * The function will, if successful, + * free any old aperture space, and set (@new_mem)->mm_node to NULL, + * and update the (@bo)->mem placement flags. If unsuccessful, the old + * data remains untouched, and it's up to the caller to free the + * memory space indicated by @new_mem. + * Returns: + * !0: Failure. + */ +int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_resource *dst_mem) +{ + struct ttm_device *bdev = bo->bdev; + struct ttm_resource_manager *dst_man = + ttm_manager_type(bo->bdev, dst_mem->mem_type); + struct ttm_tt *ttm = bo->ttm; + struct ttm_resource *src_mem = bo->resource; + struct ttm_resource_manager *src_man; + union { + struct ttm_kmap_iter_tt tt; + struct ttm_kmap_iter_linear_io io; + } _dst_iter, _src_iter; + struct ttm_kmap_iter *dst_iter, *src_iter; + bool clear; + int ret = 0; + + if (WARN_ON(!src_mem)) + return -EINVAL; + + src_man = ttm_manager_type(bdev, src_mem->mem_type); + if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) || + dst_man->use_tt)) { + ret = ttm_bo_populate(bo, ctx); + if (ret) + return ret; + } + + dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem); + if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt) + dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, ttm); + if (IS_ERR(dst_iter)) + return PTR_ERR(dst_iter); + + src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem); + if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt) + src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, ttm); + if (IS_ERR(src_iter)) { + ret = PTR_ERR(src_iter); + goto out_src_iter; + } + + clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm)); + if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) + ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter); + + if (!src_iter->ops->maps_tt) + ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, src_mem); + ttm_bo_move_sync_cleanup(bo, dst_mem); + +out_src_iter: + if (!dst_iter->ops->maps_tt) + ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_move_memcpy); + +static void ttm_transfered_destroy(struct ttm_buffer_object *bo) +{ + struct ttm_transfer_obj *fbo; + + fbo = container_of(bo, struct ttm_transfer_obj, base); + dma_resv_fini(&fbo->base.base._resv); + ttm_bo_put(fbo->bo); + kfree(fbo); +} + +/** + * ttm_buffer_object_transfer + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, + * holding the data of @bo with the old placement. + * + * This is a utility function that may be called after an accelerated move + * has been scheduled. A new buffer object is created as a placeholder for + * the old data while it's being copied. When that buffer object is idle, + * it can be destroyed, releasing the space of the old placement. + * Returns: + * !0: Failure. + */ + +static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + struct ttm_buffer_object **new_obj) +{ + struct ttm_transfer_obj *fbo; + int ret; + + fbo = kmalloc_obj(*fbo); + if (!fbo) + return -ENOMEM; + + fbo->base = *bo; + + /** + * Fix up members that we shouldn't copy directly: + * TODO: Explicit member copy would probably be better here. + */ + + atomic_inc(&ttm_glob.bo_count); + drm_vma_node_reset(&fbo->base.base.vma_node); + + kref_init(&fbo->base.kref); + fbo->base.destroy = &ttm_transfered_destroy; + fbo->base.pin_count = 0; + if (bo->type != ttm_bo_type_sg) + fbo->base.base.resv = &fbo->base.base._resv; + + dma_resv_init(&fbo->base.base._resv); + fbo->base.base.dev = NULL; + ret = dma_resv_trylock(&fbo->base.base._resv); + WARN_ON(!ret); + + ret = dma_resv_reserve_fences(&fbo->base.base._resv, TTM_NUM_MOVE_FENCES); + if (ret) { + dma_resv_unlock(&fbo->base.base._resv); + kfree(fbo); + return ret; + } + + if (fbo->base.resource) { + ttm_resource_set_bo(fbo->base.resource, &fbo->base); + bo->resource = NULL; + ttm_bo_set_bulk_move(&fbo->base, NULL); + } else { + fbo->base.bulk_move = NULL; + } + + ttm_bo_get(bo); + fbo->bo = bo; + + ttm_bo_move_to_lru_tail_unlocked(&fbo->base); + + *new_obj = &fbo->base; + return 0; +} + +/** + * ttm_io_prot + * + * @bo: ttm buffer object + * @res: ttm resource object + * @tmp: Page protection flag for a normal, cached mapping. + * + * Utility function that returns the pgprot_t that should be used for + * setting up a PTE with the caching model indicated by @c_state. + */ +pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, + pgprot_t tmp) +{ + struct ttm_resource_manager *man; + enum ttm_caching caching; + + man = ttm_manager_type(bo->bdev, res->mem_type); + if (man->use_tt) { + caching = bo->ttm->caching; + if (bo->ttm->page_flags & TTM_TT_FLAG_DECRYPTED) + tmp = pgprot_decrypted(tmp); + } else { + caching = res->bus.caching; + } + + return ttm_prot_from_caching(caching, tmp); +} +EXPORT_SYMBOL(ttm_io_prot); + +static int ttm_bo_ioremap(struct ttm_buffer_object *bo, + unsigned long offset, + unsigned long size, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *mem = bo->resource; + + if (mem->bus.addr) { + map->bo_kmap_type = ttm_bo_map_premapped; + map->virtual = ((u8 *)mem->bus.addr) + offset; + } else { + resource_size_t res = mem->bus.offset + offset; + + map->bo_kmap_type = ttm_bo_map_iomap; + if (mem->bus.caching == ttm_write_combined) + map->virtual = ioremap_wc(res, size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + map->virtual = ioremap_cache(res, size); +#endif + else + map->virtual = ioremap(res, size); + } + return (!map->virtual) ? -ENOMEM : 0; +} + +static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, + unsigned long start_page, + unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *mem = bo->resource; + struct ttm_operation_ctx ctx = { }; + struct ttm_tt *ttm = bo->ttm; + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, mem->mem_type); + pgprot_t prot; + int ret; + + BUG_ON(!ttm); + + ret = ttm_bo_populate(bo, &ctx); + if (ret) + return ret; + + if (num_pages == 1 && ttm->caching == ttm_cached && + !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) { + /* + * We're mapping a single page, and the desired + * page protection is consistent with the bo. + */ + + map->bo_kmap_type = ttm_bo_map_kmap; + map->page = ttm->pages[start_page]; + map->virtual = kmap(map->page); + } else { + /* + * We need to use vmap to get the desired page protection + * or to make the buffer object look contiguous. + */ + prot = ttm_io_prot(bo, mem, PAGE_KERNEL); + map->bo_kmap_type = ttm_bo_map_vmap; + map->virtual = vmap(ttm->pages + start_page, num_pages, + 0, prot); + } + return (!map->virtual) ? -ENOMEM : 0; +} + +/** + * ttm_bo_kmap_try_from_panic + * + * @bo: The buffer object + * @page: The page to map + * + * Sets up a kernel virtual mapping using kmap_local_page_try_from_panic(). + * This should only be called from the panic handler, if you make sure the bo + * is the one being displayed, so is properly allocated, and protected. + * + * Returns the vaddr, that you can use to write to the bo, and that you should + * pass to kunmap_local() when you're done with this page, or NULL if the bo + * is in iomem. + */ +void *ttm_bo_kmap_try_from_panic(struct ttm_buffer_object *bo, unsigned long page) +{ + if (page + 1 > PFN_UP(bo->resource->size)) + return NULL; + + if (!bo->resource->bus.is_iomem && bo->ttm->pages && bo->ttm->pages[page]) + return kmap_local_page_try_from_panic(bo->ttm->pages[page]); + + return NULL; +} +EXPORT_SYMBOL(ttm_bo_kmap_try_from_panic); + +/** + * ttm_bo_kmap + * + * @bo: The buffer object. + * @start_page: The first page to map. + * @num_pages: Number of pages to map. + * @map: pointer to a struct ttm_bo_kmap_obj representing the map. + * + * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the + * data in the buffer object. The ttm_kmap_obj_virtual function can then be + * used to obtain a virtual address to the data. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ +int ttm_bo_kmap(struct ttm_buffer_object *bo, + unsigned long start_page, unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *res = bo->resource; + unsigned long offset, size; + int ret; + + map->virtual = NULL; + map->bo = bo; + if (num_pages > PFN_UP(res->size)) + return -EINVAL; + if ((start_page + num_pages) > PFN_UP(res->size)) + return -EINVAL; + + ret = ttm_mem_io_reserve(bo->bdev, res); + if (ret) + return ret; + if (!res->bus.is_iomem) { + return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); + } else { + offset = start_page << PAGE_SHIFT; + size = num_pages << PAGE_SHIFT; + return ttm_bo_ioremap(bo, offset, size, map); + } +} +EXPORT_SYMBOL(ttm_bo_kmap); + +/** + * ttm_bo_kunmap + * + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_kmap. + */ +void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) +{ + if (!map->virtual) + return; + switch (map->bo_kmap_type) { + case ttm_bo_map_iomap: + iounmap(map->virtual); + break; + case ttm_bo_map_vmap: + vunmap(map->virtual); + break; + case ttm_bo_map_kmap: + kunmap(map->page); + break; + case ttm_bo_map_premapped: + break; + default: + BUG(); + } + ttm_mem_io_free(map->bo->bdev, map->bo->resource); + map->virtual = NULL; + map->page = NULL; +} +EXPORT_SYMBOL(ttm_bo_kunmap); + +/** + * ttm_bo_vmap + * + * @bo: The buffer object. + * @map: pointer to a struct iosys_map representing the map. + * + * Sets up a kernel virtual mapping, using ioremap or vmap to the + * data in the buffer object. The parameter @map returns the virtual + * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap(). + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ +int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) +{ + struct ttm_resource *mem = bo->resource; + int ret; + + dma_resv_assert_held(bo->base.resv); + + ret = ttm_mem_io_reserve(bo->bdev, mem); + if (ret) + return ret; + + if (mem->bus.is_iomem) { + void __iomem *vaddr_iomem; + + if (mem->bus.addr) + vaddr_iomem = (void __iomem *)mem->bus.addr; + else if (mem->bus.caching == ttm_write_combined) + vaddr_iomem = ioremap_wc(mem->bus.offset, + bo->base.size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + vaddr_iomem = ioremap_cache(mem->bus.offset, + bo->base.size); +#endif + else + vaddr_iomem = ioremap(mem->bus.offset, bo->base.size); + + if (!vaddr_iomem) + return -ENOMEM; + + iosys_map_set_vaddr_iomem(map, vaddr_iomem); + + } else { + struct ttm_operation_ctx ctx = { }; + struct ttm_tt *ttm = bo->ttm; + pgprot_t prot; + void *vaddr; + + ret = ttm_bo_populate(bo, &ctx); + if (ret) + return ret; + + /* + * We need to use vmap to get the desired page protection + * or to make the buffer object look contiguous. + */ + prot = ttm_io_prot(bo, mem, PAGE_KERNEL); + vaddr = vmap(ttm->pages, ttm->num_pages, 0, prot); + if (!vaddr) + return -ENOMEM; + + iosys_map_set_vaddr(map, vaddr); + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_vmap); + +/** + * ttm_bo_vunmap + * + * @bo: The buffer object. + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_vmap(). + */ +void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map) +{ + struct ttm_resource *mem = bo->resource; + + dma_resv_assert_held(bo->base.resv); + + if (iosys_map_is_null(map)) + return; + + if (!map->is_iomem) + vunmap(map->vaddr); + else if (!mem->bus.addr) + iounmap(map->vaddr_iomem); + iosys_map_clear(map); + + ttm_mem_io_free(bo->bdev, mem); +} +EXPORT_SYMBOL(ttm_bo_vunmap); + +static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, + bool dst_use_tt) +{ + long ret; + + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, 15 * HZ); + if (ret == 0) + return -EBUSY; + if (ret < 0) + return ret; + + if (!dst_use_tt) + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->resource); + return 0; +} + +static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, + struct dma_fence *fence, + bool dst_use_tt) +{ + struct ttm_buffer_object *ghost_obj; + int ret; + + /** + * This should help pipeline ordinary buffer moves. + * + * Hang old buffer memory on a new buffer object, + * and leave it to be released when the GPU + * operation has completed. + */ + + ret = ttm_buffer_object_transfer(bo, &ghost_obj); + if (ret) + return ret; + + dma_resv_add_fence(&ghost_obj->base._resv, fence, + DMA_RESV_USAGE_KERNEL); + + /** + * If we're not moving to fixed memory, the TTM object + * needs to stay alive. Otherwhise hang it on the ghost + * bo to be unbound and destroyed. + */ + + if (dst_use_tt) + ghost_obj->ttm = NULL; + else + bo->ttm = NULL; + + dma_resv_unlock(&ghost_obj->base._resv); + ttm_bo_put(ghost_obj); + return 0; +} + +static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, + struct dma_fence *fence) +{ + struct ttm_resource_manager *from; + struct dma_fence *tmp; + int i; + + from = ttm_manager_type(bo->bdev, bo->resource->mem_type); + + /** + * BO doesn't have a TTM we need to bind/unbind. Just remember + * this eviction and free up the allocation. + * The fence will be saved in the first free slot or in the slot + * already used to store a fence from the same context. Since + * drivers can't use more than TTM_NUM_MOVE_FENCES contexts for + * evictions we should always find a slot to use. + */ + spin_lock(&from->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + tmp = from->eviction_fences[i]; + if (!tmp) + break; + if (fence->context != tmp->context) + continue; + if (dma_fence_is_later(fence, tmp)) { + dma_fence_put(tmp); + break; + } + goto unlock; + } + if (i < TTM_NUM_MOVE_FENCES) { + from->eviction_fences[i] = dma_fence_get(fence); + } else { + WARN(1, "not enough fence slots for all fence contexts"); + spin_unlock(&from->eviction_lock); + dma_fence_wait(fence, false); + goto end; + } + +unlock: + spin_unlock(&from->eviction_lock); +end: + ttm_resource_free(bo, &bo->resource); +} + +/** + * ttm_bo_move_accel_cleanup - cleanup helper for hw copies + * + * @bo: A pointer to a struct ttm_buffer_object. + * @fence: A fence object that signals when moving is complete. + * @evict: This is an evict move. Don't return until the buffer is idle. + * @pipeline: evictions are to be pipelined. + * @new_mem: struct ttm_resource indicating where to move. + * + * Accelerated move function to be called when an accelerated move + * has been scheduled. The function will create a new temporary buffer object + * representing the old placement, and put the sync object on both buffer + * objects. After that the newly created buffer object is unref'd to be + * destroyed when the move is complete. This will help pipeline + * buffer moves. + */ +int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, + struct dma_fence *fence, + bool evict, + bool pipeline, + struct ttm_resource *new_mem) +{ + struct ttm_device *bdev = bo->bdev; + struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->resource->mem_type); + struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); + int ret = 0; + + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + if (!evict) + ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt); + else if (!from->use_tt && pipeline) + ttm_bo_move_pipeline_evict(bo, fence); + else + ret = ttm_bo_wait_free_node(bo, man->use_tt); + + if (ret) + return ret; + + ttm_bo_assign_mem(bo, new_mem); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); + +/** + * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_mem: struct ttm_resource indicating where to move. + * + * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed + * by the caller to be idle. Typically used after memcpy buffer moves. + */ +void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo, + struct ttm_resource *new_mem) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, new_mem->mem_type); + int ret; + + ret = ttm_bo_wait_free_node(bo, man->use_tt); + if (WARN_ON(ret)) + return; + + ttm_bo_assign_mem(bo, new_mem); +} +EXPORT_SYMBOL(ttm_bo_move_sync_cleanup); + +/** + * ttm_bo_pipeline_gutting - purge the contents of a bo + * @bo: The buffer object + * + * Purge the contents of a bo, async if the bo is not idle. + * After a successful call, the bo is left unpopulated in + * system placement. The function may wait uninterruptible + * for idle on OOM. + * + * Return: 0 if successful, negative error code on failure. + */ +int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) +{ + struct ttm_buffer_object *ghost; + struct ttm_tt *ttm; + int ret; + + /* If already idle, no need for ghost object dance. */ + if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) { + if (!bo->ttm) { + /* See comment below about clearing. */ + ret = ttm_tt_create(bo, true); + if (ret) + return ret; + } else { + ttm_tt_unpopulate(bo->bdev, bo->ttm); + if (bo->type == ttm_bo_type_device) + ttm_tt_mark_for_clear(bo->ttm); + } + ttm_resource_free(bo, &bo->resource); + return 0; + } + + /* + * We need an unpopulated ttm_tt after giving our current one, + * if any, to the ghost object. And we can't afford to fail + * creating one *after* the operation. If the bo subsequently gets + * resurrected, make sure it's cleared (if ttm_bo_type_device) + * to avoid leaking sensitive information to user-space. + */ + + ttm = bo->ttm; + bo->ttm = NULL; + ret = ttm_tt_create(bo, true); + swap(bo->ttm, ttm); + if (ret) + return ret; + + ret = ttm_buffer_object_transfer(bo, &ghost); + if (ret) + goto error_destroy_tt; + + ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); + /* Last resort, wait for the BO to be idle when we are OOM */ + if (ret) { + dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + } + + dma_resv_unlock(&ghost->base._resv); + ttm_bo_put(ghost); + bo->ttm = ttm; + return 0; + +error_destroy_tt: + ttm_tt_destroy(bo->bdev, ttm); + return ret; +} + +static bool ttm_lru_walk_trylock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) +{ + struct ttm_operation_ctx *ctx = curs->arg->ctx; + + curs->needs_unlock = false; + + if (dma_resv_trylock(bo->base.resv)) { + curs->needs_unlock = true; + return true; + } + + if (bo->base.resv == ctx->resv && ctx->allow_res_evict) { + dma_resv_assert_held(bo->base.resv); + return true; + } + + return false; +} + +static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) +{ + struct ttm_lru_walk_arg *arg = curs->arg; + int ret; + + if (arg->ctx->interruptible) + ret = dma_resv_lock_interruptible(bo->base.resv, arg->ticket); + else + ret = dma_resv_lock(bo->base.resv, arg->ticket); + + if (!ret) { + curs->needs_unlock = true; + /* + * Only a single ticketlock per loop. Ticketlocks are prone + * to return -EDEADLK causing the eviction to fail, so + * after waiting for the ticketlock, revert back to + * trylocking for this walk. + */ + arg->ticket = NULL; + } else if (ret == -EDEADLK) { + /* Caller needs to exit the ww transaction. */ + ret = -ENOSPC; + } + + return ret; +} + +/** + * ttm_lru_walk_for_evict() - Perform a LRU list walk, with actions taken on + * valid items. + * @walk: describe the walks and actions taken + * @bdev: The TTM device. + * @man: The struct ttm_resource manager whose LRU lists we're walking. + * @target: The end condition for the walk. + * + * The LRU lists of @man are walk, and for each struct ttm_resource encountered, + * the corresponding ttm_buffer_object is locked and taken a reference on, and + * the LRU lock is dropped. the LRU lock may be dropped before locking and, in + * that case, it's verified that the item actually remains on the LRU list after + * the lock, and that the buffer object didn't switch resource in between. + * + * With a locked object, the actions indicated by @walk->process_bo are + * performed, and after that, the bo is unlocked, the refcount dropped and the + * next struct ttm_resource is processed. Here, the walker relies on + * TTM's restartable LRU list implementation. + * + * Typically @walk->process_bo() would return the number of pages evicted, + * swapped or shrunken, so that when the total exceeds @target, or when the + * LRU list has been walked in full, iteration is terminated. It's also terminated + * on error. Note that the definition of @target is done by the caller, it + * could have a different meaning than the number of pages. + * + * Note that the way dma_resv individualization is done, locking needs to be done + * either with the LRU lock held (trylocking only) or with a reference on the + * object. + * + * Return: The progress made towards target or negative error code on error. + */ +s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev, + struct ttm_resource_manager *man, s64 target) +{ + struct ttm_bo_lru_cursor cursor; + struct ttm_buffer_object *bo; + s64 progress = 0; + s64 lret; + + ttm_bo_lru_for_each_reserved_guarded(&cursor, man, &walk->arg, bo) { + lret = walk->ops->process_bo(walk, bo); + if (lret == -EBUSY || lret == -EALREADY) + lret = 0; + progress = (lret < 0) ? lret : progress + lret; + if (progress < 0 || progress >= target) + break; + } + if (IS_ERR(bo)) + return PTR_ERR(bo); + + return progress; +} +EXPORT_SYMBOL(ttm_lru_walk_for_evict); + +static void ttm_bo_lru_cursor_cleanup_bo(struct ttm_bo_lru_cursor *curs) +{ + struct ttm_buffer_object *bo = curs->bo; + + if (bo) { + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); + curs->bo = NULL; + } +} + +/** + * ttm_bo_lru_cursor_fini() - Stop using a struct ttm_bo_lru_cursor + * and clean up any iteration it was used for. + * @curs: The cursor. + */ +void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs) +{ + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + + ttm_bo_lru_cursor_cleanup_bo(curs); + spin_lock(lru_lock); + ttm_resource_cursor_fini(&curs->res_curs); + spin_unlock(lru_lock); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_fini); + +/** + * ttm_bo_lru_cursor_init() - Initialize a struct ttm_bo_lru_cursor + * @curs: The ttm_bo_lru_cursor to initialize. + * @man: The ttm resource_manager whose LRU lists to iterate over. + * @arg: The ttm_lru_walk_arg to govern the walk. + * + * Initialize a struct ttm_bo_lru_cursor. + * + * Return: Pointer to @curs. The function does not fail. + */ +struct ttm_bo_lru_cursor * +ttm_bo_lru_cursor_init(struct ttm_bo_lru_cursor *curs, + struct ttm_resource_manager *man, + struct ttm_lru_walk_arg *arg) +{ + memset(curs, 0, sizeof(*curs)); + ttm_resource_cursor_init(&curs->res_curs, man); + curs->arg = arg; + + return curs; +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_init); + +static struct ttm_buffer_object * +__ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) +{ + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + struct ttm_resource *res = NULL; + struct ttm_buffer_object *bo; + struct ttm_lru_walk_arg *arg = curs->arg; + bool first = !curs->bo; + + ttm_bo_lru_cursor_cleanup_bo(curs); + + spin_lock(lru_lock); + for (;;) { + int mem_type, ret = 0; + bool bo_locked = false; + + if (first) { + res = ttm_resource_manager_first(&curs->res_curs); + first = false; + } else { + res = ttm_resource_manager_next(&curs->res_curs); + } + if (!res) + break; + + bo = res->bo; + if (ttm_lru_walk_trylock(curs, bo)) + bo_locked = true; + else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only) + continue; + + if (!ttm_bo_get_unless_zero(bo)) { + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); + continue; + } + + mem_type = res->mem_type; + spin_unlock(lru_lock); + if (!bo_locked) + ret = ttm_lru_walk_ticketlock(curs, bo); + + /* + * Note that in between the release of the lru lock and the + * ticketlock, the bo may have switched resource, + * and also memory type, since the resource may have been + * freed and allocated again with a different memory type. + * In that case, just skip it. + */ + curs->bo = bo; + if (!ret && bo->resource && bo->resource->mem_type == mem_type) + return bo; + + ttm_bo_lru_cursor_cleanup_bo(curs); + if (ret && ret != -EALREADY) + return ERR_PTR(ret); + + spin_lock(lru_lock); + } + + spin_unlock(lru_lock); + return res ? bo : NULL; +} + +/** + * ttm_bo_lru_cursor_next() - Continue iterating a manager's LRU lists + * to find and lock buffer object. + * @curs: The cursor initialized using ttm_bo_lru_cursor_init() and + * ttm_bo_lru_cursor_first(). + * + * Return: A pointer to a locked and reference-counted buffer object, + * or NULL if none could be found and looping should be terminated. + */ +struct ttm_buffer_object *ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) +{ + return __ttm_bo_lru_cursor_next(curs); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_next); + +/** + * ttm_bo_lru_cursor_first() - Start iterating a manager's LRU lists + * to find and lock buffer object. + * @curs: The cursor initialized using ttm_bo_lru_cursor_init(). + * + * Return: A pointer to a locked and reference-counted buffer object, + * or NULL if none could be found and looping should be terminated. + */ +struct ttm_buffer_object *ttm_bo_lru_cursor_first(struct ttm_bo_lru_cursor *curs) +{ + ttm_bo_lru_cursor_cleanup_bo(curs); + return __ttm_bo_lru_cursor_next(curs); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_first); + +/** + * ttm_bo_shrink() - Helper to shrink a ttm buffer object. + * @ctx: The struct ttm_operation_ctx used for the shrinking operation. + * @bo: The buffer object. + * @flags: Flags governing the shrinking behaviour. + * + * The function uses the ttm_tt_back_up functionality to back up or + * purge a struct ttm_tt. If the bo is not in system, it's first + * moved there. + * + * Return: The number of pages shrunken or purged, or + * negative error code on failure. + */ +long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, + const struct ttm_bo_shrink_flags flags) +{ + static const struct ttm_place sys_placement_flags = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_SYSTEM, + .flags = 0, + }; + static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_placement_flags, + }; + struct ttm_device *bdev = bo->bdev; + long lret; + + dma_resv_assert_held(bo->base.resv); + + if (flags.allow_move && bo->resource->mem_type != TTM_PL_SYSTEM) { + int ret = ttm_bo_validate(bo, &sys_placement, ctx); + + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret) { + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; + return ret; + } + } + + ttm_bo_unmap_virtual(bo); + lret = ttm_bo_wait_ctx(bo, ctx); + if (lret < 0) + return lret; + + lret = ttm_tt_backup(bdev, bo->ttm, (struct ttm_backup_flags) + {.purge = flags.purge, + .writeback = flags.writeback}); + + if (lret > 0) { + spin_lock(&bdev->lru_lock); + ttm_resource_del_bulk_move_unevictable(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } + + if (lret < 0 && lret != -EINTR) + return -EBUSY; + + return lret; +} +EXPORT_SYMBOL(ttm_bo_shrink); + +/** + * ttm_bo_shrink_suitable() - Whether a bo is suitable for shinking + * @ctx: The struct ttm_operation_ctx governing the shrinking. + * @bo: The candidate for shrinking. + * + * Check whether the object, given the information available to TTM, + * is suitable for shinking, This function can and should be used + * before attempting to shrink an object. + * + * Return: true if suitable. false if not. + */ +bool ttm_bo_shrink_suitable(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) +{ + return bo->ttm && ttm_tt_is_populated(bo->ttm) && !bo->pin_count && + (!ctx->no_wait_gpu || + dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)); +} +EXPORT_SYMBOL(ttm_bo_shrink_suitable); + +/** + * ttm_bo_shrink_avoid_wait() - Whether to avoid waiting for GPU + * during shrinking + * + * In some situations, like direct reclaim, waiting (in particular gpu waiting) + * should be avoided since it may stall a system that could otherwise make progress + * shrinking something else less time consuming. + * + * Return: true if gpu waiting should be avoided, false if not. + */ +bool ttm_bo_shrink_avoid_wait(void) +{ + return !current_is_kswapd(); +} +EXPORT_SYMBOL(ttm_bo_shrink_avoid_wait); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c new file mode 100644 index 000000000..a80510489 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -0,0 +1,513 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include <linux/export.h> + +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include <drm/drm_drv.h> +#include <drm/drm_managed.h> + +static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + struct vm_fault *vmf) +{ + long err = 0; + + /* + * Quick non-stalling check for idle. + */ + if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_KERNEL)) + return 0; + + /* + * If possible, avoid waiting for GPU with mmap_lock + * held. We only do this if the fault allows retry and this + * is the first attempt. + */ + if (fault_flag_allow_retry_first(vmf->flags)) { + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + return VM_FAULT_RETRY; + + drm_gem_object_get(&bo->base); + mmap_read_unlock(vmf->vma->vm_mm); + (void)dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_KERNEL, true, + MAX_SCHEDULE_TIMEOUT); + dma_resv_unlock(bo->base.resv); + drm_gem_object_put(&bo->base); + return VM_FAULT_RETRY; + } + + /* + * Ordinary wait. + */ + err = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_KERNEL, true, + MAX_SCHEDULE_TIMEOUT); + if (unlikely(err < 0)) { + return (err != -ERESTARTSYS) ? VM_FAULT_SIGBUS : + VM_FAULT_NOPAGE; + } + + return 0; +} + +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_device *bdev = bo->bdev; + + if (bdev->funcs->io_mem_pfn) + return bdev->funcs->io_mem_pfn(bo, page_offset); + + return (bo->resource->bus.offset >> PAGE_SHIFT) + page_offset; +} + +/** + * ttm_bo_vm_reserve - Reserve a buffer object in a retryable vm callback + * @bo: The buffer object + * @vmf: The fault structure handed to the callback + * + * vm callbacks like fault() and *_mkwrite() allow for the mmap_lock to be dropped + * during long waits, and after the wait the callback will be restarted. This + * is to allow other threads using the same virtual memory space concurrent + * access to map(), unmap() completely unrelated buffer objects. TTM buffer + * object reservations sometimes wait for GPU and should therefore be + * considered long waits. This function reserves the buffer object interruptibly + * taking this into account. Starvation is avoided by the vm system not + * allowing too many repeated restarts. + * This function is intended to be used in customized fault() and _mkwrite() + * handlers. + * + * Return: + * 0 on success and the bo was reserved. + * VM_FAULT_RETRY if blocking wait. + * VM_FAULT_NOPAGE if blocking wait and retrying was not allowed. + */ +vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, + struct vm_fault *vmf) +{ + /* + * Work around locking order reversal in fault / nopfn + * between mmap_lock and bo_reserve: Perform a trylock operation + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. + */ + if (unlikely(!dma_resv_trylock(bo->base.resv))) { + /* + * If the fault allows retry and this is the first + * fault attempt, we try to release the mmap_lock + * before waiting + */ + if (fault_flag_allow_retry_first(vmf->flags)) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + drm_gem_object_get(&bo->base); + mmap_read_unlock(vmf->vma->vm_mm); + if (!dma_resv_lock_interruptible(bo->base.resv, + NULL)) + dma_resv_unlock(bo->base.resv); + drm_gem_object_put(&bo->base); + } + + return VM_FAULT_RETRY; + } + + if (dma_resv_lock_interruptible(bo->base.resv, NULL)) + return VM_FAULT_NOPAGE; + } + + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + if (!(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE)) { + dma_resv_unlock(bo->base.resv); + return VM_FAULT_SIGBUS; + } + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_vm_reserve); + +/** + * ttm_bo_vm_fault_reserved - TTM fault helper + * @vmf: The struct vm_fault given as argument to the fault callback + * @prot: The page protection to be used for this memory area. + * @num_prefault: Maximum number of prefault pages. The caller may want to + * specify this based on madvice settings and the size of the GPU object + * backed by the memory. + * + * This function inserts one or more page table entries pointing to the + * memory backing the buffer object, and then returns a return code + * instructing the caller to retry the page access. + * + * Return: + * VM_FAULT_NOPAGE on success or pending signal + * VM_FAULT_SIGBUS on unspecified error + * VM_FAULT_OOM on out-of-memory + * VM_FAULT_RETRY if retryable wait + */ +vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, + pgprot_t prot, + pgoff_t num_prefault) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct page *page; + int err; + pgoff_t i; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address = vmf->address; + + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + ret = ttm_bo_vm_fault_idle(bo, vmf); + if (unlikely(ret != 0)) + return ret; + + err = ttm_mem_io_reserve(bo->bdev, bo->resource); + if (unlikely(err != 0)) + return VM_FAULT_SIGBUS; + + page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + + vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); + page_last = vma_pages(vma) + vma->vm_pgoff - + drm_vma_node_start(&bo->base.vma_node); + + if (unlikely(page_offset >= PFN_UP(bo->base.size))) + return VM_FAULT_SIGBUS; + + prot = ttm_io_prot(bo, bo->resource, prot); + if (!bo->resource->bus.is_iomem) { + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + }; + + ttm = bo->ttm; + err = ttm_bo_populate(bo, &ctx); + if (err) { + if (err == -EINTR || err == -ERESTARTSYS || + err == -EAGAIN) + return VM_FAULT_NOPAGE; + + pr_debug("TTM fault hit %pe.\n", ERR_PTR(err)); + return VM_FAULT_SIGBUS; + } + } else { + /* Iomem should not be marked encrypted */ + prot = pgprot_decrypted(prot); + } + + /* + * Speculatively prefault a number of pages. Only error on + * first page. + */ + for (i = 0; i < num_prefault; ++i) { + if (bo->resource->bus.is_iomem) { + pfn = ttm_bo_io_mem_pfn(bo, page_offset); + } else { + page = ttm->pages[page_offset]; + if (unlikely(!page && i == 0)) { + return VM_FAULT_OOM; + } else if (unlikely(!page)) { + break; + } + pfn = page_to_pfn(page); + } + + /* + * Note that the value of @prot at this point may differ from + * the value of @vma->vm_page_prot in the caching- and + * encryption bits. This is because the exact location of the + * data may not be known at mmap() time and may also change + * at arbitrary times while the data is mmap'ed. + * See vmf_insert_pfn_prot() for a discussion. + */ + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + + /* Never error on prefaulted PTEs */ + if (unlikely((ret & VM_FAULT_ERROR))) { + if (i == 0) + return VM_FAULT_NOPAGE; + else + break; + } + + address += PAGE_SIZE; + if (unlikely(++page_offset >= page_last)) + break; + } + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); + +static void ttm_bo_release_dummy_page(struct drm_device *dev, void *res) +{ + struct page *dummy_page = (struct page *)res; + + __free_page(dummy_page); +} + +vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address; + unsigned long pfn; + struct page *page; + + /* Allocate new dummy page to map all the VA range in this VMA to it*/ + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + /* Set the page to be freed using drmm release action */ + if (drmm_add_action_or_reset(bo->base.dev, ttm_bo_release_dummy_page, + page)) + return VM_FAULT_OOM; + + pfn = page_to_pfn(page); + + /* Prefault the entire VMA range right away to avoid further faults */ + for (address = vma->vm_start; address < vma->vm_end; + address += PAGE_SIZE) + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_dummy_page); + +vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + vm_fault_t ret; + pgprot_t prot; + int idx; + + ret = ttm_bo_vm_reserve(bo, vmf); + if (ret) + return ret; + + prot = vma->vm_page_prot; + if (drm_dev_enter(bo->base.dev, &idx)) { + ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); + drm_dev_exit(idx); + } else { + ret = ttm_bo_vm_dummy_page(vmf, prot); + } + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return ret; + + dma_resv_unlock(bo->base.resv); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_fault); + +void ttm_bo_vm_open(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + + WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); + + drm_gem_object_get(&bo->base); +} +EXPORT_SYMBOL(ttm_bo_vm_open); + +void ttm_bo_vm_close(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + + drm_gem_object_put(&bo->base); + vma->vm_private_data = NULL; +} +EXPORT_SYMBOL(ttm_bo_vm_close); + +static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo, + unsigned long offset, + uint8_t *buf, int len, int write) +{ + unsigned long page = offset >> PAGE_SHIFT; + unsigned long bytes_left = len; + int ret; + + /* Copy a page at a time, that way no extra virtual address + * mapping is needed + */ + offset -= page << PAGE_SHIFT; + do { + unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); + struct ttm_bo_kmap_obj map; + void *ptr; + bool is_iomem; + + ret = ttm_bo_kmap(bo, page, 1, &map); + if (ret) + return ret; + + ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset; + WARN_ON_ONCE(is_iomem); + if (write) + memcpy(ptr, buf, bytes); + else + memcpy(buf, ptr, bytes); + ttm_bo_kunmap(&map); + + page++; + buf += bytes; + bytes_left -= bytes; + offset = 0; + } while (bytes_left); + + return len; +} + +/** + * ttm_bo_access - Helper to access a buffer object + * + * @bo: ttm buffer object + * @offset: access offset into buffer object + * @buf: pointer to caller memory to read into or write from + * @len: length of access + * @write: write access + * + * Utility function to access a buffer object. Useful when buffer object cannot + * be easily mapped (non-contiguous, non-visible, etc...). Should not directly + * be exported to user space via a peak / poke interface. + * + * Returns: + * @len if successful, negative error code on failure. + */ +int ttm_bo_access(struct ttm_buffer_object *bo, unsigned long offset, + void *buf, int len, int write) +{ + int ret; + + if (len < 1 || (offset + len) > bo->base.size) + return -EIO; + + ret = ttm_bo_reserve(bo, true, false, NULL); + if (ret) + return ret; + + if (!bo->resource) { + ret = -ENODATA; + goto unlock; + } + + switch (bo->resource->mem_type) { + case TTM_PL_SYSTEM: + fallthrough; + case TTM_PL_TT: + ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write); + break; + default: + if (bo->bdev->funcs->access_memory) + ret = bo->bdev->funcs->access_memory + (bo, offset, buf, len, write); + else + ret = -EIO; + } + +unlock: + ttm_bo_unreserve(bo); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_access); + +int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + unsigned long offset = (addr) - vma->vm_start + + ((vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node)) + << PAGE_SHIFT); + + return ttm_bo_access(bo, offset, buf, len, write); +} +EXPORT_SYMBOL(ttm_bo_vm_access); + +static const struct vm_operations_struct ttm_bo_vm_ops = { + .fault = ttm_bo_vm_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access, +}; + +/** + * ttm_bo_mmap_obj - mmap memory backed by a ttm buffer object. + * + * @vma: vma as input from the fbdev mmap method. + * @bo: The bo backing the address space. + * + * Maps a buffer object. + */ +int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) +{ + /* Enforce no COW since would have really strange behavior with it. */ + if (is_cow_mapping(vma->vm_flags)) + return -EINVAL; + + drm_gem_object_get(&bo->base); + + /* + * Drivers may want to override the vm_ops field. Otherwise we + * use TTM's default callbacks. + */ + if (!vma->vm_ops) + vma->vm_ops = &ttm_bo_vm_ops; + + /* + * Note: We're transferring the bo reference to + * vma->vm_private_data here. + */ + + vma->vm_private_data = bo; + + vm_flags_set(vma, VM_PFNMAP | VM_IO | VM_DONTEXPAND | VM_DONTDUMP); + return 0; +} +EXPORT_SYMBOL(ttm_bo_mmap_obj); diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c new file mode 100644 index 000000000..d3bfb9a69 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -0,0 +1,319 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +/* + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#define pr_fmt(fmt) "[TTM DEVICE] " fmt + +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/mm.h> + +#include <drm/ttm/ttm_allocation.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_placement.h> + +#include "ttm_module.h" +#include "ttm_bo_internal.h" + +/* + * ttm_global_mutex - protecting the global state + */ +static DEFINE_MUTEX(ttm_global_mutex); +static unsigned ttm_glob_use_count; +struct ttm_global ttm_glob; +EXPORT_SYMBOL(ttm_glob); + +struct dentry *ttm_debugfs_root; + +static void ttm_global_release(void) +{ + struct ttm_global *glob = &ttm_glob; + + mutex_lock(&ttm_global_mutex); + if (--ttm_glob_use_count > 0) + goto out; + + ttm_pool_mgr_fini(); + debugfs_remove(ttm_debugfs_root); + + __free_page(glob->dummy_read_page); + memset(glob, 0, sizeof(*glob)); +out: + mutex_unlock(&ttm_global_mutex); +} + +static int ttm_global_init(void) +{ + struct ttm_global *glob = &ttm_glob; + unsigned long num_pages, num_dma32; + struct sysinfo si; + int ret = 0; + + mutex_lock(&ttm_global_mutex); + if (++ttm_glob_use_count > 1) + goto out; + + si_meminfo(&si); + + ttm_debugfs_root = debugfs_create_dir("ttm", NULL); + if (IS_ERR(ttm_debugfs_root)) { + ttm_debugfs_root = NULL; + } + + /* Limit the number of pages in the pool to about 50% of the total + * system memory. + */ + num_pages = ((u64)si.totalram * si.mem_unit) >> PAGE_SHIFT; + num_pages /= 2; + + /* But for DMA32 we limit ourself to only use 2GiB maximum. */ + num_dma32 = (u64)(si.totalram - si.totalhigh) * si.mem_unit + >> PAGE_SHIFT; + num_dma32 = min(num_dma32, 2UL << (30 - PAGE_SHIFT)); + + ttm_pool_mgr_init(num_pages); + ttm_tt_mgr_init(num_pages, num_dma32); + + glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32 | + __GFP_NOWARN); + + /* Retry without GFP_DMA32 for platforms DMA32 is not available */ + if (unlikely(glob->dummy_read_page == NULL)) { + glob->dummy_read_page = alloc_page(__GFP_ZERO); + if (unlikely(glob->dummy_read_page == NULL)) { + ret = -ENOMEM; + goto out; + } + pr_warn("Using GFP_DMA32 fallback for dummy_read_page\n"); + } + + INIT_LIST_HEAD(&glob->device_list); + atomic_set(&glob->bo_count, 0); + + debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root, + &glob->bo_count); +out: + if (ret && ttm_debugfs_root) + debugfs_remove(ttm_debugfs_root); + if (ret) + --ttm_glob_use_count; + mutex_unlock(&ttm_global_mutex); + return ret; +} + +/** + * ttm_device_prepare_hibernation - move GTT BOs to shmem for hibernation. + * + * @bdev: A pointer to a struct ttm_device to prepare hibernation for. + * + * Return: 0 on success, negative number on failure. + */ +int ttm_device_prepare_hibernation(struct ttm_device *bdev) +{ + struct ttm_operation_ctx ctx = { }; + int ret; + + do { + ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL); + } while (ret > 0); + return ret; +} +EXPORT_SYMBOL(ttm_device_prepare_hibernation); + +/* + * A buffer object shrink method that tries to swap out the first + * buffer object on the global::swap_lru list. + */ +int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags) +{ + struct ttm_global *glob = &ttm_glob; + struct ttm_device *bdev; + int ret = 0; + + mutex_lock(&ttm_global_mutex); + list_for_each_entry(bdev, &glob->device_list, device_list) { + ret = ttm_device_swapout(bdev, ctx, gfp_flags); + if (ret > 0) { + list_move_tail(&bdev->device_list, &glob->device_list); + break; + } + } + mutex_unlock(&ttm_global_mutex); + return ret; +} + +int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + gfp_t gfp_flags) +{ + struct ttm_resource_manager *man; + unsigned i; + s64 lret; + + for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) { + man = ttm_manager_type(bdev, i); + if (!man || !man->use_tt) + continue; + + lret = ttm_bo_swapout(bdev, ctx, man, gfp_flags, 1); + /* Can be both positive (num_pages) and negative (error) */ + if (lret) + return lret; + } + return 0; +} +EXPORT_SYMBOL(ttm_device_swapout); + +/** + * ttm_device_init + * + * @bdev: A pointer to a struct ttm_device to initialize. + * @funcs: Function table for the device. + * @dev: The core kernel device pointer for DMA mappings and allocations. + * @mapping: The address space to use for this bo. + * @vma_manager: A pointer to a vma manager. + * @alloc_flags: TTM_ALLOCATION_* flags. + * + * Initializes a struct ttm_device: + * Returns: + * !0: Failure. + */ +int ttm_device_init(struct ttm_device *bdev, const struct ttm_device_funcs *funcs, + struct device *dev, struct address_space *mapping, + struct drm_vma_offset_manager *vma_manager, + unsigned int alloc_flags) +{ + struct ttm_global *glob = &ttm_glob; + int ret, nid; + + if (WARN_ON(vma_manager == NULL)) + return -EINVAL; + + ret = ttm_global_init(); + if (ret) + return ret; + + bdev->wq = alloc_workqueue("ttm", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_UNBOUND, 16); + if (!bdev->wq) { + ttm_global_release(); + return -ENOMEM; + } + + bdev->alloc_flags = alloc_flags; + bdev->funcs = funcs; + + ttm_sys_man_init(bdev); + + if (dev) + nid = dev_to_node(dev); + else + nid = NUMA_NO_NODE; + + ttm_pool_init(&bdev->pool, dev, nid, alloc_flags); + + bdev->vma_manager = vma_manager; + spin_lock_init(&bdev->lru_lock); + INIT_LIST_HEAD(&bdev->unevictable); + bdev->dev_mapping = mapping; + mutex_lock(&ttm_global_mutex); + list_add_tail(&bdev->device_list, &glob->device_list); + mutex_unlock(&ttm_global_mutex); + + return 0; +} +EXPORT_SYMBOL(ttm_device_init); + +void ttm_device_fini(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man; + unsigned i; + + mutex_lock(&ttm_global_mutex); + list_del(&bdev->device_list); + mutex_unlock(&ttm_global_mutex); + + drain_workqueue(bdev->wq); + destroy_workqueue(bdev->wq); + + man = ttm_manager_type(bdev, TTM_PL_SYSTEM); + ttm_resource_manager_set_used(man, false); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL); + + spin_lock(&bdev->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + if (list_empty(&man->lru[0])) + pr_debug("Swap list %d was clean\n", i); + spin_unlock(&bdev->lru_lock); + + ttm_pool_fini(&bdev->pool); + ttm_global_release(); +} +EXPORT_SYMBOL(ttm_device_fini); + +static void ttm_device_clear_lru_dma_mappings(struct ttm_device *bdev, + struct list_head *list) +{ + struct ttm_resource *res; + + spin_lock(&bdev->lru_lock); + while ((res = ttm_lru_first_res_or_null(list))) { + struct ttm_buffer_object *bo = res->bo; + + /* Take ref against racing releases once lru_lock is unlocked */ + if (!ttm_bo_get_unless_zero(bo)) + continue; + + list_del_init(&bo->resource->lru.link); + spin_unlock(&bdev->lru_lock); + + if (bo->ttm) + ttm_tt_unpopulate(bo->bdev, bo->ttm); + + ttm_bo_put(bo); + spin_lock(&bdev->lru_lock); + } + spin_unlock(&bdev->lru_lock); +} + +void ttm_device_clear_dma_mappings(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man; + unsigned int i, j; + + ttm_device_clear_lru_dma_mappings(bdev, &bdev->unevictable); + + for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) { + man = ttm_manager_type(bdev, i); + if (!man || !man->use_tt) + continue; + + for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) + ttm_device_clear_lru_dma_mappings(bdev, &man->lru[j]); + } +} +EXPORT_SYMBOL(ttm_device_clear_dma_mappings); diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c new file mode 100644 index 000000000..bc7a83a9f --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#include <linux/export.h> + +#include <drm/ttm/ttm_execbuf_util.h> +#include <drm/ttm/ttm_bo.h> + +static void ttm_eu_backoff_reservation_reverse(struct list_head *list, + struct ttm_validate_buffer *entry) +{ + list_for_each_entry_continue_reverse(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + dma_resv_unlock(bo->base.resv); + } +} + +void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list) +{ + struct ttm_validate_buffer *entry; + + if (list_empty(list)) + return; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + ttm_bo_move_to_lru_tail_unlocked(bo); + dma_resv_unlock(bo->base.resv); + } + + if (ticket) + ww_acquire_fini(ticket); +} +EXPORT_SYMBOL(ttm_eu_backoff_reservation); + +/* + * Reserve buffers for validation. + * + * If a buffer in the list is marked for CPU access, we back off and + * wait for that buffer to become free for GPU access. + * + * If a buffer is reserved for another validation, the validator with + * the highest validation sequence backs off and waits for that buffer + * to become unreserved. This prevents deadlocks when validating multiple + * buffers in different orders. + */ + +int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list, bool intr, + struct list_head *dups) +{ + struct ttm_validate_buffer *entry; + int ret; + + if (list_empty(list)) + return 0; + + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + unsigned int num_fences; + + ret = ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); + if (ret == -EALREADY && dups) { + struct ttm_validate_buffer *safe = entry; + entry = list_prev_entry(entry, head); + list_del(&safe->head); + list_add(&safe->head, dups); + continue; + } + + num_fences = max(entry->num_shared, 1u); + if (!ret) { + ret = dma_resv_reserve_fences(bo->base.resv, + num_fences); + if (!ret) + continue; + } + + /* uh oh, we lost out, drop every reservation and try + * to only reserve this buffer, then start over if + * this succeeds. + */ + ttm_eu_backoff_reservation_reverse(list, entry); + + if (ret == -EDEADLK) { + ret = ttm_bo_reserve_slowpath(bo, intr, ticket); + } + + if (!ret) + ret = dma_resv_reserve_fences(bo->base.resv, + num_fences); + + if (unlikely(ret != 0)) { + if (ticket) { + ww_acquire_done(ticket); + ww_acquire_fini(ticket); + } + return ret; + } + + /* move this item to the front of the list, + * forces correct iteration of the loop without keeping track + */ + list_del(&entry->head); + list_add(&entry->head, list); + } + + return 0; +} +EXPORT_SYMBOL(ttm_eu_reserve_buffers); + +void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, + struct dma_fence *fence) +{ + struct ttm_validate_buffer *entry; + + if (list_empty(list)) + return; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + dma_resv_add_fence(bo->base.resv, fence, entry->num_shared ? + DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE); + ttm_bo_move_to_lru_tail_unlocked(bo); + dma_resv_unlock(bo->base.resv); + } + if (ticket) + ww_acquire_fini(ticket); +} +EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c new file mode 100644 index 000000000..aa137ead5 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + * Jerome Glisse + */ +#include <linux/module.h> +#include <linux/device.h> +#include <linux/pgtable.h> +#include <linux/sched.h> +#include <linux/debugfs.h> +#include <drm/drm_sysfs.h> +#include <drm/ttm/ttm_caching.h> + +#include "ttm_module.h" + +/** + * DOC: TTM + * + * TTM is a memory manager for accelerator devices with dedicated memory. + * + * The basic idea is that resources are grouped together in buffer objects of + * certain size and TTM handles lifetime, movement and CPU mappings of those + * objects. + * + * TODO: Add more design background and information here. + */ + +/** + * ttm_prot_from_caching - Modify the page protection according to the + * ttm cacing mode + * @caching: The ttm caching mode + * @tmp: The original page protection + * + * Return: The modified page protection + */ +pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp) +{ + /* Cached mappings need no adjustment */ + if (caching == ttm_cached) + return tmp; + +#if defined(__i386__) || defined(__x86_64__) + if (caching == ttm_write_combined) + tmp = pgprot_writecombine(tmp); +#ifndef CONFIG_UML + else if (boot_cpu_data.x86 > 3) + tmp = pgprot_noncached(tmp); +#endif /* CONFIG_UML */ +#endif /* __i386__ || __x86_64__ */ +#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ + defined(__powerpc__) || defined(__mips__) || defined(__loongarch__) || \ + defined(__riscv) + if (caching == ttm_write_combined) + tmp = pgprot_writecombine(tmp); + else + tmp = pgprot_noncached(tmp); +#endif +#if defined(__sparc__) + tmp = pgprot_noncached(tmp); +#endif + return tmp; +} + +MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse"); +MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/ttm_module.h b/drivers/gpu/drm/ttm/ttm_module.h new file mode 100644 index 000000000..767fe22ae --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_module.h @@ -0,0 +1,43 @@ +/************************************************************************** + * + * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#ifndef _TTM_MODULE_H_ +#define _TTM_MODULE_H_ + +#define TTM_PFX "[TTM] " + +struct dentry; +struct ttm_device; + +extern struct dentry *ttm_debugfs_root; + +void ttm_sys_man_init(struct ttm_device *bdev); + +#endif /* _TTM_MODULE_H_ */ diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c new file mode 100644 index 000000000..278bbe7a1 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -0,0 +1,1458 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +/* Pooling of allocated pages is necessary because changing the caching + * attributes on x86 of the linear mapping requires a costly cross CPU TLB + * invalidate for those addresses. + * + * Additional to that allocations from the DMA coherent API are pooled as well + * cause they are rather slow compared to alloc_pages+map. + */ + +#include <linux/export.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/debugfs.h> +#include <linux/highmem.h> +#include <linux/sched/mm.h> + +#ifdef CONFIG_X86 +#include <asm/set_memory.h> +#endif + +#include <drm/ttm/ttm_backup.h> +#include <drm/ttm/ttm_pool.h> +#include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_bo.h> + +#include "ttm_module.h" +#include "ttm_pool_internal.h" + +#ifdef CONFIG_FAULT_INJECTION +#include <linux/fault-inject.h> +static DECLARE_FAULT_ATTR(backup_fault_inject); +#else +#define should_fail(...) false +#endif + +/** + * struct ttm_pool_dma - Helper object for coherent DMA mappings + * + * @addr: original DMA address returned for the mapping + * @vaddr: original vaddr return for the mapping and order in the lower bits + */ +struct ttm_pool_dma { + dma_addr_t addr; + unsigned long vaddr; +}; + +/** + * struct ttm_pool_alloc_state - Current state of the tt page allocation process + * @pages: Pointer to the next tt page pointer to populate. + * @caching_divide: Pointer to the first page pointer whose page has a staged but + * not committed caching transition from write-back to @tt_caching. + * @dma_addr: Pointer to the next tt dma_address entry to populate if any. + * @remaining_pages: Remaining pages to populate. + * @tt_caching: The requested cpu-caching for the pages allocated. + */ +struct ttm_pool_alloc_state { + struct page **pages; + struct page **caching_divide; + dma_addr_t *dma_addr; + pgoff_t remaining_pages; + enum ttm_caching tt_caching; +}; + +/** + * struct ttm_pool_tt_restore - State representing restore from backup + * @pool: The pool used for page allocation while restoring. + * @snapshot_alloc: A snapshot of the most recent struct ttm_pool_alloc_state. + * @alloced_page: Pointer to the page most recently allocated from a pool or system. + * @first_dma: The dma address corresponding to @alloced_page if dma_mapping + * is requested. + * @alloced_pages: The number of allocated pages present in the struct ttm_tt + * page vector from this restore session. + * @restored_pages: The number of 4K pages restored for @alloced_page (which + * is typically a multi-order page). + * @page_caching: The struct ttm_tt requested caching + * @order: The order of @alloced_page. + * + * Recovery from backup might fail when we've recovered less than the + * full ttm_tt. In order not to loose any data (yet), keep information + * around that allows us to restart a failed ttm backup recovery. + */ +struct ttm_pool_tt_restore { + struct ttm_pool *pool; + struct ttm_pool_alloc_state snapshot_alloc; + struct page *alloced_page; + dma_addr_t first_dma; + pgoff_t alloced_pages; + pgoff_t restored_pages; + enum ttm_caching page_caching; + unsigned int order; +}; + +static unsigned long page_pool_size; + +MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool per NUMA node"); +module_param(page_pool_size, ulong, 0644); + +static unsigned long pool_node_limit[MAX_NUMNODES]; +static atomic_long_t allocated_pages[MAX_NUMNODES]; + +static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS]; +static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS]; + +static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS]; +static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS]; + +static spinlock_t shrinker_lock; +static struct list_head shrinker_list; +static struct shrinker *mm_shrinker; +static DECLARE_RWSEM(pool_shrink_rwsem); + +static int ttm_pool_nid(struct ttm_pool *pool) +{ + int nid = NUMA_NO_NODE; + if (pool) + nid = pool->nid; + if (nid == NUMA_NO_NODE) + nid = numa_node_id(); + return nid; +} + +/* Allocate pages of size 1 << order with the given gfp_flags */ +static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags, + unsigned int order) +{ + const unsigned int beneficial_order = ttm_pool_beneficial_order(pool); + unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; + struct ttm_pool_dma *dma; + struct page *p; + void *vaddr; + + /* Don't set the __GFP_COMP flag for higher order allocations. + * Mapping pages directly into an userspace process and calling + * put_page() on a TTM allocated page is illegal. + */ + if (order) + gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | + __GFP_THISNODE; + + /* + * Do not add latency to the allocation path for allocations orders + * device tolds us do not bring them additional performance gains. + */ + if (beneficial_order && order > beneficial_order) + gfp_flags &= ~__GFP_DIRECT_RECLAIM; + + if (!ttm_pool_uses_dma_alloc(pool)) { + p = alloc_pages_node(pool->nid, gfp_flags, order); + if (p) { + p->private = order; + mod_lruvec_page_state(p, NR_GPU_ACTIVE, 1 << order); + } + return p; + } + + dma = kmalloc_obj(*dma); + if (!dma) + return NULL; + + if (order) + attr |= DMA_ATTR_NO_WARN; + + vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE, + &dma->addr, gfp_flags, attr); + if (!vaddr) + goto error_free; + + /* TODO: This is an illegal abuse of the DMA API, but we need to rework + * TTM page fault handling and extend the DMA API to clean this up. + */ + if (is_vmalloc_addr(vaddr)) + p = vmalloc_to_page(vaddr); + else + p = virt_to_page(vaddr); + + dma->vaddr = (unsigned long)vaddr | order; + p->private = (unsigned long)dma; + return p; + +error_free: + kfree(dma); + return NULL; +} + +static void __free_pages_gpu_account(struct page *p, unsigned int order, + bool reclaim) +{ + mod_lruvec_page_state(p, reclaim ? NR_GPU_RECLAIM : NR_GPU_ACTIVE, + -(1 << order)); + __free_pages(p, order); +} + +/* Reset the caching and pages of size 1 << order */ +static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, + unsigned int order, struct page *p, bool reclaim) +{ + unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; + struct ttm_pool_dma *dma; + void *vaddr; + +#ifdef CONFIG_X86 + /* We don't care that set_pages_wb is inefficient here. This is only + * used when we have to shrink and CPU overhead is irrelevant then. + */ + if (caching != ttm_cached && !PageHighMem(p)) + set_pages_wb(p, 1 << order); +#endif + + if (!pool || !ttm_pool_uses_dma_alloc(pool)) { + __free_pages_gpu_account(p, order, reclaim); + return; + } + + if (order) + attr |= DMA_ATTR_NO_WARN; + + dma = (void *)p->private; + vaddr = (void *)(dma->vaddr & PAGE_MASK); + dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr, + attr); + kfree(dma); +} + +/* Apply any cpu-caching deferred during page allocation */ +static int ttm_pool_apply_caching(struct ttm_pool_alloc_state *alloc) +{ +#ifdef CONFIG_X86 + unsigned int num_pages = alloc->pages - alloc->caching_divide; + + if (!num_pages) + return 0; + + switch (alloc->tt_caching) { + case ttm_cached: + break; + case ttm_write_combined: + return set_pages_array_wc(alloc->caching_divide, num_pages); + case ttm_uncached: + return set_pages_array_uc(alloc->caching_divide, num_pages); + } +#endif + alloc->caching_divide = alloc->pages; + return 0; +} + +/* DMA Map pages of 1 << order size and return the resulting dma_address. */ +static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, + struct page *p, dma_addr_t *dma_addr) +{ + dma_addr_t addr; + + if (ttm_pool_uses_dma_alloc(pool)) { + struct ttm_pool_dma *dma = (void *)p->private; + + addr = dma->addr; + } else { + size_t size = (1ULL << order) * PAGE_SIZE; + + addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(pool->dev, addr)) + return -EFAULT; + } + + *dma_addr = addr; + + return 0; +} + +/* Unmap pages of 1 << order size */ +static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, + unsigned int num_pages) +{ + /* Unmapped while freeing the page */ + if (ttm_pool_uses_dma_alloc(pool)) + return; + + dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT, + DMA_BIDIRECTIONAL); +} + +/* Give pages into a specific pool_type */ +static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) +{ + unsigned int i, num_pages = 1 << pt->order; + int nid = page_to_nid(p); + + for (i = 0; i < num_pages; ++i) { + if (PageHighMem(p)) + clear_highpage(p + i); + else + clear_page(page_address(p + i)); + } + + INIT_LIST_HEAD(&p->lru); + rcu_read_lock(); + list_lru_add(&pt->pages, &p->lru, nid, NULL); + rcu_read_unlock(); + + atomic_long_add(num_pages, &allocated_pages[nid]); + mod_lruvec_page_state(p, NR_GPU_ACTIVE, -num_pages); + mod_lruvec_page_state(p, NR_GPU_RECLAIM, num_pages); +} + +static enum lru_status take_one_from_lru(struct list_head *item, + struct list_lru_one *list, + void *cb_arg) +{ + struct page **out_page = cb_arg; + struct page *p = container_of(item, struct page, lru); + list_lru_isolate(list, item); + + *out_page = p; + return LRU_REMOVED; +} + +/* Take pages from a specific pool_type, return NULL when nothing available */ +static struct page *ttm_pool_type_take(struct ttm_pool_type *pt, int nid) +{ + int ret; + struct page *p = NULL; + unsigned long nr_to_walk = 1; + + ret = list_lru_walk_node(&pt->pages, nid, take_one_from_lru, (void *)&p, &nr_to_walk); + if (ret == 1 && p) { + atomic_long_sub(1 << pt->order, &allocated_pages[nid]); + mod_lruvec_page_state(p, NR_GPU_ACTIVE, (1 << pt->order)); + mod_lruvec_page_state(p, NR_GPU_RECLAIM, -(1 << pt->order)); + } + return p; +} + +/* Initialize and add a pool type to the global shrinker list */ +static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool, + enum ttm_caching caching, unsigned int order) +{ + pt->pool = pool; + pt->caching = caching; + pt->order = order; + list_lru_init(&pt->pages); + + spin_lock(&shrinker_lock); + list_add_tail(&pt->shrinker_list, &shrinker_list); + spin_unlock(&shrinker_lock); +} + +static enum lru_status pool_move_to_dispose_list(struct list_head *item, + struct list_lru_one *list, + void *cb_arg) +{ + struct list_head *dispose = cb_arg; + + list_lru_isolate_move(list, item, dispose); + + return LRU_REMOVED; +} + +static void ttm_pool_dispose_list(struct ttm_pool_type *pt, + struct list_head *dispose) +{ + while (!list_empty(dispose)) { + struct page *p; + p = list_first_entry(dispose, struct page, lru); + list_del_init(&p->lru); + atomic_long_sub(1 << pt->order, &allocated_pages[page_to_nid(p)]); + ttm_pool_free_page(pt->pool, pt->caching, pt->order, p, true); + } +} + +/* Remove a pool_type from the global shrinker list and free all pages */ +static void ttm_pool_type_fini(struct ttm_pool_type *pt) +{ + LIST_HEAD(dispose); + + spin_lock(&shrinker_lock); + list_del(&pt->shrinker_list); + spin_unlock(&shrinker_lock); + + list_lru_walk(&pt->pages, pool_move_to_dispose_list, &dispose, LONG_MAX); + ttm_pool_dispose_list(pt, &dispose); +} + +/* Return the pool_type to use for the given caching and order */ +static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool, + enum ttm_caching caching, + unsigned int order) +{ + if (ttm_pool_uses_dma_alloc(pool)) + return &pool->caching[caching].orders[order]; + +#ifdef CONFIG_X86 + switch (caching) { + case ttm_write_combined: + if (ttm_pool_uses_dma32(pool)) + return &global_dma32_write_combined[order]; + + return &global_write_combined[order]; + case ttm_uncached: + if (ttm_pool_uses_dma32(pool)) + return &global_dma32_uncached[order]; + + return &global_uncached[order]; + default: + break; + } +#endif + + return NULL; +} + +/* Free pages using the per-node shrinker list */ +static unsigned int ttm_pool_shrink(int nid, unsigned long num_to_free) +{ + LIST_HEAD(dispose); + struct ttm_pool_type *pt; + unsigned int num_pages; + + down_read(&pool_shrink_rwsem); + spin_lock(&shrinker_lock); + pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list); + list_move_tail(&pt->shrinker_list, &shrinker_list); + spin_unlock(&shrinker_lock); + + num_pages = list_lru_walk_node(&pt->pages, nid, pool_move_to_dispose_list, &dispose, &num_to_free); + num_pages *= 1 << pt->order; + + ttm_pool_dispose_list(pt, &dispose); + up_read(&pool_shrink_rwsem); + + return num_pages; +} + +/* Return the allocation order based for a page */ +static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) +{ + if (ttm_pool_uses_dma_alloc(pool)) { + struct ttm_pool_dma *dma = (void *)p->private; + + return dma->vaddr & ~PAGE_MASK; + } + + return p->private; +} + +/* + * Split larger pages so that we can free each PAGE_SIZE page as soon + * as it has been backed up, in order to avoid memory pressure during + * reclaim. + */ +static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p) +{ + unsigned int order = ttm_pool_page_order(pool, p); + pgoff_t nr; + + if (!order) + return; + + split_page(p, order); + nr = 1UL << order; + while (nr--) + (p++)->private = 0; +} + +/** + * DOC: Partial backup and restoration of a struct ttm_tt. + * + * Swapout using ttm_backup_backup_page() and swapin using + * ttm_backup_copy_page() may fail. + * The former most likely due to lack of swap-space or memory, the latter due + * to lack of memory or because of signal interruption during waits. + * + * Backup failure is easily handled by using a ttm_tt pages vector that holds + * both backup handles and page pointers. This has to be taken into account when + * restoring such a ttm_tt from backup, and when freeing it while backed up. + * When restoring, for simplicity, new pages are actually allocated from the + * pool and the contents of any old pages are copied in and then the old pages + * are released. + * + * For restoration failures, the struct ttm_pool_tt_restore holds sufficient state + * to be able to resume an interrupted restore, and that structure is freed once + * the restoration is complete. If the struct ttm_tt is destroyed while there + * is a valid struct ttm_pool_tt_restore attached, that is also properly taken + * care of. + */ + +/* Is restore ongoing for the currently allocated page? */ +static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore) +{ + return restore && restore->restored_pages < (1 << restore->order); +} + +/* DMA unmap and free a multi-order page, either to the relevant pool or to system. */ +static pgoff_t ttm_pool_unmap_and_free(struct ttm_pool *pool, struct page *page, + const dma_addr_t *dma_addr, enum ttm_caching caching) +{ + struct ttm_pool_type *pt = NULL; + unsigned int order; + pgoff_t nr; + + if (pool) { + order = ttm_pool_page_order(pool, page); + nr = (1UL << order); + if (dma_addr) + ttm_pool_unmap(pool, *dma_addr, nr); + + pt = ttm_pool_select_type(pool, caching, order); + } else { + order = page->private; + nr = (1UL << order); + } + + if (pt) + ttm_pool_type_give(pt, page); + else + ttm_pool_free_page(pool, caching, order, page, false); + + return nr; +} + +/* Populate the page-array using the most recent allocated multi-order page. */ +static void ttm_pool_allocated_page_commit(struct page *allocated, + dma_addr_t first_dma, + struct ttm_pool_alloc_state *alloc, + pgoff_t nr) +{ + pgoff_t i; + + for (i = 0; i < nr; ++i) + *alloc->pages++ = allocated++; + + alloc->remaining_pages -= nr; + + if (!alloc->dma_addr) + return; + + for (i = 0; i < nr; ++i) { + *alloc->dma_addr++ = first_dma; + first_dma += PAGE_SIZE; + } +} + +/* + * When restoring, restore backed-up content to the newly allocated page and + * if successful, populate the page-table and dma-address arrays. + */ +static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore, + struct file *backup, + const struct ttm_operation_ctx *ctx, + struct ttm_pool_alloc_state *alloc) + +{ + pgoff_t i, nr = 1UL << restore->order; + struct page **first_page = alloc->pages; + struct page *p; + int ret = 0; + + for (i = restore->restored_pages; i < nr; ++i) { + p = first_page[i]; + if (ttm_backup_page_ptr_is_handle(p)) { + unsigned long handle = ttm_backup_page_ptr_to_handle(p); + gfp_t additional_gfp = ctx->gfp_retry_mayfail ? + __GFP_RETRY_MAYFAIL | __GFP_NOWARN : 0; + + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible && + should_fail(&backup_fault_inject, 1)) { + ret = -EINTR; + break; + } + + if (handle == 0) { + restore->restored_pages++; + continue; + } + + ret = ttm_backup_copy_page(backup, restore->alloced_page + i, + handle, ctx->interruptible, + additional_gfp); + if (ret) + break; + + ttm_backup_drop(backup, handle); + } else if (p) { + /* + * We could probably avoid splitting the old page + * using clever logic, but ATM we don't care, as + * we prioritize releasing memory ASAP. Note that + * here, the old retained page is always write-back + * cached. + */ + ttm_pool_split_for_swap(restore->pool, p); + copy_highpage(restore->alloced_page + i, p); + __free_pages_gpu_account(p, 0, false); + } + + restore->restored_pages++; + first_page[i] = ttm_backup_handle_to_page_ptr(0); + } + + if (ret) { + if (!restore->restored_pages) { + dma_addr_t *dma_addr = alloc->dma_addr ? &restore->first_dma : NULL; + + ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, + dma_addr, restore->page_caching); + restore->restored_pages = nr; + } + return ret; + } + + ttm_pool_allocated_page_commit(restore->alloced_page, restore->first_dma, + alloc, nr); + if (restore->page_caching == alloc->tt_caching || PageHighMem(restore->alloced_page)) + alloc->caching_divide = alloc->pages; + restore->snapshot_alloc = *alloc; + restore->alloced_pages += nr; + + return 0; +} + +/* If restoring, save information needed for ttm_pool_restore_commit(). */ +static void +ttm_pool_page_allocated_restore(struct ttm_pool *pool, unsigned int order, + struct page *p, + enum ttm_caching page_caching, + dma_addr_t first_dma, + struct ttm_pool_tt_restore *restore, + const struct ttm_pool_alloc_state *alloc) +{ + restore->pool = pool; + restore->order = order; + restore->restored_pages = 0; + restore->page_caching = page_caching; + restore->first_dma = first_dma; + restore->alloced_page = p; + restore->snapshot_alloc = *alloc; +} + +/* + * Called when we got a page, either from a pool or newly allocated. + * if needed, dma map the page and populate the dma address array. + * Populate the page address array. + * If the caching is consistent, update any deferred caching. Otherwise + * stage this page for an upcoming deferred caching update. + */ +static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, + struct page *p, enum ttm_caching page_caching, + struct ttm_pool_alloc_state *alloc, + struct ttm_pool_tt_restore *restore) +{ + bool caching_consistent; + dma_addr_t first_dma; + int r = 0; + + caching_consistent = (page_caching == alloc->tt_caching) || PageHighMem(p); + + if (caching_consistent) { + r = ttm_pool_apply_caching(alloc); + if (r) + return r; + } + + if (alloc->dma_addr) { + r = ttm_pool_map(pool, order, p, &first_dma); + if (r) + return r; + } + + if (restore) { + ttm_pool_page_allocated_restore(pool, order, p, page_caching, + first_dma, restore, alloc); + } else { + ttm_pool_allocated_page_commit(p, first_dma, alloc, 1UL << order); + + if (caching_consistent) + alloc->caching_divide = alloc->pages; + } + + return 0; +} + +/** + * ttm_pool_free_range() - Free a range of TTM pages + * @pool: The pool used for allocating. + * @tt: The struct ttm_tt holding the page pointers. + * @caching: The page caching mode used by the range. + * @start_page: index for first page to free. + * @end_page: index for last page to free + 1. + * + * During allocation the ttm_tt page-vector may be populated with ranges of + * pages with different attributes if allocation hit an error without being + * able to completely fulfill the allocation. This function can be used + * to free these individual ranges. + */ +static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, + enum ttm_caching caching, + pgoff_t start_page, pgoff_t end_page) +{ + struct page **pages = &tt->pages[start_page]; + struct file *backup = tt->backup; + pgoff_t i, nr; + + for (i = start_page; i < end_page; i += nr, pages += nr) { + struct page *p = *pages; + + nr = 1; + if (ttm_backup_page_ptr_is_handle(p)) { + unsigned long handle = ttm_backup_page_ptr_to_handle(p); + + if (handle != 0) + ttm_backup_drop(backup, handle); + } else if (p) { + dma_addr_t *dma_addr = tt->dma_address ? + tt->dma_address + i : NULL; + + nr = ttm_pool_unmap_and_free(pool, p, dma_addr, caching); + } + } +} + +static void ttm_pool_alloc_state_init(const struct ttm_tt *tt, + struct ttm_pool_alloc_state *alloc) +{ + alloc->pages = tt->pages; + alloc->caching_divide = tt->pages; + alloc->dma_addr = tt->dma_address; + alloc->remaining_pages = tt->num_pages; + alloc->tt_caching = tt->caching; +} + +/* + * Find a suitable allocation order based on highest desired order + * and number of remaining pages + */ +static unsigned int ttm_pool_alloc_find_order(unsigned int highest, + const struct ttm_pool_alloc_state *alloc) +{ + return min_t(unsigned int, highest, __fls(alloc->remaining_pages)); +} + +static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx, + struct ttm_pool_alloc_state *alloc, + struct ttm_pool_tt_restore *restore) +{ + enum ttm_caching page_caching; + gfp_t gfp_flags = GFP_USER; + pgoff_t caching_divide; + unsigned int order; + bool allow_pools; + struct page *p; + int r; + + WARN_ON(!alloc->remaining_pages || ttm_tt_is_populated(tt)); + WARN_ON(alloc->dma_addr && !pool->dev); + + if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (ctx->gfp_retry_mayfail) + gfp_flags |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + if (ttm_pool_uses_dma32(pool)) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= GFP_HIGHUSER; + + page_caching = tt->caching; + allow_pools = true; + for (order = ttm_pool_alloc_find_order(MAX_PAGE_ORDER, alloc); + alloc->remaining_pages; + order = ttm_pool_alloc_find_order(order, alloc)) { + struct ttm_pool_type *pt; + + /* First, try to allocate a page from a pool if one exists. */ + p = NULL; + pt = ttm_pool_select_type(pool, page_caching, order); + if (pt && allow_pools) + p = ttm_pool_type_take(pt, ttm_pool_nid(pool)); + + /* + * If that fails or previously failed, allocate from system. + * Note that this also disallows additional pool allocations using + * write-back cached pools of the same order. Consider removing + * that behaviour. + */ + if (!p) { + page_caching = ttm_cached; + allow_pools = false; + p = ttm_pool_alloc_page(pool, gfp_flags, order); + } + /* If that fails, lower the order if possible and retry. */ + if (!p) { + if (order) { + --order; + page_caching = tt->caching; + allow_pools = true; + continue; + } + r = -ENOMEM; + goto error_free_all; + } + r = ttm_pool_page_allocated(pool, order, p, page_caching, alloc, + restore); + if (r) + goto error_free_page; + + if (ttm_pool_restore_valid(restore)) { + r = ttm_pool_restore_commit(restore, tt->backup, ctx, alloc); + if (r) + goto error_free_all; + } + } + + r = ttm_pool_apply_caching(alloc); + if (r) + goto error_free_all; + + kfree(tt->restore); + tt->restore = NULL; + + return 0; + +error_free_page: + ttm_pool_free_page(pool, page_caching, order, p, false); + +error_free_all: + if (tt->restore) + return r; + + caching_divide = alloc->caching_divide - tt->pages; + ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide); + ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, + tt->num_pages - alloc->remaining_pages); + + return r; +} + +/** + * ttm_pool_alloc - Fill a ttm_tt object + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx) +{ + struct ttm_pool_alloc_state alloc; + + if (WARN_ON(ttm_tt_is_backed_up(tt))) + return -EINVAL; + + ttm_pool_alloc_state_init(tt, &alloc); + + return __ttm_pool_alloc(pool, tt, ctx, &alloc, NULL); +} +EXPORT_SYMBOL(ttm_pool_alloc); + +/** + * ttm_pool_restore_and_alloc - Fill a ttm_tt, restoring previously backed-up + * content. + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. Read in backed-up content. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx) +{ + struct ttm_pool_tt_restore *restore = tt->restore; + struct ttm_pool_alloc_state alloc; + + if (WARN_ON(!ttm_tt_is_backed_up(tt))) + return -EINVAL; + + if (!restore) { + gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; + + ttm_pool_alloc_state_init(tt, &alloc); + if (ctx->gfp_retry_mayfail) + gfp |= __GFP_RETRY_MAYFAIL; + + restore = kzalloc_obj(*restore, gfp); + if (!restore) + return -ENOMEM; + + restore->snapshot_alloc = alloc; + restore->pool = pool; + restore->restored_pages = 1; + + tt->restore = restore; + } else { + alloc = restore->snapshot_alloc; + if (ttm_pool_restore_valid(restore)) { + int ret = ttm_pool_restore_commit(restore, tt->backup, + ctx, &alloc); + + if (ret) + return ret; + } + if (!alloc.remaining_pages) + return 0; + } + + return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore); +} + +/** + * ttm_pool_free - Free the backing pages from a ttm_tt object + * + * @pool: Pool to give pages back to. + * @tt: ttm_tt object to unpopulate + * + * Give the packing pages back to a pool or free them + */ +void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) +{ + int nid = ttm_pool_nid(pool); + + ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages); + + while (atomic_long_read(&allocated_pages[nid]) > pool_node_limit[nid]) { + unsigned long diff = atomic_long_read(&allocated_pages[nid]) - pool_node_limit[nid]; + ttm_pool_shrink(nid, diff); + } +} +EXPORT_SYMBOL(ttm_pool_free); + +/** + * ttm_pool_drop_backed_up() - Release content of a swapped-out struct ttm_tt + * @tt: The struct ttm_tt. + * + * Release handles with associated content or any remaining pages of + * a backed-up struct ttm_tt. + */ +void ttm_pool_drop_backed_up(struct ttm_tt *tt) +{ + struct ttm_pool_tt_restore *restore; + pgoff_t start_page = 0; + + WARN_ON(!ttm_tt_is_backed_up(tt)); + + restore = tt->restore; + + /* + * Unmap and free any uncommitted restore page. + * any tt page-array backup entries already read back has + * been cleared already + */ + if (ttm_pool_restore_valid(restore)) { + dma_addr_t *dma_addr = tt->dma_address ? &restore->first_dma : NULL; + + ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, + dma_addr, restore->page_caching); + restore->restored_pages = 1UL << restore->order; + } + + /* + * If a restore is ongoing, part of the tt pages may have a + * caching different than writeback. + */ + if (restore) { + pgoff_t mid = restore->snapshot_alloc.caching_divide - tt->pages; + + start_page = restore->alloced_pages; + WARN_ON(mid > start_page); + /* Pages that might be dma-mapped and non-cached */ + ttm_pool_free_range(restore->pool, tt, tt->caching, + 0, mid); + /* Pages that might be dma-mapped but cached */ + ttm_pool_free_range(restore->pool, tt, ttm_cached, + mid, restore->alloced_pages); + kfree(restore); + tt->restore = NULL; + } + + ttm_pool_free_range(NULL, tt, ttm_cached, start_page, tt->num_pages); +} + +/** + * ttm_pool_backup() - Back up or purge a struct ttm_tt + * @pool: The pool used when allocating the struct ttm_tt. + * @tt: The struct ttm_tt. + * @flags: Flags to govern the backup behaviour. + * + * Back up or purge a struct ttm_tt. If @purge is true, then + * all pages will be freed directly to the system rather than to the pool + * they were allocated from, making the function behave similarly to + * ttm_pool_free(). If @purge is false the pages will be backed up instead, + * exchanged for handles. + * A subsequent call to ttm_pool_restore_and_alloc() will then read back the content and + * a subsequent call to ttm_pool_drop_backed_up() will drop it. + * If backup of a page fails for whatever reason, @ttm will still be + * partially backed up, retaining those pages for which backup fails. + * In that case, this function can be retried, possibly after freeing up + * memory resources. + * + * Return: Number of pages actually backed up or freed, or negative + * error code on error. + */ +long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_backup_flags *flags) +{ + struct file *backup = tt->backup; + struct page *page; + unsigned long handle; + gfp_t alloc_gfp; + gfp_t gfp; + int ret = 0; + pgoff_t shrunken = 0; + pgoff_t i, num_pages; + + if (WARN_ON(ttm_tt_is_backed_up(tt))) + return -EINVAL; + + if ((!ttm_backup_bytes_avail() && !flags->purge) || + ttm_pool_uses_dma_alloc(pool) || ttm_tt_is_backed_up(tt)) + return -EBUSY; + +#ifdef CONFIG_X86 + /* Anything returned to the system needs to be cached. */ + if (tt->caching != ttm_cached) + set_pages_array_wb(tt->pages, tt->num_pages); +#endif + + if (tt->dma_address || flags->purge) { + for (i = 0; i < tt->num_pages; i += num_pages) { + unsigned int order; + + page = tt->pages[i]; + if (unlikely(!page)) { + num_pages = 1; + continue; + } + + order = ttm_pool_page_order(pool, page); + num_pages = 1UL << order; + if (tt->dma_address) + ttm_pool_unmap(pool, tt->dma_address[i], + num_pages); + if (flags->purge) { + shrunken += num_pages; + page->private = 0; + __free_pages_gpu_account(page, order, false); + memset(tt->pages + i, 0, + num_pages * sizeof(*tt->pages)); + } + } + } + + if (flags->purge) + return shrunken; + + if (ttm_pool_uses_dma32(pool)) + gfp = GFP_DMA32; + else + gfp = GFP_HIGHUSER; + + alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL; + + num_pages = tt->num_pages; + + /* Pretend doing fault injection by shrinking only half of the pages. */ + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(&backup_fault_inject, 1)) + num_pages = DIV_ROUND_UP(num_pages, 2); + + for (i = 0; i < num_pages; ++i) { + s64 shandle; + + page = tt->pages[i]; + if (unlikely(!page)) + continue; + + ttm_pool_split_for_swap(pool, page); + + shandle = ttm_backup_backup_page(backup, page, flags->writeback, i, + gfp, alloc_gfp); + if (shandle < 0) { + /* We allow partially shrunken tts */ + ret = shandle; + break; + } + handle = shandle; + tt->pages[i] = ttm_backup_handle_to_page_ptr(handle); + __free_pages_gpu_account(page, 0, false); + shrunken++; + } + + return shrunken ? shrunken : ret; +} + +/** + * ttm_pool_init - Initialize a pool + * + * @pool: the pool to initialize + * @dev: device for DMA allocations and mappings + * @nid: NUMA node to use for allocations + * @alloc_flags: TTM_ALLOCATION_POOL_* flags + * + * Initialize the pool and its pool types. + */ +void ttm_pool_init(struct ttm_pool *pool, struct device *dev, + int nid, unsigned int alloc_flags) +{ + unsigned int i, j; + + WARN_ON(!dev && ttm_pool_uses_dma_alloc(pool)); + + pool->dev = dev; + pool->nid = nid; + pool->alloc_flags = alloc_flags; + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + for (j = 0; j < NR_PAGE_ORDERS; ++j) { + struct ttm_pool_type *pt; + + /* Initialize only pool types which are actually used */ + pt = ttm_pool_select_type(pool, i, j); + if (pt != &pool->caching[i].orders[j]) + continue; + + ttm_pool_type_init(pt, pool, i, j); + } + } +} +EXPORT_SYMBOL(ttm_pool_init); + +/** + * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete. + * + * This is useful to guarantee that all shrinker invocations have seen an + * update, before freeing memory, similar to rcu. + */ +static void ttm_pool_synchronize_shrinkers(void) +{ + down_write(&pool_shrink_rwsem); + up_write(&pool_shrink_rwsem); +} + +/** + * ttm_pool_fini - Cleanup a pool + * + * @pool: the pool to clean up + * + * Free all pages in the pool and unregister the types from the global + * shrinker. + */ +void ttm_pool_fini(struct ttm_pool *pool) +{ + unsigned int i, j; + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + for (j = 0; j < NR_PAGE_ORDERS; ++j) { + struct ttm_pool_type *pt; + + pt = ttm_pool_select_type(pool, i, j); + if (pt != &pool->caching[i].orders[j]) + continue; + + ttm_pool_type_fini(pt); + } + } + + /* We removed the pool types from the LRU, but we need to also make sure + * that no shrinker is concurrently freeing pages from the pool. + */ + ttm_pool_synchronize_shrinkers(); +} +EXPORT_SYMBOL(ttm_pool_fini); + +/* Free average pool number of pages. */ +#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS) + +static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_freed = 0; + + do + num_freed += ttm_pool_shrink(sc->nid, sc->nr_to_scan); + while (num_freed < sc->nr_to_scan && + atomic_long_read(&allocated_pages[sc->nid])); + + sc->nr_scanned = num_freed; + + return num_freed ?: SHRINK_STOP; +} + +/* Return the number of pages available or SHRINK_EMPTY if we have none */ +static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_pages = atomic_long_read(&allocated_pages[sc->nid]); + + return num_pages ? num_pages : SHRINK_EMPTY; +} + +#ifdef CONFIG_DEBUG_FS +/* Count the number of pages available in a pool_type */ +static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) +{ + return list_lru_count(&pt->pages); +} + +/* Print a nice header for the order */ +static void ttm_pool_debugfs_header(struct seq_file *m) +{ + unsigned int i; + + seq_puts(m, "\t "); + for (i = 0; i < NR_PAGE_ORDERS; ++i) + seq_printf(m, " ---%2u---", i); + seq_puts(m, "\n"); +} + +/* Dump information about the different pool types */ +static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, + struct seq_file *m) +{ + unsigned int i; + + for (i = 0; i < NR_PAGE_ORDERS; ++i) + seq_printf(m, " %8u", ttm_pool_type_count(&pt[i])); + seq_puts(m, "\n"); +} + +/* Dump the total amount of allocated pages */ +static void ttm_pool_debugfs_footer(struct seq_file *m) +{ + int nid; + + for_each_node(nid) { + seq_printf(m, "\ntotal node%d\t: %8lu of %8lu\n", nid, + atomic_long_read(&allocated_pages[nid]), pool_node_limit[nid]); + } +} + +/* Dump the information for the global pools */ +static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data) +{ + ttm_pool_debugfs_header(m); + + spin_lock(&shrinker_lock); + seq_puts(m, "wc\t:"); + ttm_pool_debugfs_orders(global_write_combined, m); + seq_puts(m, "uc\t:"); + ttm_pool_debugfs_orders(global_uncached, m); + seq_puts(m, "wc 32\t:"); + ttm_pool_debugfs_orders(global_dma32_write_combined, m); + seq_puts(m, "uc 32\t:"); + ttm_pool_debugfs_orders(global_dma32_uncached, m); + spin_unlock(&shrinker_lock); + + ttm_pool_debugfs_footer(m); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals); + +/** + * ttm_pool_debugfs - Debugfs dump function for a pool + * + * @pool: the pool to dump the information for + * @m: seq_file to dump to + * + * Make a debugfs dump with the per pool and global information. + */ +int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) +{ + unsigned int i; + + if (!ttm_pool_uses_dma_alloc(pool)) { + seq_puts(m, "unused\n"); + return 0; + } + + ttm_pool_debugfs_header(m); + + spin_lock(&shrinker_lock); + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + if (!ttm_pool_select_type(pool, i, 0)) + continue; + seq_puts(m, "DMA "); + switch (i) { + case ttm_cached: + seq_puts(m, "\t:"); + break; + case ttm_write_combined: + seq_puts(m, "wc\t:"); + break; + case ttm_uncached: + seq_puts(m, "uc\t:"); + break; + } + ttm_pool_debugfs_orders(pool->caching[i].orders, m); + } + spin_unlock(&shrinker_lock); + + ttm_pool_debugfs_footer(m); + return 0; +} +EXPORT_SYMBOL(ttm_pool_debugfs); + +/* Test the shrinker functions and dump the result */ +static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) +{ + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = TTM_SHRINKER_BATCH, + }; + unsigned long count; + int nid; + + fs_reclaim_acquire(GFP_KERNEL); + for_each_node(nid) { + sc.nid = nid; + count = ttm_pool_shrinker_count(mm_shrinker, &sc); + seq_printf(m, "%d: %lu/%lu\n", nid, count, + ttm_pool_shrinker_scan(mm_shrinker, &sc)); + } + fs_reclaim_release(GFP_KERNEL); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); + +#endif + +static inline u64 ttm_get_node_memory_size(int nid) +{ + /* + * This is directly using si_meminfo_node implementation as the + * function is not exported. + */ + int zone_type; + u64 managed_pages = 0; + + pg_data_t *pgdat = NODE_DATA(nid); + + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) + managed_pages += + zone_managed_pages(&pgdat->node_zones[zone_type]); + return managed_pages * PAGE_SIZE; +} + +/** + * ttm_pool_mgr_init - Initialize globals + * + * @num_pages: default number of pages + * + * Initialize the global locks and lists for the MM shrinker. + */ +int ttm_pool_mgr_init(unsigned long num_pages) +{ + unsigned int i; + + int nid; + for_each_node(nid) { + if (!page_pool_size) { + u64 node_size = ttm_get_node_memory_size(nid); + pool_node_limit[nid] = (node_size >> PAGE_SHIFT) / 2; + } else { + pool_node_limit[nid] = page_pool_size; + } + } + + spin_lock_init(&shrinker_lock); + INIT_LIST_HEAD(&shrinker_list); + + for (i = 0; i < NR_PAGE_ORDERS; ++i) { + ttm_pool_type_init(&global_write_combined[i], NULL, + ttm_write_combined, i); + ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i); + + ttm_pool_type_init(&global_dma32_write_combined[i], NULL, + ttm_write_combined, i); + ttm_pool_type_init(&global_dma32_uncached[i], NULL, + ttm_uncached, i); + } + +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL, + &ttm_pool_debugfs_globals_fops); + debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL, + &ttm_pool_debugfs_shrink_fops); +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("backup_fault_inject", ttm_debugfs_root, + &backup_fault_inject); +#endif +#endif + + mm_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE, "drm-ttm_pool"); + if (!mm_shrinker) + return -ENOMEM; + + mm_shrinker->count_objects = ttm_pool_shrinker_count; + mm_shrinker->scan_objects = ttm_pool_shrinker_scan; + mm_shrinker->batch = TTM_SHRINKER_BATCH; + mm_shrinker->seeks = 1; + + shrinker_register(mm_shrinker); + + return 0; +} + +/** + * ttm_pool_mgr_fini - Finalize globals + * + * Cleanup the global pools and unregister the MM shrinker. + */ +void ttm_pool_mgr_fini(void) +{ + unsigned int i; + + for (i = 0; i < NR_PAGE_ORDERS; ++i) { + ttm_pool_type_fini(&global_write_combined[i]); + ttm_pool_type_fini(&global_uncached[i]); + + ttm_pool_type_fini(&global_dma32_write_combined[i]); + ttm_pool_type_fini(&global_dma32_uncached[i]); + } + + shrinker_free(mm_shrinker); + WARN_ON(!list_empty(&shrinker_list)); +} diff --git a/drivers/gpu/drm/ttm/ttm_pool_internal.h b/drivers/gpu/drm/ttm/ttm_pool_internal.h new file mode 100644 index 000000000..24c179fd6 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_pool_internal.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* Copyright (c) 2025 Valve Corporation */ + +#ifndef _TTM_POOL_INTERNAL_H_ +#define _TTM_POOL_INTERNAL_H_ + +#include <drm/ttm/ttm_allocation.h> +#include <drm/ttm/ttm_pool.h> + +static inline bool ttm_pool_uses_dma_alloc(struct ttm_pool *pool) +{ + return pool->alloc_flags & TTM_ALLOCATION_POOL_USE_DMA_ALLOC; +} + +static inline bool ttm_pool_uses_dma32(struct ttm_pool *pool) +{ + return pool->alloc_flags & TTM_ALLOCATION_POOL_USE_DMA32; +} + +static inline unsigned int ttm_pool_beneficial_order(struct ttm_pool *pool) +{ + return pool->alloc_flags & 0xff; +} + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c new file mode 100644 index 000000000..b818836f0 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -0,0 +1,242 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_range_manager.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/drm_mm.h> + +#include <linux/export.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +/* + * Currently we use a spinlock for the lock, but a mutex *may* be + * more appropriate to reduce scheduling latency if the range manager + * ends up with very fragmented allocation patterns. + */ + +struct ttm_range_manager { + struct ttm_resource_manager manager; + struct drm_mm mm; + spinlock_t lock; +}; + +static inline struct ttm_range_manager * +to_range_manager(struct ttm_resource_manager *man) +{ + return container_of(man, struct ttm_range_manager, manager); +} + +static int ttm_range_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct ttm_range_manager *rman = to_range_manager(man); + struct ttm_range_mgr_node *node; + struct drm_mm *mm = &rman->mm; + enum drm_mm_insert_mode mode; + unsigned long lpfn; + int ret; + + lpfn = place->lpfn; + if (!lpfn) + lpfn = man->size; + + node = kzalloc_flex(*node, mm_nodes, 1); + if (!node) + return -ENOMEM; + + mode = DRM_MM_INSERT_BEST; + if (place->flags & TTM_PL_FLAG_TOPDOWN) + mode = DRM_MM_INSERT_HIGH; + + ttm_resource_init(bo, place, &node->base); + + spin_lock(&rman->lock); + ret = drm_mm_insert_node_in_range(mm, &node->mm_nodes[0], + PFN_UP(node->base.size), + bo->page_alignment, 0, + place->fpfn, lpfn, mode); + spin_unlock(&rman->lock); + + if (unlikely(ret)) { + ttm_resource_fini(man, &node->base); + kfree(node); + return ret; + } + + node->base.start = node->mm_nodes[0].start; + *res = &node->base; + return 0; +} + +static void ttm_range_man_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res); + struct ttm_range_manager *rman = to_range_manager(man); + + spin_lock(&rman->lock); + drm_mm_remove_node(&node->mm_nodes[0]); + spin_unlock(&rman->lock); + + ttm_resource_fini(man, res); + kfree(node); +} + +static bool ttm_range_man_intersects(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct drm_mm_node *node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + u32 num_pages = PFN_UP(size); + + /* Don't evict BOs outside of the requested placement range */ + if (place->fpfn >= (node->start + num_pages) || + (place->lpfn && place->lpfn <= node->start)) + return false; + + return true; +} + +static bool ttm_range_man_compatible(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct drm_mm_node *node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + u32 num_pages = PFN_UP(size); + + if (node->start < place->fpfn || + (place->lpfn && (node->start + num_pages) > place->lpfn)) + return false; + + return true; +} + +static void ttm_range_man_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct ttm_range_manager *rman = to_range_manager(man); + + spin_lock(&rman->lock); + drm_mm_print(&rman->mm, printer); + spin_unlock(&rman->lock); +} + +static const struct ttm_resource_manager_func ttm_range_manager_func = { + .alloc = ttm_range_man_alloc, + .free = ttm_range_man_free, + .intersects = ttm_range_man_intersects, + .compatible = ttm_range_man_compatible, + .debug = ttm_range_man_debug +}; + +/** + * ttm_range_man_init_nocheck - Initialise a generic range manager for the + * selected memory type. + * + * @bdev: ttm device + * @type: memory manager type + * @use_tt: if the memory manager uses tt + * @p_size: size of area to be managed in pages. + * + * The range manager is installed for this device in the type slot. + * + * Return: %0 on success or a negative error code on failure + */ +int ttm_range_man_init_nocheck(struct ttm_device *bdev, + unsigned type, bool use_tt, + unsigned long p_size) +{ + struct ttm_resource_manager *man; + struct ttm_range_manager *rman; + + rman = kzalloc_obj(*rman); + if (!rman) + return -ENOMEM; + + man = &rman->manager; + man->use_tt = use_tt; + + man->func = &ttm_range_manager_func; + + ttm_resource_manager_init(man, bdev, p_size); + + drm_mm_init(&rman->mm, 0, p_size); + spin_lock_init(&rman->lock); + + ttm_set_driver_manager(bdev, type, &rman->manager); + ttm_resource_manager_set_used(man, true); + return 0; +} +EXPORT_SYMBOL(ttm_range_man_init_nocheck); + +/** + * ttm_range_man_fini_nocheck - Remove the generic range manager from a slot + * and tear it down. + * + * @bdev: ttm device + * @type: memory manager type + * + * Return: %0 on success or a negative error code on failure + */ +int ttm_range_man_fini_nocheck(struct ttm_device *bdev, + unsigned type) +{ + struct ttm_resource_manager *man = ttm_manager_type(bdev, type); + struct ttm_range_manager *rman = to_range_manager(man); + struct drm_mm *mm = &rman->mm; + int ret; + + if (!man) + return 0; + + ttm_resource_manager_set_used(man, false); + + ret = ttm_resource_manager_evict_all(bdev, man); + if (ret) + return ret; + + spin_lock(&rman->lock); + drm_mm_takedown(mm); + spin_unlock(&rman->lock); + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(bdev, type, NULL); + kfree(rman); + return 0; +} +EXPORT_SYMBOL(ttm_range_man_fini_nocheck); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c new file mode 100644 index 000000000..154d67392 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -0,0 +1,955 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/io-mapping.h> +#include <linux/iosys-map.h> +#include <linux/scatterlist.h> +#include <linux/cgroup_dmem.h> + +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_tt.h> + +#include <drm/drm_print.h> +#include <drm/drm_util.h> + +/* Detach the cursor from the bulk move list */ +static void +ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor) +{ + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + cursor->bulk = NULL; + list_del_init(&cursor->bulk_link); +} + +/* Move the cursor to the end of the bulk move list it's in */ +static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk, + struct ttm_resource_cursor *cursor) +{ + struct ttm_lru_bulk_move_pos *pos; + + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + if (WARN_ON_ONCE(bulk != cursor->bulk)) { + list_del_init(&cursor->bulk_link); + return; + } + + pos = &bulk->pos[cursor->mem_type][cursor->priority]; + if (pos->last) + list_move(&cursor->hitch.link, &pos->last->lru.link); + ttm_resource_cursor_clear_bulk(cursor); +} + +/* Move all cursors attached to a bulk move to its end */ +static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk) +{ + struct ttm_resource_cursor *cursor, *next; + + list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link) + ttm_resource_cursor_move_bulk_tail(bulk, cursor); +} + +/* Remove a cursor from an empty bulk move list */ +static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk) +{ + struct ttm_resource_cursor *cursor, *next; + + list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link) + ttm_resource_cursor_clear_bulk(cursor); +} + +/** + * ttm_resource_cursor_init() - Initialize a struct ttm_resource_cursor + * @cursor: The cursor to initialize. + * @man: The resource manager. + * + * Initialize the cursor before using it for iteration. + */ +void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, + struct ttm_resource_manager *man) +{ + cursor->priority = 0; + cursor->man = man; + ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH); + INIT_LIST_HEAD(&cursor->bulk_link); + INIT_LIST_HEAD(&cursor->hitch.link); +} + +/** + * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage + * @cursor: The struct ttm_resource_cursor to finalize. + * + * The function pulls the LRU list cursor off any lists it was previously + * attached to. Needs to be called with the LRU lock held. The function + * can be called multiple times after each other. + */ +void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor) +{ + lockdep_assert_held(&cursor->man->bdev->lru_lock); + list_del_init(&cursor->hitch.link); + ttm_resource_cursor_clear_bulk(cursor); +} + +/** + * ttm_lru_bulk_move_init - initialize a bulk move structure + * @bulk: the structure to init + * + * For now just memset the structure to zero. + */ +void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk) +{ + memset(bulk, 0, sizeof(*bulk)); + INIT_LIST_HEAD(&bulk->cursor_list); +} +EXPORT_SYMBOL(ttm_lru_bulk_move_init); + +/** + * ttm_lru_bulk_move_fini - finalize a bulk move structure + * @bdev: The struct ttm_device + * @bulk: the structure to finalize + * + * Sanity checks that bulk moves don't have any + * resources left and hence no cursors attached. + */ +void ttm_lru_bulk_move_fini(struct ttm_device *bdev, + struct ttm_lru_bulk_move *bulk) +{ + spin_lock(&bdev->lru_lock); + ttm_bulk_move_drop_cursors(bulk); + spin_unlock(&bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_lru_bulk_move_fini); + +/** + * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail. + * + * @bulk: bulk move structure + * + * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that + * resource order never changes. Should be called with &ttm_device.lru_lock held. + */ +void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk) +{ + unsigned i, j; + + ttm_bulk_move_adjust_cursors(bulk); + for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) { + for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) { + struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j]; + struct ttm_resource_manager *man; + + if (!pos->first) + continue; + + lockdep_assert_held(&pos->first->bo->bdev->lru_lock); + dma_resv_assert_held(pos->first->bo->base.resv); + dma_resv_assert_held(pos->last->bo->base.resv); + + man = ttm_manager_type(pos->first->bo->bdev, i); + list_bulk_move_tail(&man->lru[j], &pos->first->lru.link, + &pos->last->lru.link); + } + } +} +EXPORT_SYMBOL(ttm_lru_bulk_move_tail); + +/* Return the bulk move pos object for this resource */ +static struct ttm_lru_bulk_move_pos * +ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res) +{ + return &bulk->pos[res->mem_type][res->bo->priority]; +} + +/* Return the previous resource on the list (skip over non-resource list items) */ +static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur) +{ + struct ttm_lru_item *lru = &cur->lru; + + do { + lru = list_prev_entry(lru, link); + } while (!ttm_lru_item_is_res(lru)); + + return ttm_lru_item_to_res(lru); +} + +/* Return the next resource on the list (skip over non-resource list items) */ +static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur) +{ + struct ttm_lru_item *lru = &cur->lru; + + do { + lru = list_next_entry(lru, link); + } while (!ttm_lru_item_is_res(lru)); + + return ttm_lru_item_to_res(lru); +} + +/* Move the resource to the tail of the bulk move range */ +static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos, + struct ttm_resource *res) +{ + if (pos->last != res) { + if (pos->first == res) + pos->first = ttm_lru_next_res(res); + list_move(&res->lru.link, &pos->last->lru.link); + pos->last = res; + } +} + +/* Add the resource to a bulk_move cursor */ +static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk, + struct ttm_resource *res) +{ + struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); + + if (!pos->first) { + pos->first = res; + pos->last = res; + } else { + WARN_ON(pos->first->bo->base.resv != res->bo->base.resv); + ttm_lru_bulk_move_pos_tail(pos, res); + } +} + +/* Remove the resource from a bulk_move range */ +static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk, + struct ttm_resource *res) +{ + struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); + + if (unlikely(WARN_ON(!pos->first || !pos->last) || + (pos->first == res && pos->last == res))) { + pos->first = NULL; + pos->last = NULL; + } else if (pos->first == res) { + pos->first = ttm_lru_next_res(res); + } else if (pos->last == res) { + pos->last = ttm_lru_prev_res(res); + } else { + list_move(&res->lru.link, &pos->last->lru.link); + } +} + +static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_object *bo) +{ + /* + * Take care when creating a new resource for a bo, that it is not considered + * swapped if it's not the current resource for the bo and is thus logically + * associated with the ttm_tt. Think a VRAM resource created to move a + * swapped-out bo to VRAM. + */ + if (bo->resource != res || !bo->ttm) + return false; + + dma_resv_assert_held(bo->base.resv); + return ttm_tt_is_swapped(bo->ttm); +} + +static bool ttm_resource_unevictable(struct ttm_resource *res, struct ttm_buffer_object *bo) +{ + return bo->pin_count || ttm_resource_is_swapped(res, bo); +} + +/* Add the resource to a bulk move if the BO is configured for it */ +void ttm_resource_add_bulk_move(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + if (bo->bulk_move && !ttm_resource_unevictable(res, bo)) + ttm_lru_bulk_move_add(bo->bulk_move, res); +} + +/* Remove the resource from a bulk move if the BO is configured for it */ +void ttm_resource_del_bulk_move(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + if (bo->bulk_move && !ttm_resource_unevictable(res, bo)) + ttm_lru_bulk_move_del(bo->bulk_move, res); +} + +/* + * Remove a resource from its bulk_move, bypassing the unevictable check. + * Use only when the resource is known to still be tracked in the range despite + * the BO having just become unevictable; asserts that this is the case. + */ +void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + WARN_ON_ONCE(!ttm_resource_unevictable(res, bo)); + if (bo->bulk_move) + ttm_lru_bulk_move_del(bo->bulk_move, res); +} + +/* Move a resource to the LRU or bulk tail */ +void ttm_resource_move_to_lru_tail(struct ttm_resource *res) +{ + struct ttm_buffer_object *bo = res->bo; + struct ttm_device *bdev = bo->bdev; + + lockdep_assert_held(&bo->bdev->lru_lock); + + if (ttm_resource_unevictable(res, bo)) { + list_move_tail(&res->lru.link, &bdev->unevictable); + + } else if (bo->bulk_move) { + struct ttm_lru_bulk_move_pos *pos = + ttm_lru_bulk_move_pos(bo->bulk_move, res); + + ttm_lru_bulk_move_pos_tail(pos, res); + } else { + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, res->mem_type); + list_move_tail(&res->lru.link, &man->lru[bo->priority]); + } +} + +/** + * ttm_resource_init - resource object constructor + * @bo: buffer object this resource is allocated for + * @place: placement of the resource + * @res: the resource object to initialize + * + * Initialize a new resource object. Counterpart of ttm_resource_fini(). + */ +void ttm_resource_init(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *res) +{ + struct ttm_resource_manager *man; + + res->start = 0; + res->size = bo->base.size; + res->mem_type = place->mem_type; + res->placement = place->flags; + res->bus.addr = NULL; + res->bus.offset = 0; + res->bus.is_iomem = false; + res->bus.caching = ttm_cached; + res->bo = bo; + + man = ttm_manager_type(bo->bdev, place->mem_type); + spin_lock(&bo->bdev->lru_lock); + if (ttm_resource_unevictable(res, bo)) + list_add_tail(&res->lru.link, &bo->bdev->unevictable); + else + list_add_tail(&res->lru.link, &man->lru[bo->priority]); + man->usage += res->size; + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_resource_init); + +/** + * ttm_resource_fini - resource destructor + * @man: the resource manager this resource belongs to + * @res: the resource to clean up + * + * Should be used by resource manager backends to clean up the TTM resource + * objects before freeing the underlying structure. Makes sure the resource is + * removed from the LRU before destruction. + * Counterpart of ttm_resource_init(). + */ +void ttm_resource_fini(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct ttm_device *bdev = man->bdev; + + spin_lock(&bdev->lru_lock); + list_del_init(&res->lru.link); + man->usage -= res->size; + spin_unlock(&bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_resource_fini); + +int ttm_resource_alloc(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res_ptr, + struct dmem_cgroup_pool_state **ret_limit_pool) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, place->mem_type); + struct dmem_cgroup_pool_state *pool = NULL; + int ret; + + if (man->cg) { + ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool); + if (ret) { + if (ret == -EAGAIN) + ret = -ENOSPC; + return ret; + } + } + + ret = man->func->alloc(man, bo, place, res_ptr); + if (ret) { + if (pool) + dmem_cgroup_uncharge(pool, bo->base.size); + return ret; + } + + (*res_ptr)->css = pool; + + spin_lock(&bo->bdev->lru_lock); + ttm_resource_add_bulk_move(*res_ptr, bo); + spin_unlock(&bo->bdev->lru_lock); + return 0; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc); + +void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) +{ + struct ttm_resource_manager *man; + struct dmem_cgroup_pool_state *pool; + + if (!*res) + return; + + spin_lock(&bo->bdev->lru_lock); + ttm_resource_del_bulk_move(*res, bo); + spin_unlock(&bo->bdev->lru_lock); + + pool = (*res)->css; + man = ttm_manager_type(bo->bdev, (*res)->mem_type); + man->func->free(man, *res); + *res = NULL; + if (man->cg) + dmem_cgroup_uncharge(pool, bo->base.size); +} +EXPORT_SYMBOL(ttm_resource_free); + +/** + * ttm_resource_intersects - test for intersection + * + * @bdev: TTM device structure + * @res: The resource to test + * @place: The placement to test + * @size: How many bytes the new allocation needs. + * + * Test if @res intersects with @place and @size. Used for testing if evictions + * are valuable or not. + * + * Returns true if the res placement intersects with @place and @size. + */ +bool ttm_resource_intersects(struct ttm_device *bdev, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, res->mem_type); + if (!place || !man->func->intersects) + return true; + + return man->func->intersects(man, res, place, size); +} + +/** + * ttm_resource_compatible - check if resource is compatible with placement + * + * @res: the resource to check + * @placement: the placement to check against + * @evicting: true if the caller is doing evictions + * + * Returns true if the placement is compatible. + */ +bool ttm_resource_compatible(struct ttm_resource *res, + struct ttm_placement *placement, + bool evicting) +{ + struct ttm_buffer_object *bo = res->bo; + struct ttm_device *bdev = bo->bdev; + unsigned i; + + if (res->placement & TTM_PL_FLAG_TEMPORARY) + return false; + + for (i = 0; i < placement->num_placement; i++) { + const struct ttm_place *place = &placement->placement[i]; + struct ttm_resource_manager *man; + + if (res->mem_type != place->mem_type) + continue; + + if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) + continue; + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS && + !(res->placement & TTM_PL_FLAG_CONTIGUOUS)) + continue; + + man = ttm_manager_type(bdev, res->mem_type); + if (man->func->compatible && + !man->func->compatible(man, res, place, bo->base.size)) + continue; + + return true; + } + return false; +} + +void ttm_resource_set_bo(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + spin_lock(&bo->bdev->lru_lock); + res->bo = bo; + spin_unlock(&bo->bdev->lru_lock); +} + +/** + * ttm_resource_manager_init + * + * @man: memory manager object to init + * @bdev: ttm device this manager belongs to + * @size: size of managed resources in arbitrary units + * + * Initialize core parts of a manager object. + */ +void ttm_resource_manager_init(struct ttm_resource_manager *man, + struct ttm_device *bdev, + uint64_t size) +{ + unsigned i; + + man->bdev = bdev; + man->size = size; + man->usage = 0; + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&man->lru[i]); + spin_lock_init(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) + man->eviction_fences[i] = NULL; +} +EXPORT_SYMBOL(ttm_resource_manager_init); + +/* + * ttm_resource_manager_evict_all + * + * @bdev: device to use + * @man: manager to use + * + * Evict all the objects out of a memory manager until it is empty. + * Part of memory manager cleanup sequence. + */ +int ttm_resource_manager_evict_all(struct ttm_device *bdev, + struct ttm_resource_manager *man) +{ + struct ttm_operation_ctx ctx = { }; + struct dma_fence *fence; + int ret, i; + + do { + ret = ttm_bo_evict_first(bdev, man, &ctx); + cond_resched(); + } while (!ret); + + if (ret && ret != -ENOENT) + return ret; + + ret = 0; + + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (fence && !dma_fence_is_signaled(fence)) { + dma_fence_get(fence); + spin_unlock(&man->eviction_lock); + ret = dma_fence_wait(fence, false); + dma_fence_put(fence); + if (ret) + return ret; + spin_lock(&man->eviction_lock); + } + } + spin_unlock(&man->eviction_lock); + + return ret; +} +EXPORT_SYMBOL(ttm_resource_manager_evict_all); + +/** + * ttm_resource_manager_usage + * + * @man: A memory manager object. + * + * Return how many resources are currently used. + */ +uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man) +{ + uint64_t usage; + + if (WARN_ON_ONCE(!man->bdev)) + return 0; + + spin_lock(&man->bdev->lru_lock); + usage = man->usage; + spin_unlock(&man->bdev->lru_lock); + return usage; +} +EXPORT_SYMBOL(ttm_resource_manager_usage); + +/** + * ttm_resource_manager_debug + * + * @man: manager type to dump. + * @p: printer to use for debug. + */ +void ttm_resource_manager_debug(struct ttm_resource_manager *man, + struct drm_printer *p) +{ + drm_printf(p, " use_type: %d\n", man->use_type); + drm_printf(p, " use_tt: %d\n", man->use_tt); + drm_printf(p, " size: %llu\n", man->size); + drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man)); + if (man->func->debug) + man->func->debug(man, p); +} +EXPORT_SYMBOL(ttm_resource_manager_debug); + +static void +ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, + struct ttm_lru_item *next_lru) +{ + struct ttm_resource *next = ttm_lru_item_to_res(next_lru); + struct ttm_lru_bulk_move *bulk; + + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + bulk = next->bo->bulk_move; + + if (cursor->bulk != bulk) { + if (bulk) { + list_move_tail(&cursor->bulk_link, &bulk->cursor_list); + cursor->mem_type = next->mem_type; + } else { + list_del_init(&cursor->bulk_link); + } + cursor->bulk = bulk; + } +} + +/** + * ttm_resource_manager_first() - Start iterating over the resources + * of a resource manager + * @cursor: cursor to record the position + * + * Initializes the cursor and starts iterating. When done iterating, + * the caller must explicitly call ttm_resource_cursor_fini(). + * + * Return: The first resource from the resource manager. + */ +struct ttm_resource * +ttm_resource_manager_first(struct ttm_resource_cursor *cursor) +{ + struct ttm_resource_manager *man = cursor->man; + + if (WARN_ON_ONCE(!man)) + return NULL; + + lockdep_assert_held(&man->bdev->lru_lock); + + list_move(&cursor->hitch.link, &man->lru[cursor->priority]); + return ttm_resource_manager_next(cursor); +} + +/** + * ttm_resource_manager_next() - Continue iterating over the resource manager + * resources + * @cursor: cursor to record the position + * + * Return: the next resource from the resource manager. + */ +struct ttm_resource * +ttm_resource_manager_next(struct ttm_resource_cursor *cursor) +{ + struct ttm_resource_manager *man = cursor->man; + struct ttm_lru_item *lru; + + lockdep_assert_held(&man->bdev->lru_lock); + + for (;;) { + lru = &cursor->hitch; + list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) { + if (ttm_lru_item_is_res(lru)) { + ttm_resource_cursor_check_bulk(cursor, lru); + list_move(&cursor->hitch.link, &lru->link); + return ttm_lru_item_to_res(lru); + } + } + + if (++cursor->priority >= TTM_MAX_BO_PRIORITY) + break; + + list_move(&cursor->hitch.link, &man->lru[cursor->priority]); + ttm_resource_cursor_clear_bulk(cursor); + } + + return NULL; +} + +/** + * ttm_lru_first_res_or_null() - Return the first resource on an lru list + * @head: The list head of the lru list. + * + * Return: Pointer to the first resource on the lru list or NULL if + * there is none. + */ +struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head) +{ + struct ttm_lru_item *lru; + + list_for_each_entry(lru, head, link) { + if (ttm_lru_item_is_res(lru)) + return ttm_lru_item_to_res(lru); + } + + return NULL; +} + +static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter, + struct iosys_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_iomap *iter_io = + container_of(iter, typeof(*iter_io), base); + void __iomem *addr; + +retry: + while (i >= iter_io->cache.end) { + iter_io->cache.sg = iter_io->cache.sg ? + sg_next(iter_io->cache.sg) : iter_io->st->sgl; + iter_io->cache.i = iter_io->cache.end; + iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >> + PAGE_SHIFT; + iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) - + iter_io->start; + } + + if (i < iter_io->cache.i) { + iter_io->cache.end = 0; + iter_io->cache.sg = NULL; + goto retry; + } + + addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs + + (((resource_size_t)i - iter_io->cache.i) + << PAGE_SHIFT)); + iosys_map_set_vaddr_iomem(dmap, addr); +} + +static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter, + struct iosys_map *map) +{ + io_mapping_unmap_local(map->vaddr_iomem); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = { + .map_local = ttm_kmap_iter_iomap_map_local, + .unmap_local = ttm_kmap_iter_iomap_unmap_local, + .maps_tt = false, +}; + +/** + * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap + * @iter_io: The struct ttm_kmap_iter_iomap to initialize. + * @iomap: The struct io_mapping representing the underlying linear io_memory. + * @st: sg_table into @iomap, representing the memory of the struct + * ttm_resource. + * @start: Offset that needs to be subtracted from @st to make + * sg_dma_address(st->sgl) - @start == 0 for @iomap start. + * + * Return: Pointer to the embedded struct ttm_kmap_iter. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, + struct io_mapping *iomap, + struct sg_table *st, + resource_size_t start) +{ + iter_io->base.ops = &ttm_kmap_iter_io_ops; + iter_io->iomap = iomap; + iter_io->st = st; + iter_io->start = start; + memset(&iter_io->cache, 0, sizeof(iter_io->cache)); + + return &iter_io->base; +} +EXPORT_SYMBOL(ttm_kmap_iter_iomap_init); + +/** + * DOC: Linear io iterator + * + * This code should die in the not too near future. Best would be if we could + * make io-mapping use memremap for all io memory, and have memremap + * implement a kmap_local functionality. We could then strip a huge amount of + * code. These linear io iterators are implemented to mimic old functionality, + * and they don't use kmap_local semantics at all internally. Rather ioremap or + * friends, and at least on 32-bit they add global TLB flushes and points + * of failure. + */ + +static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter, + struct iosys_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_linear_io *iter_io = + container_of(iter, typeof(*iter_io), base); + + *dmap = iter_io->dmap; + iosys_map_incr(dmap, i * PAGE_SIZE); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = { + .map_local = ttm_kmap_iter_linear_io_map_local, + .maps_tt = false, +}; + +/** + * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory + * @iter_io: The iterator to initialize + * @bdev: The TTM device + * @mem: The ttm resource representing the iomap. + * + * This function is for internal TTM use only. It sets up a memcpy kmap iterator + * pointing at a linear chunk of io memory. + * + * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on + * failure. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, + struct ttm_device *bdev, + struct ttm_resource *mem) +{ + int ret; + + ret = ttm_mem_io_reserve(bdev, mem); + if (ret) + goto out_err; + if (!mem->bus.is_iomem) { + ret = -EINVAL; + goto out_io_free; + } + + if (mem->bus.addr) { + iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); + iter_io->needs_unmap = false; + } else { + iter_io->needs_unmap = true; + memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); + if (mem->bus.caching == ttm_write_combined) + iosys_map_set_vaddr_iomem(&iter_io->dmap, + ioremap_wc(mem->bus.offset, + mem->size)); + else if (mem->bus.caching == ttm_cached) + iosys_map_set_vaddr(&iter_io->dmap, + memremap(mem->bus.offset, mem->size, + MEMREMAP_WB | + MEMREMAP_WT | + MEMREMAP_WC)); + + /* If uncached requested or if mapping cached or wc failed */ + if (iosys_map_is_null(&iter_io->dmap)) + iosys_map_set_vaddr_iomem(&iter_io->dmap, + ioremap(mem->bus.offset, + mem->size)); + + if (iosys_map_is_null(&iter_io->dmap)) { + ret = -ENOMEM; + goto out_io_free; + } + } + + iter_io->base.ops = &ttm_kmap_iter_linear_io_ops; + return &iter_io->base; + +out_io_free: + ttm_mem_io_free(bdev, mem); +out_err: + return ERR_PTR(ret); +} + +/** + * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory + * @iter_io: The iterator to finalize + * @bdev: The TTM device + * @mem: The ttm resource representing the iomap. + * + * This function is for internal TTM use only. It cleans up a memcpy kmap + * iterator initialized by ttm_kmap_iter_linear_io_init. + */ +void +ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, + struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) { + if (iter_io->dmap.is_iomem) + iounmap(iter_io->dmap.vaddr_iomem); + else + memunmap(iter_io->dmap.vaddr); + } + + ttm_mem_io_free(bdev, mem); +} + +#if defined(CONFIG_DEBUG_FS) + +static int ttm_resource_manager_show(struct seq_file *m, void *unused) +{ + struct ttm_resource_manager *man = + (struct ttm_resource_manager *)m->private; + struct drm_printer p = drm_seq_file_printer(m); + ttm_resource_manager_debug(man, &p); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager); + +#endif + +/** + * ttm_resource_manager_create_debugfs - Create debugfs entry for specified + * resource manager. + * @man: The TTM resource manager for which the debugfs stats file to be created + * @parent: debugfs directory in which the file will reside + * @name: The filename to create. + * + * This function sets up a debugfs file that can be used to look + * at debug statistics of the specified ttm_resource_manager. + */ +void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, + struct dentry *parent, + const char *name) +{ +#if defined(CONFIG_DEBUG_FS) + debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops); +#endif +} +EXPORT_SYMBOL(ttm_resource_manager_create_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_sys_manager.c b/drivers/gpu/drm/ttm/ttm_sys_manager.c new file mode 100644 index 000000000..04398bb9c --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_sys_manager.c @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_placement.h> +#include <linux/slab.h> + +#include "ttm_module.h" + +static int ttm_sys_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + *res = kzalloc_obj(**res); + if (!*res) + return -ENOMEM; + + ttm_resource_init(bo, place, *res); + return 0; +} + +static void ttm_sys_man_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + ttm_resource_fini(man, res); + kfree(res); +} + +static const struct ttm_resource_manager_func ttm_sys_manager_func = { + .alloc = ttm_sys_man_alloc, + .free = ttm_sys_man_free, +}; + +void ttm_sys_man_init(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man = &bdev->sysman; + + /* + * Initialize the system memory buffer type. + * Other types need to be driver / IOCTL initialized. + */ + man->use_tt = true; + man->func = &ttm_sys_manager_func; + + ttm_resource_manager_init(man, bdev, 0); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); + ttm_resource_manager_set_used(man, true); +} diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c new file mode 100644 index 000000000..b645a1818 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -0,0 +1,565 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include <linux/cc_platform.h> +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/file.h> +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/shmem_fs.h> +#include <drm/drm_cache.h> +#include <drm/drm_device.h> +#include <drm/drm_print.h> +#include <drm/drm_util.h> +#include <drm/ttm/ttm_backup.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_tt.h> + +#include "ttm_module.h" +#include "ttm_pool_internal.h" + +static unsigned long ttm_pages_limit; + +MODULE_PARM_DESC(pages_limit, "Limit for the allocated pages"); +module_param_named(pages_limit, ttm_pages_limit, ulong, 0644); + +static unsigned long ttm_dma32_pages_limit; + +MODULE_PARM_DESC(dma32_pages_limit, "Limit for the allocated DMA32 pages"); +module_param_named(dma32_pages_limit, ttm_dma32_pages_limit, ulong, 0644); + +static atomic_long_t ttm_pages_allocated; +static atomic_long_t ttm_dma32_pages_allocated; + +/* + * Allocates a ttm structure for the given BO. + */ +int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) +{ + struct ttm_device *bdev = bo->bdev; + struct drm_device *ddev = bo->base.dev; + uint32_t page_flags = 0; + + dma_resv_assert_held(bo->base.resv); + + if (bo->ttm) + return 0; + + switch (bo->type) { + case ttm_bo_type_device: + if (zero_alloc) + page_flags |= TTM_TT_FLAG_ZERO_ALLOC; + break; + case ttm_bo_type_kernel: + break; + case ttm_bo_type_sg: + page_flags |= TTM_TT_FLAG_EXTERNAL; + break; + default: + pr_err("Illegal buffer object type\n"); + return -EINVAL; + } + /* + * When using dma_alloc_coherent with memory encryption the + * mapped TT pages need to be decrypted or otherwise the drivers + * will end up sending encrypted mem to the gpu. + */ + if (ttm_pool_uses_dma_alloc(&bdev->pool) && + cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { + page_flags |= TTM_TT_FLAG_DECRYPTED; + drm_info_once(ddev, "TT memory decryption enabled."); + } + + bo->ttm = bdev->funcs->ttm_tt_create(bo, page_flags); + if (unlikely(bo->ttm == NULL)) + return -ENOMEM; + + WARN_ON(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE && + !(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)); + + return 0; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_create); + +/* + * Allocates storage for pointers to the pages that back the ttm. + */ +static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->pages = kvcalloc(ttm->num_pages, sizeof(void*), GFP_KERNEL); + if (!ttm->pages) + return -ENOMEM; + + return 0; +} + +static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->pages = kvcalloc(ttm->num_pages, sizeof(*ttm->pages) + + sizeof(*ttm->dma_address), GFP_KERNEL); + if (!ttm->pages) + return -ENOMEM; + + ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); + return 0; +} + +static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->dma_address = kvzalloc_objs(*ttm->dma_address, ttm->num_pages); + if (!ttm->dma_address) + return -ENOMEM; + + return 0; +} + +void ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) +{ + bdev->funcs->ttm_tt_destroy(bdev, ttm); +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_destroy); + +static void ttm_tt_init_fields(struct ttm_tt *ttm, + struct ttm_buffer_object *bo, + uint32_t page_flags, + enum ttm_caching caching, + unsigned long extra_pages) +{ + ttm->num_pages = (PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT) + extra_pages; + ttm->page_flags = page_flags; + ttm->dma_address = NULL; + ttm->swap_storage = NULL; + ttm->sg = bo->sg; + ttm->caching = caching; + ttm->restore = NULL; + ttm->backup = NULL; +} + +int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + uint32_t page_flags, enum ttm_caching caching, + unsigned long extra_pages) +{ + ttm_tt_init_fields(ttm, bo, page_flags, caching, extra_pages); + + if (ttm_tt_alloc_page_directory(ttm)) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_tt_init); + +void ttm_tt_fini(struct ttm_tt *ttm) +{ + WARN_ON(ttm->page_flags & TTM_TT_FLAG_PRIV_POPULATED); + + if (ttm->swap_storage) + fput(ttm->swap_storage); + ttm->swap_storage = NULL; + + if (ttm_tt_is_backed_up(ttm)) + ttm_pool_drop_backed_up(ttm); + if (ttm->backup) { + ttm_backup_fini(ttm->backup); + ttm->backup = NULL; + } + + if (ttm->pages) + kvfree(ttm->pages); + else + kvfree(ttm->dma_address); + ttm->pages = NULL; + ttm->dma_address = NULL; +} +EXPORT_SYMBOL(ttm_tt_fini); + +int ttm_sg_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + uint32_t page_flags, enum ttm_caching caching) +{ + int ret; + + ttm_tt_init_fields(ttm, bo, page_flags, caching, 0); + + if (page_flags & TTM_TT_FLAG_EXTERNAL) + ret = ttm_sg_tt_alloc_page_directory(ttm); + else + ret = ttm_dma_tt_alloc_page_directory(ttm); + if (ret) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_sg_tt_init); + +int ttm_tt_swapin(struct ttm_tt *ttm) +{ + struct address_space *swap_space; + struct file *swap_storage; + struct page *from_page; + struct page *to_page; + gfp_t gfp_mask; + int i, ret; + + swap_storage = ttm->swap_storage; + BUG_ON(swap_storage == NULL); + + swap_space = swap_storage->f_mapping; + gfp_mask = mapping_gfp_mask(swap_space); + + for (i = 0; i < ttm->num_pages; ++i) { + from_page = shmem_read_mapping_page_gfp(swap_space, i, + gfp_mask); + if (IS_ERR(from_page)) { + ret = PTR_ERR(from_page); + goto out_err; + } + to_page = ttm->pages[i]; + if (unlikely(to_page == NULL)) { + ret = -ENOMEM; + goto out_err; + } + + copy_highpage(to_page, from_page); + put_page(from_page); + } + + fput(swap_storage); + ttm->swap_storage = NULL; + ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; + + return 0; + +out_err: + return ret; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_swapin); + +/** + * ttm_tt_backup() - Helper to back up a struct ttm_tt. + * @bdev: The TTM device. + * @tt: The struct ttm_tt. + * @flags: Flags that govern the backup behaviour. + * + * Update the page accounting and call ttm_pool_shrink_tt to free pages + * or back them up. + * + * Return: Number of pages freed or swapped out, or negative error code on + * error. + */ +long ttm_tt_backup(struct ttm_device *bdev, struct ttm_tt *tt, + const struct ttm_backup_flags flags) +{ + long ret; + + if (WARN_ON(IS_ERR_OR_NULL(tt->backup))) + return 0; + + ret = ttm_pool_backup(&bdev->pool, tt, &flags); + if (ret > 0) { + tt->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED; + tt->page_flags |= TTM_TT_FLAG_BACKED_UP; + } + + return ret; +} + +int ttm_tt_restore(struct ttm_device *bdev, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx) +{ + int ret = ttm_pool_restore_and_alloc(&bdev->pool, tt, ctx); + + if (ret) + return ret; + + tt->page_flags &= ~TTM_TT_FLAG_BACKED_UP; + + return 0; +} +EXPORT_SYMBOL(ttm_tt_restore); + +/** + * ttm_tt_swapout - swap out tt object + * + * @bdev: TTM device structure. + * @ttm: The struct ttm_tt. + * @gfp_flags: Flags to use for memory allocation. + * + * Swapout a TT object to a shmem_file, return number of pages swapped out or + * negative error code. + */ +int ttm_tt_swapout(struct ttm_device *bdev, struct ttm_tt *ttm, + gfp_t gfp_flags) +{ + loff_t size = (loff_t)ttm->num_pages << PAGE_SHIFT; + struct address_space *swap_space; + struct file *swap_storage; + struct page *from_page; + struct page *to_page; + int i, ret; + + swap_storage = shmem_file_setup("ttm swap", size, EMPTY_VMA_FLAGS); + if (IS_ERR(swap_storage)) { + pr_err("Failed allocating swap storage\n"); + return PTR_ERR(swap_storage); + } + + swap_space = swap_storage->f_mapping; + gfp_flags &= mapping_gfp_mask(swap_space); + + for (i = 0; i < ttm->num_pages; ++i) { + from_page = ttm->pages[i]; + if (unlikely(from_page == NULL)) + continue; + + to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_flags); + if (IS_ERR(to_page)) { + ret = PTR_ERR(to_page); + goto out_err; + } + copy_highpage(to_page, from_page); + set_page_dirty(to_page); + mark_page_accessed(to_page); + put_page(to_page); + } + + ttm_tt_unpopulate(bdev, ttm); + ttm->swap_storage = swap_storage; + ttm->page_flags |= TTM_TT_FLAG_SWAPPED; + + return ttm->num_pages; + +out_err: + fput(swap_storage); + + return ret; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_swapout); + +int ttm_tt_populate(struct ttm_device *bdev, + struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) +{ + int ret; + + if (!ttm) + return -EINVAL; + + if (ttm_tt_is_populated(ttm)) + return 0; + + if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + atomic_long_add(ttm->num_pages, &ttm_pages_allocated); + if (ttm_pool_uses_dma32(&bdev->pool)) + atomic_long_add(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + + while (atomic_long_read(&ttm_pages_allocated) > ttm_pages_limit || + atomic_long_read(&ttm_dma32_pages_allocated) > + ttm_dma32_pages_limit) { + + ret = ttm_global_swapout(ctx, GFP_KERNEL); + if (ret == 0) + break; + if (ret < 0) + goto error; + } + + if (bdev->funcs->ttm_tt_populate) + ret = bdev->funcs->ttm_tt_populate(bdev, ttm, ctx); + else + ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); + if (ret) + goto error; + + ttm->page_flags |= TTM_TT_FLAG_PRIV_POPULATED; + ttm->page_flags &= ~TTM_TT_FLAG_BACKED_UP; + if (unlikely(ttm->page_flags & TTM_TT_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_tt_unpopulate(bdev, ttm); + return ret; + } + } + + return 0; + +error: + if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); + if (ttm_pool_uses_dma32(&bdev->pool)) + atomic_long_sub(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + return ret; +} + +#if IS_ENABLED(CONFIG_DRM_TTM_KUNIT_TEST) +EXPORT_SYMBOL(ttm_tt_populate); +#endif + +void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) +{ + if (!ttm_tt_is_populated(ttm)) + return; + + if (bdev->funcs->ttm_tt_unpopulate) + bdev->funcs->ttm_tt_unpopulate(bdev, ttm); + else + ttm_pool_free(&bdev->pool, ttm); + + if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { + atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); + if (ttm_pool_uses_dma32(&bdev->pool)) + atomic_long_sub(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + + ttm->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED; +} +EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_unpopulate); + +#ifdef CONFIG_DEBUG_FS + +/* Test the shrinker functions and dump the result */ +static int ttm_tt_debugfs_shrink_show(struct seq_file *m, void *data) +{ + struct ttm_operation_ctx ctx = { }; + + seq_printf(m, "%d\n", ttm_global_swapout(&ctx, GFP_KERNEL)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_tt_debugfs_shrink); + +#endif + + +/* + * ttm_tt_mgr_init - register with the MM shrinker + * + * Register with the MM shrinker for swapping out BOs. + */ +void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages) +{ +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("tt_shrink", 0400, ttm_debugfs_root, NULL, + &ttm_tt_debugfs_shrink_fops); +#endif + + if (!ttm_pages_limit) + ttm_pages_limit = num_pages; + + if (!ttm_dma32_pages_limit) + ttm_dma32_pages_limit = num_dma32_pages; +} + +static void ttm_kmap_iter_tt_map_local(struct ttm_kmap_iter *iter, + struct iosys_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_tt *iter_tt = + container_of(iter, typeof(*iter_tt), base); + + iosys_map_set_vaddr(dmap, kmap_local_page_prot(iter_tt->tt->pages[i], + iter_tt->prot)); +} + +static void ttm_kmap_iter_tt_unmap_local(struct ttm_kmap_iter *iter, + struct iosys_map *map) +{ + kunmap_local(map->vaddr); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_tt_ops = { + .map_local = ttm_kmap_iter_tt_map_local, + .unmap_local = ttm_kmap_iter_tt_unmap_local, + .maps_tt = true, +}; + +/** + * ttm_kmap_iter_tt_init - Initialize a struct ttm_kmap_iter_tt + * @iter_tt: The struct ttm_kmap_iter_tt to initialize. + * @tt: Struct ttm_tt holding page pointers of the struct ttm_resource. + * + * Return: Pointer to the embedded struct ttm_kmap_iter. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt, + struct ttm_tt *tt) +{ + iter_tt->base.ops = &ttm_kmap_iter_tt_ops; + iter_tt->tt = tt; + if (tt) + iter_tt->prot = ttm_prot_from_caching(tt->caching, PAGE_KERNEL); + else + iter_tt->prot = PAGE_KERNEL; + + return &iter_tt->base; +} +EXPORT_SYMBOL(ttm_kmap_iter_tt_init); + +unsigned long ttm_tt_pages_limit(void) +{ + return ttm_pages_limit; +} +EXPORT_SYMBOL(ttm_tt_pages_limit); + +/** + * ttm_tt_setup_backup() - Allocate and assign a backup structure for a ttm_tt + * @tt: The ttm_tt for wich to allocate and assign a backup structure. + * + * Assign a backup structure to be used for tt backup. This should + * typically be done at bo creation, to avoid allocations at shrinking + * time. + * + * Return: 0 on success, negative error code on failure. + */ +int ttm_tt_setup_backup(struct ttm_tt *tt) +{ + struct file *backup = + ttm_backup_shmem_create(((loff_t)tt->num_pages) << PAGE_SHIFT); + + if (WARN_ON_ONCE(!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))) + return -EINVAL; + + if (IS_ERR(backup)) + return PTR_ERR(backup); + + if (tt->backup) + ttm_backup_fini(tt->backup); + + tt->backup = backup; + return 0; +} +EXPORT_SYMBOL(ttm_tt_setup_backup); |