1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2025, Advanced Micro Devices, Inc.
*/
#include <drm/amdxdna_accel.h>
#include <drm/drm_managed.h>
#include <linux/iommu.h>
#include <linux/iova.h>
#include "amdxdna_gem.h"
#include "amdxdna_pci_drv.h"
static bool force_iova;
module_param(force_iova, bool, 0600);
MODULE_PARM_DESC(force_iova, "Force use IOVA (Default false)");
static struct iova *amdxdna_iommu_alloc_iova(struct amdxdna_dev *xdna,
size_t size,
dma_addr_t *dma_addr,
bool size_aligned)
{
unsigned long shift, end;
struct iova *iova;
end = xdna->domain->geometry.aperture_end;
shift = iova_shift(&xdna->iovad);
size = iova_align(&xdna->iovad, size);
iova = alloc_iova(&xdna->iovad, size >> shift, end >> shift, size_aligned);
if (!iova)
return ERR_PTR(-ENOMEM);
*dma_addr = iova_dma_addr(&xdna->iovad, iova);
return iova;
}
int amdxdna_dma_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
{
unsigned long contig_sz;
struct sg_table *sgt;
dma_addr_t dma_addr;
struct iova *iova;
ssize_t size;
if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHARE)
return 0;
sgt = drm_gem_shmem_get_pages_sgt(&abo->base);
if (IS_ERR(sgt)) {
XDNA_ERR(xdna, "Get sgt failed, ret %ld", PTR_ERR(sgt));
return PTR_ERR(sgt);
}
if (!sgt->orig_nents) {
XDNA_ERR(xdna, "sgl is zero length");
return -EOPNOTSUPP;
}
if (amdxdna_iova_on(xdna)) {
if (!sg_page(sgt->sgl)) {
XDNA_ERR(xdna, "sgl is not page backed");
return -EOPNOTSUPP;
}
iova = amdxdna_iommu_alloc_iova(xdna, abo->mem.size, &dma_addr,
(abo->type == AMDXDNA_BO_DEV_HEAP));
if (IS_ERR(iova)) {
XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
return PTR_ERR(iova);
}
size = iommu_map_sgtable(xdna->domain, dma_addr, sgt,
IOMMU_READ | IOMMU_WRITE);
if (size < 0) {
XDNA_ERR(xdna, "iommu_map_sgtable failed: %zd", size);
__free_iova(&xdna->iovad, iova);
return size;
}
if (size < abo->mem.size) {
iommu_unmap(xdna->domain, dma_addr, size);
__free_iova(&xdna->iovad, iova);
return -ENXIO;
}
abo->mem.dma_addr = dma_addr;
} else {
/* Device doesn't support scatter/gather list, fail non-contiguous mapping. */
contig_sz = drm_prime_get_contiguous_size(sgt);
if (contig_sz < abo->mem.size) {
XDNA_ERR(xdna,
"noncontiguous dma addr, contig size:%ld, expected size:%ld",
contig_sz, abo->mem.size);
return -EINVAL;
}
abo->mem.dma_addr = sg_dma_address(sgt->sgl);
}
return 0;
}
void amdxdna_dma_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
{
size_t size;
if (abo->mem.dma_addr == AMDXDNA_INVALID_ADDR)
return;
if (amdxdna_iova_on(xdna)) {
size = iova_align(&xdna->iovad, abo->mem.size);
iommu_unmap(xdna->domain, abo->mem.dma_addr, size);
free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, abo->mem.dma_addr));
}
abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
}
void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr)
{
struct iova *iova;
void *cpu_addr;
int ret;
iova = amdxdna_iommu_alloc_iova(xdna, size, dma_addr, true);
if (IS_ERR(iova)) {
XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
return iova;
}
cpu_addr = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
if (!cpu_addr) {
ret = -ENOMEM;
goto free_iova;
}
ret = iommu_map(xdna->domain, *dma_addr, virt_to_phys(cpu_addr),
iova_align(&xdna->iovad, size),
IOMMU_READ | IOMMU_WRITE, GFP_KERNEL);
if (ret)
goto free_cpu_addr;
return cpu_addr;
free_cpu_addr:
free_pages((unsigned long)cpu_addr, get_order(size));
free_iova:
__free_iova(&xdna->iovad, iova);
return ERR_PTR(ret);
}
void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size,
void *cpu_addr, dma_addr_t dma_addr)
{
iommu_unmap(xdna->domain, dma_addr, iova_align(&xdna->iovad, size));
free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, dma_addr));
free_pages((unsigned long)cpu_addr, get_order(size));
}
static void amdxdna_cleanup_force_iova(struct drm_device *dev, void *res)
{
struct amdxdna_dev *xdna = to_xdna_dev(dev);
if (xdna->domain) {
iommu_detach_group(xdna->domain, xdna->group);
put_iova_domain(&xdna->iovad);
iova_cache_put();
iommu_domain_free(xdna->domain);
}
iommu_group_put(xdna->group);
}
void amdxdna_iommu_fini(struct amdxdna_dev *xdna)
{
if (xdna->group && !xdna->domain)
iommu_group_put(xdna->group);
}
int amdxdna_iommu_init(struct amdxdna_dev *xdna)
{
unsigned long order;
int ret = 0;
xdna->group = iommu_group_get(xdna->ddev.dev);
if (!xdna->group || !force_iova)
return 0;
XDNA_WARN(xdna, "Enabled force_iova mode.");
xdna->domain = iommu_paging_domain_alloc_flags(xdna->ddev.dev,
IOMMU_HWPT_ALLOC_PASID);
if (IS_ERR(xdna->domain)) {
XDNA_ERR(xdna, "Failed to alloc iommu domain");
ret = PTR_ERR(xdna->domain);
goto put_group;
}
ret = iova_cache_get();
if (ret)
goto free_domain;
order = __ffs(xdna->domain->pgsize_bitmap);
init_iova_domain(&xdna->iovad, 1UL << order, 0);
ret = iommu_attach_group(xdna->domain, xdna->group);
if (ret)
goto put_iova;
ret = drmm_add_action(&xdna->ddev, amdxdna_cleanup_force_iova, NULL);
if (ret)
goto detach_group;
return 0;
detach_group:
iommu_detach_group(xdna->domain, xdna->group);
put_iova:
put_iova_domain(&xdna->iovad);
iova_cache_put();
free_domain:
iommu_domain_free(xdna->domain);
put_group:
iommu_group_put(xdna->group);
xdna->group = NULL;
xdna->domain = NULL;
return ret;
}
|
